#!/usr/bin/env node import { constants as fsConstants } from "node:fs"; import { access, readFile, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { spawn } from "node:child_process"; import { DEV_DB_ENV_CONTRACT, summarizeDbContract } from "../../internal/cloud/db-contract.mjs"; import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../../internal/protocol/index.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const reportPath = "reports/dev-gate/dev-deploy-report.json"; const namespace = "hwlab-dev"; const healthPath = "/health/live"; const d601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml"; const devTemplateJobReplacementPolicy = { status: "active", namespace, scope: "DEV-only suspended template Jobs", allowedJobs: [ { name: "hwlab-agent-worker-template", serviceId: "hwlab-agent-worker", reason: "suspended session worker template Job has immutable spec.template" }, { name: "hwlab-cli-template", serviceId: "hwlab-cli", reason: "suspended CLI template Job has immutable spec.template" } ] }; const replaceableDevTemplateJobNames = new Set(devTemplateJobReplacementPolicy.allowedJobs.map((job) => job.name)); const requiredValidationCommands = [ "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs", "node --check scripts/dev-deploy-apply.mjs", "node --check scripts/src/dev-deploy-apply.mjs", "node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked" ]; const applyCommand = "node scripts/dev-deploy-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report"; const dryRunPlanCommand = "node scripts/dev-deploy-apply.mjs --dry-run --write-report"; const blockedDryRunCommand = "node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --write-report"; const forbiddenActions = [ "prod-deploy", "secret-read", "runtime-substitute", "unidesk-restart", "code-queue-restart", "backend-core-restart", "heavy-master-e2e", "force-push" ]; const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"]; function parseArgs(argv) { const flags = new Set(argv.filter((arg) => arg.startsWith("--"))); return { apply: flags.has("--apply"), dryRun: !flags.has("--apply"), expectBlocked: flags.has("--expect-blocked"), skipLiveProbe: flags.has("--skip-live-probe"), writeReport: flags.has("--write-report"), flags }; } function addBlocker(blockers, type, scope, summary) { const key = `${type}::${scope}`; if (!blockers.some((blocker) => `${blocker.type}::${blocker.scope}` === key)) { blockers.push({ type, scope, status: "open", summary: oneLine(summary) }); } } function oneLine(value) { return String(value).replace(/\s+/g, " ").trim(); } async function readJson(relativePath, blockers) { try { return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8")); } catch (error) { addBlocker(blockers, "contract_blocker", relativePath, `Cannot parse ${relativePath}: ${error.message}`); return null; } } async function readText(relativePath) { return readFile(path.join(repoRoot, relativePath), "utf8"); } async function commandResult(command, args, timeoutMs = 15000, options = {}) { return new Promise((resolve) => { const child = spawn(command, args, { cwd: repoRoot, env: options.env ?? process.env, stdio: ["pipe", "pipe", "pipe"] }); const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs); let stdout = ""; let stderr = ""; child.stdin.on("error", () => {}); child.stdin.end(options.input ?? ""); child.stdout.on("data", (chunk) => { stdout += chunk; }); child.stderr.on("data", (chunk) => { stderr += chunk; }); child.on("error", (error) => { clearTimeout(timer); resolve({ ok: false, code: null, stdout, stderr: error.message }); }); child.on("close", (code, signal) => { clearTimeout(timer); resolve({ ok: code === 0, code, signal, stdout: stdout.trim(), stderr: stderr.trim() }); }); }); } async function findExecutable(name) { for (const dir of (process.env.PATH || "").split(path.delimiter)) { if (!dir) continue; const candidate = path.join(dir, name); try { await access(candidate, fsConstants.X_OK); return candidate; } catch {} } return null; } async function resolveD601Kubectl() { const kubectlPath = await findExecutable("kubectl"); const commandPrefix = `KUBECONFIG=${d601KubeconfigPath} kubectl`; const base = { executor: kubectlPath, kubeconfig: d601KubeconfigPath, commandPrefix, env: { ...process.env, KUBECONFIG: d601KubeconfigPath } }; if (!kubectlPath) { return { ...base, status: "blocked", reason: "kubectl is not installed in the runner" }; } try { await access(d601KubeconfigPath, fsConstants.R_OK); } catch { return { ...base, status: "blocked", reason: `D601 k3s kubeconfig is not readable at ${d601KubeconfigPath}` }; } return { ...base, status: "ready", reason: "D601 k3s kubeconfig is readable" }; } function kubectlCommand(kubectl, args) { return `${kubectl.commandPrefix} ${args.join(" ")}`; } async function kubectlResult(kubectl, args, timeoutMs = 15000, options = {}) { if (kubectl.status !== "ready") { return { ok: false, code: null, stdout: "", stderr: kubectl.reason, blocked: true }; } return commandResult(kubectl.executor, args, timeoutMs, { ...options, env: kubectl.env }); } function listItems(document) { if (!document) return []; return document.kind === "List" ? document.items ?? [] : [document]; } function containersFor(item) { return item?.spec?.template?.spec?.containers ?? []; } function serviceIdFor(item, container) { return ( item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || item?.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || container?.name ); } function itemNamespace(item) { return item?.metadata?.namespace ?? (item?.kind === "Namespace" ? item?.metadata?.name : namespace); } function uniqueStrings(values) { return [...new Set(values.filter((value) => typeof value === "string" && value.length > 0))]; } function imageRecordsFor(item) { return containersFor(item).map((container) => ({ name: container.name, serviceId: serviceIdFor(item, container), image: container.image })); } function primaryImage(images) { if (!images.length) return null; if (images.length === 1) return images[0].image ?? null; return images.map((image) => `${image.name}=${image.image}`).join(", "); } function imagesDiffer(oldImages, newImages) { if (oldImages.length !== newImages.length) return true; const oldByName = new Map(oldImages.map((image) => [image.name, image.image])); return newImages.some((image) => oldByName.get(image.name) !== image.image); } function isAllowedDesiredTemplateJob(item) { return ( item?.kind === "Job" && itemNamespace(item) === namespace && item?.spec?.suspend === true && replaceableDevTemplateJobNames.has(item?.metadata?.name) ); } export function decideDevTemplateJobReplacement({ jobName, jobNamespace = namespace, desiredSuspended, liveExists, liveSuspended, oldImages = [], newImages = [] }) { const base = { namespace: jobNamespace, jobName, allowed: replaceableDevTemplateJobNames.has(jobName), desiredSuspended: desiredSuspended === true, liveExists: liveExists === true, liveSuspended: liveExists === true ? liveSuspended === true : null, oldImage: primaryImage(oldImages), newImage: primaryImage(newImages), oldImages, newImages, replace: false, result: "not_needed", reason: "Live suspended template Job already matches the desired image." }; if (!base.allowed) { return { ...base, result: "ignored", reason: "Job is not in the DEV template Job replacement allowlist." }; } if (jobNamespace !== namespace) { return { ...base, result: "blocked", reason: `Replacement policy is DEV-only and refuses namespace ${jobNamespace}.` }; } if (desiredSuspended !== true) { return { ...base, result: "blocked", reason: "Desired Job is not suspended, so it is not a template Job replacement candidate." }; } if (liveExists !== true) { return { ...base, result: "not_found", reason: "Live Job is absent; kubectl apply can create it without replacement." }; } if (liveSuspended !== true) { return { ...base, result: "blocked", reason: "Live Job is not suspended; refusing to delete a potentially active Job." }; } if (imagesDiffer(oldImages, newImages)) { return { ...base, replace: true, result: "planned", reason: "Live suspended template Job image differs from the desired manifest; delete/recreate is required because spec.template is immutable." }; } return base; } function replicaPlan(item) { if (item?.kind === "Job") { return item?.spec?.suspend === true ? 0 : 1; } return item?.spec?.replicas ?? null; } function buildWorkloadPlan(workloads) { return listItems(workloads).map((item) => { const containers = containersFor(item).map((container) => ({ name: container.name, serviceId: serviceIdFor(item, container), image: container.image })); return { kind: item?.kind ?? "unknown", name: item?.metadata?.name ?? "unknown", namespace: item?.metadata?.namespace ?? namespace, replicas: replicaPlan(item), serviceIds: uniqueStrings(containers.map((container) => container.serviceId)), containers }; }); } function buildServicePlan(services) { return listItems(services).map((item) => ({ kind: item?.kind ?? "unknown", name: item?.metadata?.name ?? "unknown", namespace: item?.metadata?.namespace ?? namespace, serviceId: item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ?? item?.metadata?.name ?? "unknown", type: item?.spec?.type ?? "unknown", ports: (item?.spec?.ports ?? []).map((port) => ({ name: port.name, port: port.port, targetPort: port.targetPort })) })); } function buildArtifactPlan(deploy, catalog, sourceCommitId, artifactEvidence) { const requiredServices = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false); const serviceCommitIds = uniqueStrings([ ...(catalog?.services ?? []).map((service) => service.commitId), ...(artifactEvidence ?? []).map((service) => service.catalogCommitId) ]); const expectedArtifactCommit = deploy?.commitId ?? catalog?.commitId ?? "unknown"; return { expectedArtifactCommit, deployCommitId: deploy?.commitId ?? "unknown", catalogCommitId: catalog?.commitId ?? "unknown", sourceCommitId, serviceCommitIds, matchesSourceCommit: deploy?.commitId && catalog?.commitId ? commitMatchesSource(deploy.commitId, sourceCommitId) && commitMatchesSource(catalog.commitId, sourceCommitId) : false, published: catalog?.publish?.ciPublished === true, registryVerified: catalog?.publish?.registryVerified === true, imageCount: artifactEvidence.length, requiredServiceCount: requiredServices.length, disabledServiceCount: (catalog?.services ?? []).filter((service) => service.artifactRequired === false).length, unpublishedServices: requiredServices .filter((service) => service.publishState !== "published" || !service.digest?.startsWith("sha256:")) .map((service) => service.serviceId) }; } function artifactIdentityForService(deploy, catalog, serviceId) { const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null; const catalogService = catalog?.services?.find((service) => service.serviceId === serviceId) ?? null; return { serviceId, sourceCommitId: catalogService?.commitId ?? deploy?.commitId ?? "unknown", image: deployService?.image ?? catalogService?.image ?? null, imageTag: catalogService?.imageTag ?? parseImageTag(deployService?.image), digest: catalogService?.digest ?? "not_published", repositoryDigest: catalogService?.repositoryDigest ?? null, publishState: catalogService?.publishState ?? "unknown" }; } function parseImageTag(image) { if (typeof image !== "string") return null; const slashIndex = image.lastIndexOf("/"); const colonIndex = image.lastIndexOf(":"); if (colonIndex <= slashIndex) return null; return image.slice(colonIndex + 1); } function expectedRuntimeIdentityEnv(artifact) { return { HWLAB_COMMIT_ID: artifact.sourceCommitId, HWLAB_IMAGE: artifact.image, HWLAB_IMAGE_TAG: artifact.imageTag }; } function envObjectFromEnvList(envList) { const env = {}; if (!Array.isArray(envList)) return env; for (const entry of envList) { if (!entry?.name) continue; env[entry.name] = Object.hasOwn(entry, "value") ? entry.value : null; } return env; } function runtimeIdentityValueMatches(name, actual, expected) { if (typeof actual !== "string" || typeof expected !== "string") return false; if (name === "HWLAB_COMMIT_ID") return commitMatchesSource(actual, expected); return actual === expected; } export function compareRuntimeIdentityEnv(env, expected) { const fields = runtimeIdentityEnvNames.map((name) => { const actual = Object.hasOwn(env ?? {}, name) ? env[name] : null; const expectedValue = expected?.[name] ?? null; const status = runtimeIdentityValueMatches(name, actual, expectedValue) ? "match" : "drift"; return { name, expected: expectedValue, actual, status }; }); return { status: fields.every((field) => field.status === "match") ? "match" : "drift", matchesDesired: fields.every((field) => field.status === "match"), fields, drift: fields.filter((field) => field.status === "drift") }; } function formatRuntimeIdentityDrift(drift) { return drift.map((field) => `${field.name} expected ${field.expected ?? "missing"} actual ${field.actual ?? "missing"}`).join("; "); } function deploymentRevision(deployment) { return deployment?.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? null; } function containerImageForDeployment(deployment, containerName) { const containers = deployment?.spec?.template?.spec?.containers ?? []; return containers.find((container) => container.name === containerName)?.image ?? null; } function buildRolloutVerificationCommand(serviceId) { return `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/${serviceId} --timeout=180s`; } function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase) { const artifact = artifactIdentityForService(deploy, catalog, serviceId); const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"]; return { serviceId, namespace, status: "not_evaluated", observationPhase, sourceCommitId: artifact.sourceCommitId, image: artifact.image ?? "unknown", imageTag: artifact.imageTag ?? "unknown", digest: artifact.digest, repositoryDigest: artifact.repositoryDigest, publishState: artifact.publishState, rolloutRevision: null, liveImage: null, imageMatchesDesired: false, expectedRuntimeEnv: expectedRuntimeIdentityEnv(artifact), liveRuntimeEnv: null, runtimeEnvMatchesDesired: false, runtimeEnvDrift: [], verificationCommand: buildRolloutVerificationCommand(serviceId), readCommand: kubectlCommand(kubectl, commandArgs), kubeconfig: kubectl.kubeconfig }; } function skippedDeploymentRolloutObservation(kubectl, deploy, catalog, serviceId, observationPhase, reason) { return { ...buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase), status: "not_evaluated", blocker: reason }; } async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blockers, options = {}) { const observationPhase = options.observationPhase ?? "read"; const blockRuntimeEnvDrift = options.blockRuntimeEnvDrift === true; const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"]; const base = buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase); if (kubectl.status !== "ready") { addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, kubectl.reason); return { ...base, status: "blocked", blocker: kubectl.reason }; } const result = await kubectlResult(kubectl, commandArgs, 15000); if (!result.ok) { const summary = oneLine(result.stderr || result.stdout); addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, `Cannot read DEV Deployment ${serviceId} rollout revision: ${summary}`); return { ...base, status: "blocked", blocker: summary }; } try { const deployment = JSON.parse(result.stdout); const liveImage = containerImageForDeployment(deployment, serviceId); const liveRuntimeEnv = compareRuntimeIdentityEnv( envObjectFromEnvList(deployment?.spec?.template?.spec?.containers?.find((container) => container.name === serviceId)?.env), base.expectedRuntimeEnv ); if (blockRuntimeEnvDrift && !liveRuntimeEnv.matchesDesired) { addBlocker( blockers, "runtime_blocker", `rollout-runtime-env-${observationPhase}-${serviceId}`, `DEV Deployment ${serviceId} runtime identity env drift after apply: ${formatRuntimeIdentityDrift(liveRuntimeEnv.drift)}` ); } return { ...base, status: "observed", rolloutRevision: deploymentRevision(deployment), liveImage, imageMatchesDesired: Boolean(base.image && liveImage === base.image), liveRuntimeEnv, runtimeEnvMatchesDesired: liveRuntimeEnv.matchesDesired, runtimeEnvDrift: liveRuntimeEnv.drift }; } catch (error) { const summary = oneLine(error.message); addBlocker(blockers, "contract_blocker", `rollout-read-${serviceId}`, `Cannot parse DEV Deployment ${serviceId} rollout revision: ${summary}`); return { ...base, status: "blocked", blocker: summary }; } } function blockerHint(blocker) { if (blocker.scope === "artifact-publish") { return "Publish DEV images from the intended commit and update the catalog with registry digests and verification evidence."; } if (blocker.scope === "artifact-source-commit") { return "Regenerate deploy/deploy.json and deploy/artifact-catalog.dev.json for the source commit that will be promoted."; } if (blocker.scope === "kubectl" || blocker.scope === "d601-kubeconfig" || blocker.scope === "kubectl-version") { return `Provide kubectl with readable D601 k3s kubeconfig at ${d601KubeconfigPath} without exposing token or secret values.`; } if (blocker.scope === "cluster-read") { return "Restore read-only visibility for pods, services, configmaps, deployments, and jobs in hwlab-dev."; } if (blocker.scope.startsWith("rollout-read-")) { return "Restore read-only Deployment visibility in hwlab-dev so rollout revision, image tag, digest, and source commit can be reported after apply."; } if (blocker.scope.startsWith("deploy-runtime-env-") || blocker.scope.startsWith("k8s-runtime-env-")) { return "Update deploy/deploy.json and deploy/k8s/base/workloads.yaml so the service owns HWLAB_COMMIT_ID, HWLAB_IMAGE, and HWLAB_IMAGE_TAG."; } if (blocker.scope.startsWith("rollout-runtime-env-")) { return "Reconcile the DEV Deployment runtime identity env to the desired source image and rerun post-apply read-only verification."; } if (blocker.scope === "dev-health" || blocker.scope === "dev-health-identity") { return "Restore the public DEV health path and confirm it identifies HWLAB dev, not a substitute runtime."; } if (blocker.scope === "dev-health-db" || blocker.scope === "cloud-api-db" || blocker.scope === "cloud-api-db-config") { return "Configure and verify the DEV cloud-api DB Secret URL host readiness through health output, without reading secret values."; } return blocker.summary; } function buildRemainingBlockers(blockers) { return blockers.map((blocker) => ({ type: blocker.type, scope: blocker.scope, status: blocker.status, summary: blocker.summary, unblockHint: oneLine(blockerHint(blocker)) })); } function buildRollbackHint(workloads) { const deploymentNames = buildWorkloadPlan(workloads) .filter((workload) => workload.kind === "Deployment") .map((workload) => workload.name); const jobNames = buildWorkloadPlan(workloads) .filter((workload) => workload.kind === "Job") .map((workload) => workload.name); return { namespace, strategy: "DEV-only Kubernetes rollback using deployment revision history; do not touch PROD or UniDesk services.", captureBeforeApply: [ `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get pods,services,configmaps,deployments,jobs -o wide`, `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get deployments -o json` ], deploymentRollbackCommands: deploymentNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout undo deployment/${name}`), jobCleanupCommands: jobNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev delete job ${name} --ignore-not-found`), postRollbackChecks: [ `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/hwlab-cloud-api --timeout=120s`, `curl -fsS ${new URL(healthPath, DEV_ENDPOINT).href}` ] }; } function buildManualCommands(status) { if (status === "blocked") { return { status: "blocked", beforeHumanApproval: [blockedDryRunCommand, "node scripts/validate-dev-gate-report.mjs"], afterHumanApproval: [], summary: "No apply command is offered while blockers remain open." }; } return { status: "ready", beforeHumanApproval: [dryRunPlanCommand, "node scripts/validate-dev-gate-report.mjs"], afterHumanApproval: [applyCommand], summary: "Run the dry-run plan and validator immediately before requesting human approval; live apply still requires the explicit DEV confirmation flags." }; } function buildApplyBoundary(args, status, applyStep) { return { currentMode: args.apply ? "apply-requested" : "dry-run", defaultNoWrite: !args.apply, mutationAttempted: applyStep.mutationAttempted === true && args.apply, mutationAllowed: status === "pass" && args.apply, applyRequiresFlags: ["--apply", "--confirm-dev", "--confirmed-non-production"], writeScope: `KUBECONFIG=${d601KubeconfigPath} kubectl apply -k deploy/k8s/dev, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`, noWriteScope: `source/catalog/k8s validation, optional DEV health probe, KUBECONFIG=${d601KubeconfigPath} kubectl read, replacement planning, and server-side dry-run only`, forbiddenActions }; } function buildPlanConclusion(status, blockers) { return { status: status === "blocked" ? "blocked" : "ready", reason: status === "blocked" ? `${blockers.length} blocker(s) must be cleared before any live DEV apply.` : "All DEV-only preconditions passed; human approval is still required before apply.", blockerCount: blockers.length }; } function validateArgs(args, blockers) { for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push", "--heavyweight-e2e"]) { if (args.flags.has(forbidden)) { addBlocker(blockers, "safety_blocker", forbidden.slice(2), `${forbidden} is forbidden for DEV deploy apply`); } } if (args.apply && (!args.flags.has("--confirm-dev") || !args.flags.has("--confirmed-non-production"))) { addBlocker( blockers, "safety_blocker", "apply-confirmation", "Live apply requires --confirm-dev and --confirmed-non-production" ); } } function commitMatchesSource(commitId, sourceCommitId) { return sourceCommitId.startsWith(commitId) || commitId.startsWith(sourceCommitId); } function validateDeployRuntimeIdentityEnv(deploy, catalog, serviceId, blockers) { const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null; if (!deployService) return null; const artifact = artifactIdentityForService(deploy, catalog, serviceId); const expected = expectedRuntimeIdentityEnv(artifact); const comparison = compareRuntimeIdentityEnv(deployService.env ?? {}, expected); if (!comparison.matchesDesired) { addBlocker( blockers, "contract_blocker", `deploy-runtime-env-${serviceId}`, `${serviceId} deploy env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}` ); } return { serviceId, source: "deploy", expected, ...comparison }; } function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) { const artifactEvidence = []; if (!deploy || !catalog) return artifactEvidence; if (deploy.environment !== ENVIRONMENT_DEV || deploy.namespace !== namespace) { addBlocker(blockers, "safety_blocker", "deploy-target", "deploy/deploy.json must target only dev hwlab-dev"); } if (deploy.profiles?.prod?.enabled !== false) { addBlocker(blockers, "safety_blocker", "prod-profile", "PROD profile must remain disabled"); } if (catalog.environment !== ENVIRONMENT_DEV || catalog.namespace !== namespace) { addBlocker(blockers, "safety_blocker", "catalog-target", "artifact catalog must target only dev hwlab-dev"); } if (!commitMatchesSource(deploy.commitId, sourceCommitId) || !commitMatchesSource(catalog.commitId, sourceCommitId)) { addBlocker( blockers, "observability_blocker", "artifact-source-commit", `deploy/catalog artifact commit ${deploy.commitId}/${catalog.commitId} does not match source ${sourceCommitId}` ); } const deployById = new Map((deploy.services ?? []).map((service) => [service.serviceId, service])); const catalogById = new Map((catalog.services ?? []).map((service) => [service.serviceId, service])); for (const serviceId of SERVICE_IDS) { const deployService = deployById.get(serviceId); const catalogService = catalogById.get(serviceId); if (!deployService || !catalogService) { addBlocker(blockers, "contract_blocker", `service-${serviceId}`, `${serviceId} missing from deploy or catalog`); continue; } if (deployService.namespace !== namespace || deployService.profile !== ENVIRONMENT_DEV) { addBlocker(blockers, "safety_blocker", `service-target-${serviceId}`, `${serviceId} is not DEV-only`); } if (deployService.image !== catalogService.image) { addBlocker(blockers, "contract_blocker", `service-image-${serviceId}`, `${serviceId} image differs in deploy/catalog`); } artifactEvidence.push({ serviceId, deployCommitId: deploy.commitId, catalogCommitId: catalogService.commitId, image: deployService.image, digest: catalogService.digest, publishState: catalogService.publishState }); } validateDeployRuntimeIdentityEnv(deploy, catalog, "hwlab-cloud-web", blockers); const requiredServices = (catalog.services ?? []).filter((service) => service.artifactRequired !== false); const disabledServices = (catalog.services ?? []).filter((service) => service.artifactRequired === false); const hasPublishedCatalog = catalog.publish?.ciPublished === true && catalog.publish?.registryVerified === true && requiredServices.every((service) => service.digest?.startsWith("sha256:") && service.publishState === "published") && disabledServices.every((service) => service.digest === "not_published" && service.publishState === "skeleton-only"); if (!hasPublishedCatalog) { addBlocker( blockers, "observability_blocker", "artifact-publish", "DEV artifact catalog has no CI publish, registry verification, or required-service registry digests" ); } return artifactEvidence; } function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers) { const items = [...listItems(namespaceDoc), ...listItems(workloads), ...listItems(services), ...listItems(healthContract)]; if (devKustomization?.namespace !== namespace) { addBlocker(blockers, "safety_blocker", "dev-kustomization", "DEV kustomization must pin namespace hwlab-dev"); } if ((devKustomization?.resources ?? []).some((resource) => resource.includes("prod"))) { addBlocker(blockers, "safety_blocker", "dev-kustomization-prod", "DEV kustomization must not reference PROD resources"); } for (const item of items) { const itemNamespace = item?.metadata?.namespace ?? (item?.kind === "Namespace" ? item?.metadata?.name : namespace); if (itemNamespace !== namespace) { addBlocker(blockers, "safety_blocker", `k8s-${item?.kind}-${item?.metadata?.name}`, "Kubernetes resource is not in hwlab-dev"); } } const expectedImages = new Map((deploy?.services ?? []).map((service) => [service.serviceId, service.image])); const cloudWebExpectedRuntimeEnv = expectedRuntimeIdentityEnv(artifactIdentityForService(deploy, catalog, "hwlab-cloud-web")); const workloadItems = listItems(workloads); const images = []; const runtimeIdentityEnv = []; for (const item of workloadItems) { for (const container of containersFor(item)) { const serviceId = serviceIdFor(item, container); const expectedImage = expectedImages.get(serviceId); images.push({ serviceId, image: container.image, kind: item.kind, name: item.metadata?.name }); if (expectedImage && container.image !== expectedImage) { addBlocker(blockers, "contract_blocker", `k8s-image-${serviceId}`, `${serviceId} image differs in k8s workload`); } if (serviceId === "hwlab-cloud-web") { const comparison = compareRuntimeIdentityEnv(envObjectFromEnvList(container.env), cloudWebExpectedRuntimeEnv); runtimeIdentityEnv.push({ serviceId, kind: item.kind, name: item.metadata?.name, container: container.name, expected: cloudWebExpectedRuntimeEnv, ...comparison }); if (!comparison.matchesDesired) { addBlocker( blockers, "contract_blocker", `k8s-runtime-env-${serviceId}`, `${serviceId} workload env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}` ); } } } } return { resourceCounts: { workloads: workloadItems.length, services: listItems(services).length, configMaps: listItems(healthContract).filter((item) => item.kind === "ConfigMap").length }, images, runtimeIdentityEnv }; } async function checkCloudApiDb(deploy, workloads, services, blockers) { const deployEnv = deploy?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {}; const envNames = new Set(Object.keys(deployEnv)); const workloadEnv = new Map(); for (const item of listItems(workloads)) { for (const container of containersFor(item)) { if (serviceIdFor(item, container) === "hwlab-cloud-api") { for (const env of container.env ?? []) { envNames.add(env.name); workloadEnv.set(env.name, env); } } } } const requiredEnv = DEV_DB_ENV_CONTRACT.requiredEnv.map((name) => { const secretRef = DEV_DB_ENV_CONTRACT.secretRefs.find((ref) => ref.env === name) ?? null; const workloadEntry = workloadEnv.get(name) ?? null; const secretRefPresent = secretRef ? deployEnv[name] === `secretRef:${secretRef.secretName}/${secretRef.secretKey}` && workloadEntry?.valueFrom?.secretKeyRef?.name === secretRef.secretName && workloadEntry?.valueFrom?.secretKeyRef?.key === secretRef.secretKey : true; return { name, manifestPresent: Object.hasOwn(deployEnv, name), k8sPresent: workloadEnv.has(name), redacted: Boolean(secretRef), source: secretRef ? "k8s-secret-ref" : "runtime-env", secretRef: secretRef ? { secretName: secretRef.secretName, secretKey: secretRef.secretKey, present: secretRefPresent, redacted: true } : null }; }); const missingManifest = requiredEnv.filter((env) => !env.manifestPresent).map((env) => env.name); const missingK8s = requiredEnv.filter((env) => !env.k8sPresent).map((env) => env.name); const missingSecretRefs = requiredEnv .filter((env) => env.secretRef && !env.secretRef.present) .map((env) => `${env.secretRef.secretName}/${env.secretRef.secretKey}`); const dnsContract = inspectCloudApiDbOptionalAliasContract(deployEnv, workloadEnv, services); const missingDnsContract = dnsContract.missing; if (missingManifest.length > 0 || missingK8s.length > 0 || missingSecretRefs.length > 0 || missingDnsContract.length > 0) { addBlocker( blockers, "runtime_blocker", "cloud-api-db-config", `cloud-api DEV DB env/authority contract is incomplete; missing ${[...missingManifest, ...missingK8s, ...missingSecretRefs, ...missingDnsContract].join(", ")}` ); } const configReady = missingManifest.length === 0 && missingK8s.length === 0 && missingSecretRefs.length === 0 && missingDnsContract.length === 0; const health = { status: configReady ? "degraded" : "blocked", ready: false, configReady, connected: false, connectionChecked: false, fields: requiredEnv.map((env) => ({ name: env.name, present: env.manifestPresent && env.k8sPresent && (!env.secretRef || env.secretRef.present), redacted: env.redacted })), missingEnv: uniqueStrings([...missingManifest, ...missingK8s]), secretRefs: requiredEnv .filter((env) => env.secretRef) .map((env) => ({ env: env.name, secretName: env.secretRef.secretName, secretKey: env.secretRef.secretKey, present: env.secretRef.present, redacted: true })), safety: { liveDbEvidence: false } }; return { status: configReady ? "contract-ready" : "contract-blocked", requiredEnv, missingManifest, missingK8s, missingSecretRefs, missingDnsContract, dns: dnsContract, runtimeHealth: summarizeDbContract(health), secretMaterialRead: false, valuesRedacted: true, liveDbEvidence: false, fixtureEvidence: false, note: "This check records DB env, Secret reference, and Secret URL host authority only. Optional cloud-api-db alias presence is diagnostic and is not live DB evidence." }; } function inspectCloudApiDbOptionalAliasContract(deployEnv, workloadEnv, services) { const alias = DEV_DB_ENV_CONTRACT.dns; const service = listItems(services).find((item) => item?.metadata?.name === alias.serviceName) ?? null; const actual = { deployServiceName: deployEnv.HWLAB_CLOUD_DB_SERVICE_NAME, deployServiceNamespace: deployEnv.HWLAB_CLOUD_DB_SERVICE_NAMESPACE, deployHost: deployEnv.HWLAB_CLOUD_DB_HOST, deployPort: deployEnv.HWLAB_CLOUD_DB_PORT, workloadServiceName: workloadEnv.get("HWLAB_CLOUD_DB_SERVICE_NAME")?.value, workloadServiceNamespace: workloadEnv.get("HWLAB_CLOUD_DB_SERVICE_NAMESPACE")?.value, workloadHost: workloadEnv.get("HWLAB_CLOUD_DB_HOST")?.value, workloadPort: workloadEnv.get("HWLAB_CLOUD_DB_PORT")?.value, serviceName: service?.metadata?.name }; const missing = []; for (const [name, value] of Object.entries(actual)) { if (value !== undefined) { missing.push(`${name} must be absent because ${alias.serviceName} is not the readiness authority`); } } return { ready: missing.length === 0, source: alias.source, serviceName: alias.serviceName, namespace: alias.namespace, port: alias.port, portName: alias.portName, requiredForReadiness: alias.requiredForReadiness, usedForProbe: alias.usedForProbe, servicePresent: Boolean(service), secretMaterialRead: false, liveDbEvidence: false, missing, actual }; } async function probeDevEndpoint(blockers) { const url = new URL(healthPath, DEV_ENDPOINT).href; try { const response = await httpGet(url, 8000); const text = response.body; let json = null; try { json = JSON.parse(text); } catch {} if (response.statusCode < 200 || response.statusCode >= 300) { addBlocker(blockers, "network_blocker", "dev-health", `DEV health returned HTTP ${response.statusCode}`); } if (json?.serviceId !== "hwlab-cloud-api" || json?.environment !== ENVIRONMENT_DEV) { addBlocker(blockers, "runtime_blocker", "dev-health-identity", "DEV health did not identify hwlab-cloud-api in dev"); } if (json?.db?.ready !== true && json?.db?.configReady !== true) { addBlocker(blockers, "runtime_blocker", "dev-health-db", "DEV health did not confirm cloud-api DB config readiness"); } return { status: response.statusCode >= 200 && response.statusCode < 300 ? "pass" : "blocked", url, httpStatus: response.statusCode, body: text.slice(0, 500) }; } catch (error) { const reason = oneLine(error.cause?.message ?? error.message); addBlocker(blockers, "network_blocker", "dev-health", `Cannot reach ${url}: ${reason}`); return { status: "blocked", url, error: reason }; } } function httpGet(url, timeoutMs) { return new Promise((resolve, reject) => { const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => { response.setEncoding("utf8"); let body = ""; response.on("data", (chunk) => { body += chunk; }); response.on("end", () => resolve({ statusCode: response.statusCode ?? 0, body })); }); request.on("timeout", () => { request.destroy(new Error(`timeout after ${timeoutMs}ms`)); }); request.on("error", reject); request.end(); }); } async function observeCluster(kubectl, blockers) { const blockedBase = { status: "blocked", executor: kubectl.executor ?? "missing", kubeconfig: kubectl.kubeconfig, commandPrefix: kubectl.commandPrefix, resources: [] }; if (!kubectl.executor) { addBlocker(blockers, "environment_blocker", "kubectl", kubectl.reason); return blockedBase; } if (kubectl.status !== "ready") { addBlocker(blockers, "environment_blocker", "d601-kubeconfig", kubectl.reason); return blockedBase; } const version = await kubectlResult(kubectl, ["version", "--client=true", "--output=yaml"]); if (!version.ok) { addBlocker(blockers, "environment_blocker", "kubectl-version", `kubectl client failed: ${version.stderr || version.stdout}`); return blockedBase; } const getArgs = ["get", "pods,services,configmaps,deployments,jobs", "-n", namespace, "-o", "json"]; const get = await kubectlResult(kubectl, getArgs, 20000); if (!get.ok) { addBlocker(blockers, "environment_blocker", "cluster-read", `Cannot read hwlab-dev resources: ${get.stderr || get.stdout}`); return { ...blockedBase, version: version.stdout, readCommand: kubectlCommand(kubectl, getArgs) }; } const parsed = JSON.parse(get.stdout); return { status: "pass", executor: kubectl.executor, kubeconfig: kubectl.kubeconfig, commandPrefix: kubectl.commandPrefix, readCommand: kubectlCommand(kubectl, getArgs), version: version.stdout, resources: (parsed.items ?? []).map((item) => ({ kind: item.kind, name: item.metadata?.name, phase: item.status?.phase, readyReplicas: item.status?.readyReplicas, replicas: item.status?.replicas })) }; } function isNotFoundOutput(value) { return /\bnot\s+found\b|notfound/iu.test(value); } async function readLiveTemplateJob(kubectl, jobName, blockers) { if (kubectl.status !== "ready") { return { status: "not_evaluated", reason: kubectl.reason }; } const get = await kubectlResult(kubectl, ["-n", namespace, "get", "job", jobName, "-o", "json"], 15000); if (!get.ok) { const output = get.stderr || get.stdout; if (isNotFoundOutput(output)) { return { status: "not_found", liveJob: null }; } addBlocker(blockers, "environment_blocker", `template-job-read-${jobName}`, `Cannot read live DEV template Job ${jobName}: ${output}`); return { status: "blocked", reason: oneLine(output), liveJob: null }; } try { return { status: "found", liveJob: JSON.parse(get.stdout) }; } catch (error) { addBlocker(blockers, "contract_blocker", `template-job-read-${jobName}`, `Cannot parse live DEV template Job ${jobName}: ${error.message}`); return { status: "blocked", reason: oneLine(error.message), liveJob: null }; } } async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) { const desiredJobs = listItems(workloads).filter(isAllowedDesiredTemplateJob); const replacements = []; for (const desiredJob of desiredJobs) { const jobName = desiredJob.metadata.name; const live = await readLiveTemplateJob(kubectl, jobName, blockers); if (live.status === "not_evaluated") { replacements.push({ namespace, jobName, allowed: true, desiredSuspended: true, liveExists: null, liveSuspended: null, oldImage: null, newImage: primaryImage(imageRecordsFor(desiredJob)), oldImages: [], newImages: imageRecordsFor(desiredJob), replace: false, result: "not_evaluated", reason: live.reason }); continue; } if (live.status === "blocked") { replacements.push({ namespace, jobName, allowed: true, desiredSuspended: true, liveExists: null, liveSuspended: null, oldImage: null, newImage: primaryImage(imageRecordsFor(desiredJob)), oldImages: [], newImages: imageRecordsFor(desiredJob), replace: false, result: "blocked", reason: live.reason }); continue; } const decision = decideDevTemplateJobReplacement({ jobName, jobNamespace: itemNamespace(desiredJob), desiredSuspended: desiredJob.spec?.suspend, liveExists: live.status === "found", liveSuspended: live.liveJob?.spec?.suspend, oldImages: live.liveJob ? imageRecordsFor(live.liveJob) : [], newImages: imageRecordsFor(desiredJob) }); if (decision.result === "blocked") { addBlocker(blockers, "safety_blocker", `template-job-replace-${jobName}`, decision.reason); } replacements.push(decision); } return replacements; } async function executeDevTemplateJobReplacements(kubectl, replacements, blockers) { const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned"); for (const replacement of planned) { const commandArgs = [ "-n", replacement.namespace, "delete", "job", replacement.jobName, "--ignore-not-found=true" ]; const result = await kubectlResult(kubectl, commandArgs, 20000); replacement.deleteCommand = kubectlCommand(kubectl, commandArgs); replacement.deleteStdout = result.stdout; replacement.deleteStderr = result.stderr; if (!result.ok) { replacement.result = "delete_failed"; replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(result.stderr || result.stdout)}`; addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason); continue; } replacement.result = "deleted_pending_recreate"; replacement.reason = "Live suspended template Job was deleted; kubectl apply must recreate it from the desired manifest."; } return planned.length; } function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) { const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName); if (result.ok || plannedNames.length === 0) return false; const output = `${result.stdout}\n${result.stderr}`; const errorSections = output.split("Error from server").slice(1); const immutableSections = errorSections.filter( (section) => section.includes("spec.template") && section.includes("field is immutable") ); if (immutableSections.length === 0 || immutableSections.length !== errorSections.length) return false; for (const name of plannedNames) { if (!immutableSections.some((section) => section.includes(name))) return false; } return immutableSections.every( (section) => section.includes("Resource=jobs") && section.includes("Kind=Job") && plannedNames.some((name) => section.includes(name)) ); } async function runApplyStep(args, kubectl, blockers, templateJobReplacements) { if (blockers.length > 0) { return { status: "not_run", summary: "Skipped because preflight blockers are open" }; } let mutationAttempted = false; const replacementsNeeded = templateJobReplacements.filter((replacement) => replacement.replace); if (args.apply) { const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers); mutationAttempted = replaceCount > 0; if (blockers.length > 0) { return { status: "blocked", command: kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"]), mutationAttempted, summary: "Skipped kubectl apply because DEV template Job replacement failed" }; } } const commandArgs = args.apply ? ["apply", "-k", "deploy/k8s/dev"] : ["apply", "--dry-run=server", "-k", "deploy/k8s/dev"]; const result = await kubectlResult(kubectl, commandArgs, 30000); const expectedImmutableDryRun = !args.apply && isExpectedTemplateJobImmutableFailure(result, replacementsNeeded); mutationAttempted = mutationAttempted || args.apply; if (!result.ok && !expectedImmutableDryRun) { addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", result.stderr || result.stdout); } for (const replacement of templateJobReplacements) { if (replacement.result === "deleted_pending_recreate") { replacement.result = result.ok ? "replaced" : "delete_succeeded_apply_failed"; replacement.reason = result.ok ? "Live suspended template Job was deleted and recreated by kubectl apply." : "Live suspended template Job was deleted, but kubectl apply failed before confirming recreation."; } } return { status: result.ok || expectedImmutableDryRun ? "pass" : "blocked", command: kubectlCommand(kubectl, commandArgs), stdout: result.stdout, stderr: result.stderr, mutationAttempted, expectedImmutableTemplateJobDryRun: expectedImmutableDryRun, expectedImmutableTemplateJobs: expectedImmutableDryRun ? replacementsNeeded.map((replacement) => replacement.jobName) : [] }; } async function gitHeadCommit() { const result = await commandResult("git", ["rev-parse", "--short=12", "HEAD"]); return result.ok ? result.stdout : "unknown"; } export function resolveApplySourceCommit(deploy, catalog, gitHeadCommitId) { const deployCommit = deploy?.commitId; const catalogCommit = catalog?.commitId; if ( typeof deployCommit === "string" && typeof catalogCommit === "string" && commitMatchesSource(deployCommit, catalogCommit) ) { return deployCommit.length >= catalogCommit.length ? deployCommit : catalogCommit; } return gitHeadCommitId; } export async function runDevDeployApply(argv, io = {}) { const stdout = io.stdout ?? process.stdout; const args = parseArgs(argv); const blockers = []; validateArgs(args, blockers); const gitHeadCommitIdPromise = gitHeadCommit(); const [deploy, catalog, devKustomization, namespaceDoc, workloads, services, healthContract] = await Promise.all([ readJson("deploy/deploy.json", blockers), readJson("deploy/artifact-catalog.dev.json", blockers), readJson("deploy/k8s/dev/kustomization.yaml", blockers), readJson("deploy/k8s/base/namespace.yaml", blockers), readJson("deploy/k8s/base/workloads.yaml", blockers), readJson("deploy/k8s/base/services.yaml", blockers), readJson("deploy/k8s/dev/health-contract.yaml", blockers) ]); const gitHeadCommitId = await gitHeadCommitIdPromise; const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId); const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers); const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers); const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers); if (!(await readText("internal/cloud/server.mjs")).includes('url.pathname === "/health/live"')) { addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path"); } const kubectl = await resolveD601Kubectl(); const clusterObservation = await observeCluster(kubectl, blockers); const cloudWebRolloutBeforeApply = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, { observationPhase: "before_apply", blockRuntimeEnvDrift: false }); const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers); const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers); const applyStep = await runApplyStep( args, kubectl, blockers, templateJobReplacements ); const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass" ? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, { observationPhase: "after_apply", blockRuntimeEnvDrift: true }) : skippedDeploymentRolloutObservation( kubectl, deploy, catalog, "hwlab-cloud-web", "after_apply", args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification" ); const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply; const status = blockers.length > 0 ? "blocked" : "pass"; const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence); const workloadPlan = buildWorkloadPlan(workloads); const servicePlan = buildServicePlan(services); const planConclusion = buildPlanConclusion(status, blockers); const manualCommands = buildManualCommands(status); const remainingBlockers = buildRemainingBlockers(blockers); const report = { $schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json", $id: "https://hwlab.pikastech.local/reports/dev-gate/dev-deploy-report.json", reportVersion: "v1", issue: "pikasTech/HWLAB#33", supports: ["pikasTech/HWLAB#63", "pikasTech/HWLAB#50", "pikasTech/HWLAB#7", "pikasTech/HWLAB#33", "pikasTech/HWLAB#17", "pikasTech/HWLAB#32", "pikasTech/HWLAB#30"], taskId: "dev-deploy-apply", commitId, acceptanceLevel: "dev_deploy_apply", devOnly: true, prodDisabled: true, reportLifecycle: activeReportLifecycle("Current DEV deploy apply report; active public API/health endpoint is 16667 and browser endpoint is 16666."), status, generatedAt: new Date().toISOString(), namespace, endpoint: DEV_ENDPOINT, sourceContract: { status: "pass", documents: [ "docs/dev-acceptance-matrix.md", "docs/m0-contract-audit.md", "docs/dev-deploy-apply.md", "docs/artifact-catalog.md", "deploy/README.md" ], summary: "DEV deploy apply is pinned to hwlab-dev and the frozen HWLAB service IDs" }, validationCommands: requiredValidationCommands, localSmoke: { status: "not_run", commands: ["node scripts/m1-contract-smoke.mjs"], evidence: ["This deploy apply task did not run local smoke because artifact/executor blockers stop promotion first."], summary: "Local smoke remains available but is not a substitute for DEV artifact publish evidence." }, dryRun: { status, commands: [ "node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run", "node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked" ], evidence: [ `artifact services checked: ${artifactEvidence.length}`, `expected artifact commit: ${artifactPlan.expectedArtifactCommit}`, `namespace: ${namespace}`, `workloads planned: ${workloadPlan.length}`, `kubectl executor: ${kubectl.executor ?? "missing"}`, `kubectl kubeconfig: ${kubectl.kubeconfig}`, `live health: ${liveProbe.status}`, `template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}` ], summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed." }, devPreconditions: { status, requirements: [ "DEV artifact catalog must contain CI publish evidence and registry digests", "kubectl must be available for D601 hwlab-dev and must not target PROD", "hwlab-cloud-api /health/live must report serviceId, dev environment, and DB env readiness without exposing secret values", "pods, services, configmaps, and image commit tags must be observable in hwlab-dev" ], summary: status === "blocked" ? "One or more required DEV deploy preconditions are blocked." : "DEV deploy preconditions are satisfied." }, devDeployApply: { mode: args.apply ? "apply" : "dry-run", conclusion: planConclusion, applyBoundary: buildApplyBoundary(args, status, applyStep), mutationAttempted: applyStep.mutationAttempted === true && args.apply, target: { environment: ENVIRONMENT_DEV, namespace, endpoint: DEV_ENDPOINT, prodDisabled: true }, artifactPlan, workloadPlan, servicePlan, templateJobReplacementPolicy: devTemplateJobReplacementPolicy, templateJobReplacements, cloudWebRollout, cloudWebRolloutBeforeApply, cloudWebRolloutAfterApply, applyStep, manualCommands, rollbackHint: buildRollbackHint(workloads), remainingBlockers, artifactEvidence, cloudApiDb, k8sManifest, clusterObservation, liveProbe }, blockers, notes: "DEV-only result. No PROD resources, secret values, force push, heavy e2e, browser e2e, or UniDesk runtime substitute were used." }; if (args.writeReport) { await writeFile(path.join(repoRoot, reportPath), `${JSON.stringify(report, null, 2)}\n`); } stdout.write(`${JSON.stringify(report, null, 2)}\n`); return status === "blocked" && !args.expectBlocked ? 2 : 0; } export function formatFailure(error) { return { reportVersion: "v1", issue: "pikasTech/HWLAB#33", taskId: "dev-deploy-apply", status: "failed", blockers: [{ type: "observability_blocker", scope: "script", status: "open", summary: oneLine(error.message) }] }; }