feat: parallelize dev ci cd artifact flow

scripts/src/dev-base-image-preflight.mjs

@@ -21,7 +21,9 @@ const allowedExplicitBaseImages = Object.freeze([
   "hwlab-dev-base:*",
   "*/hwlab-dev-base:*",
   "hwlab-node20-base:*",
-  "*/hwlab-node20-base:*"
+  "*/hwlab-node20-base:*",
+  "hwlab-node-runtime-base:*",
+  "*/hwlab-node-runtime-base:*"
 ]);
 
 function baseImageRecommendation(envValue = recommendedNode20Image) {
@@ -131,7 +133,7 @@ function isNode20BaseRef(imageRef) {
 
 function isHwlabDevBaseRef(imageRef) {
   const lowerRef = imageRef.toLowerCase();
-  return /(^|\/)(hwlab-dev-base|hwlab-node20-base)(?=[:@]|$)/u.test(lowerRef);
+  return /(^|\/)(hwlab-dev-base|hwlab-node20-base|hwlab-node-runtime-base)(?=[:@]|$)/u.test(lowerRef);
 }
 
 function isTaggedImage(image) {
@@ -612,3 +614,17 @@ export function formatPreflightResult(result) {
 export function exitCodeForPreflight(result) {
   return result.publishUsable ? 0 : 2;
 }
+
+export function summarizeForRuntimeBase(result) {
+  return {
+    status: result.status,
+    publishUsable: result.publishUsable,
+    imageSource: result.imageSource,
+    requestedImage: result.requestedImage,
+    localTag: result.localTag,
+    imageId: result.imageId,
+    blockers: result.blockers,
+    nextSteps: result.nextSteps,
+    containerEngine: result.containerEngine
+  };
+}

scripts/src/dev-cd-apply.mjs

@@ -513,19 +513,24 @@ async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, ar
   const reportCommitMatches = commitMatches(reportSource, promotion.commitId);
   const deployCommitMatches = commitMatches(deployBefore.commitId, promotion.commitId);
   const isMergeCommit = parents.length >= 2;
-  const status = (
+  const desiredStateStatus = (
     deployCommitMatches &&
     catalogCommitMatches &&
-    reportCommitMatches &&
     (
       promotionParentMatches ||
       artifactParentMatches ||
       commitMatches(mergeCommitId, promotion.commitId)
     )
   ) ? "pass" : "degraded";
+  const reportStatus = artifactEvidence.report.exists
+    ? (reportCommitMatches ? "matches_promotion" : "stale_or_unrelated")
+    : "missing";
 
   return {
-    status,
+    status: desiredStateStatus,
+    reportStatus,
+    reportIsReleaseGate: false,
+    reportPolicy: "CI artifact reports are audit snapshots; deploy.json, artifact catalog, workloads, and registry manifests are the CD release gate.",
     controlCommit: {
       commitId: control.commitId,
       shortCommitId: control.shortCommitId
@@ -557,6 +562,7 @@ async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, ar
       deployCommitMatches,
       catalogCommitMatches,
       reportCommitMatches,
+      reportExists: artifactEvidence.report.exists,
       desiredStatePaths: artifactDesiredStatePaths
     }
   };
@@ -2335,7 +2341,7 @@ function buildReport({
         "docs/dev-deploy-apply.md",
         "docs/artifact-catalog.md"
       ],
-      summary: "DEV CD publish/apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the apply desired-state source."
+      summary: "DEV CD apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the publish-control desired-state source; CI artifact reports are audit snapshots."
     },
     validationCommands: [
       "node --check scripts/dev-cd-apply.mjs",
@@ -2367,8 +2373,8 @@ function buildReport({
         "DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.",
         "DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.",
         "HEAD must match the requested target ref before publish.",
-        "Lease/hwlab-dev-cd-lock must be acquired before publish/apply.",
-        "Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.",
+        "Lease/hwlab-dev-cd-lock must be acquired before DEV apply/verify side effects.",
+        "Artifact publish must run under CI artifact identity, with HWLAB_CD_TRANSACTION_ID accepted only for the legacy transition path.",
         "DEV DB role/database provisioning and runtime migration must run through repo-owned commands before workload apply.",
         "deploy/deploy.json, artifact catalog, and workloads must converge before apply.",
         "When deploy/deploy.json already points to a published promotion commit, HEAD may be a later control commit and the transaction must consume that promotion without republishing HEAD.",

scripts/src/dev-cd-apply.test.mjs

@@ -587,6 +587,56 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont
   assertNoReadOnlySideEffects(commandLog);
 });
 
+test("dev-cd status treats stale CI artifact report as audit-only when desired state matches", async () => {
+  const repoRoot = await makeRepo({ commitId: "abc1234" });
+  await writeFile(
+    artifactReportFixturePath,
+    `${JSON.stringify({
+      status: "published",
+      commitId: "9999999",
+      artifactPublish: {
+        status: "published",
+        sourceCommitId: "9999999",
+        serviceCount: 0,
+        requiredServiceCount: 0,
+        publishedCount: 0,
+        services: []
+      }
+    }, null, 2)}\n`
+  );
+  const commandLog = [];
+  let output = "";
+  const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
+    repoRoot,
+    env: {},
+    runCommand: makeRunCommand({
+      commandLog,
+      targetCommitId: laterControlCommit,
+      headCommitId: laterControlCommit,
+      extraGitRefs: {
+        abc1234: publishedCommit
+      },
+      latestArtifactMergeCommit: artifactMergeCommit,
+      commitParents: {
+        [artifactMergeCommit]: [publishedCommit, artifactReportCommit],
+        [artifactReportCommit]: [publishedCommit]
+      }
+    }),
+    httpGetJson: makeHttpGetJson(),
+    now: () => new Date(iso(100000)),
+    stdout: { write: (chunk) => { output += chunk; } }
+  });
+
+  const status = JSON.parse(output);
+  assert.equal(code, 0);
+  assert.equal(status.status, "pass");
+  assert.equal(status.target.artifactBoundary.status, "pass");
+  assert.equal(status.target.artifactBoundary.reportStatus, "stale_or_unrelated");
+  assert.equal(status.target.artifactBoundary.reportIsReleaseGate, false);
+  assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
+  assertNoReadOnlySideEffects(commandLog);
+});
+
 test("dev-cd status exposes moving origin/main when checkout is behind a deploy-json promotion control ref", async () => {
   const repoRoot = await makeRepo({ commitId: "abc1234" });
   const commandLog = [];
@@ -603,8 +653,8 @@ test("dev-cd status exposes moving origin/main when checkout is behind a deploy-
       },
       latestArtifactMergeCommit: artifactMergeCommit,
       commitParents: {
-        [artifactMergeCommit]: [publishedCommit, artifactReportCommit],
-        [artifactReportCommit]: [publishedCommit]
+        [artifactMergeCommit]: [laterControlCommit, artifactReportCommit],
+        [artifactReportCommit]: [laterControlCommit]
       }
     }),
     httpGetJson: makeHttpGetJson(),
@@ -666,7 +716,7 @@ test("apply stops before lock acquisition when checkout is behind deploy-json pr
   assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
 });
 
-test("dev-cd status degrades artifact merge provenance when catalog/report do not match deploy-json promotion", async () => {
+test("dev-cd status degrades artifact merge provenance when catalog does not match deploy-json promotion", async () => {
   const repoRoot = await makeRepo({ commitId: "abc1234" });
   await writeFile(
     path.join(repoRoot, "deploy/artifact-catalog.dev.json"),
@@ -751,8 +801,8 @@ test("apply stops before lock acquisition when artifact merge provenance mismatc
       },
       latestArtifactMergeCommit: artifactMergeCommit,
       commitParents: {
-        [artifactMergeCommit]: [publishedCommit, artifactReportCommit],
-        [artifactReportCommit]: [publishedCommit]
+        [artifactMergeCommit]: [laterControlCommit, artifactReportCommit],
+        [artifactReportCommit]: [laterControlCommit]
       }
     }),
     httpGetJson: makeHttpGetJson(),

scripts/src/dev-cd-transaction-guard.mjs

@@ -5,11 +5,45 @@ const transactionEnvNames = [
   "HWLAB_CD_TRANSACTION_OWNER",
   "HWLAB_CD_LOCK_NAME"
 ];
+const ciArtifactEnvNames = [
+  "HWLAB_CI_ARTIFACT_RUN_ID",
+  "HWLAB_CI_ARTIFACT_RUN_OWNER"
+];
 
 export function hasDevCdTransactionEnv(env = process.env) {
   return typeof env.HWLAB_CD_TRANSACTION_ID === "string" && env.HWLAB_CD_TRANSACTION_ID.trim().length > 0;
 }
 
+export function hasDevCiArtifactEnv(env = process.env) {
+  return typeof env.HWLAB_CI_ARTIFACT_RUN_ID === "string" && env.HWLAB_CI_ARTIFACT_RUN_ID.trim().length > 0;
+}
+
+export function hasDevArtifactSideEffectEnv(env = process.env) {
+  return hasDevCdTransactionEnv(env) || hasDevCiArtifactEnv(env);
+}
+
+export function devArtifactProducer(env = process.env) {
+  if (hasDevCiArtifactEnv(env)) {
+    return {
+      kind: "ci-artifact",
+      runId: env.HWLAB_CI_ARTIFACT_RUN_ID,
+      owner: env.HWLAB_CI_ARTIFACT_RUN_OWNER ?? null
+    };
+  }
+  if (hasDevCdTransactionEnv(env)) {
+    return {
+      kind: "cd-transaction-legacy",
+      runId: env.HWLAB_CD_TRANSACTION_ID,
+      owner: env.HWLAB_CD_TRANSACTION_OWNER ?? null
+    };
+  }
+  return {
+    kind: "missing",
+    runId: null,
+    owner: null
+  };
+}
+
 export function devCdTransactionGuardFailure({
   script,
   mode,
@@ -32,7 +66,25 @@ export function devCdTransactionGuardFailure({
   };
 }
 
+export function devArtifactSideEffectGuardFailure({
+  script,
+  mode,
+  requiredEnv = "HWLAB_CI_ARTIFACT_RUN_ID or HWLAB_CD_TRANSACTION_ID"
+}) {
+  return {
+    ...devCdTransactionGuardFailure({ script, mode, requiredEnv }),
+    acceptedEnv: [...ciArtifactEnvNames, ...transactionEnvNames],
+    entrypoint: `HWLAB_CI_ARTIFACT_RUN_ID=<run-id> node scripts/dev-artifact-publish.mjs --publish --report ${tempReportPath("dev-artifacts.json")}`,
+    summary: `${script} ${mode} is a DEV artifact side-effect step and must run inside a CI artifact run or the legacy DEV CD transaction.`
+  };
+}
+
 export function requireDevCdTransactionForSideEffect({ env = process.env, script, mode }) {
   if (hasDevCdTransactionEnv(env)) return null;
   return devCdTransactionGuardFailure({ script, mode });
 }
+
+export function requireDevArtifactSideEffectEnv({ env = process.env, script, mode }) {
+  if (hasDevArtifactSideEffectEnv(env)) return null;
+  return devArtifactSideEffectGuardFailure({ script, mode });
+}

scripts/src/dev-deploy-apply.mjs

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { constants as fsConstants } from "node:fs";
-import { access, readFile, writeFile } from "node:fs/promises";
+import { access, mkdir, readFile, writeFile } from "node:fs/promises";
 import { request as httpRequest } from "node:http";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
@@ -88,6 +88,8 @@ const forbiddenActions = [
   "force-push"
 ];
 const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"];
+const digestPattern = /^sha256:[a-f0-9]{64}$/u;
+const defaultCdConcurrency = parseBoundedInteger(process.env.HWLAB_DEV_CD_CONCURRENCY, 4, 1, 8);
 
 export function parseArgs(argv) {
   const flags = new Set();
@@ -95,6 +97,7 @@ export function parseArgs(argv) {
   let kubeconfig = null;
   let kubeconfigSpecified = false;
   let reportPath = defaultReportPath;
+  let rolloutConcurrency = defaultCdConcurrency;
 
   for (let index = 0; index < argv.length; index += 1) {
     const arg = argv[index];
@@ -141,6 +144,22 @@ export function parseArgs(argv) {
       }
       continue;
     }
+    if (arg === "--rollout-concurrency") {
+      flags.add("--rollout-concurrency");
+      const next = argv[index + 1];
+      if (typeof next !== "string" || next.startsWith("--")) {
+        errors.push("--rollout-concurrency requires a numeric value");
+      } else {
+        rolloutConcurrency = parseBoundedInteger(next, defaultCdConcurrency, 1, 8);
+        index += 1;
+      }
+      continue;
+    }
+    if (arg.startsWith("--rollout-concurrency=")) {
+      flags.add("--rollout-concurrency");
+      rolloutConcurrency = parseBoundedInteger(arg.slice("--rollout-concurrency=".length), defaultCdConcurrency, 1, 8);
+      continue;
+    }
     flags.add(arg);
   }
 
@@ -151,6 +170,7 @@ export function parseArgs(argv) {
     skipLiveProbe: flags.has("--skip-live-probe"),
     writeReport: flags.has("--report-output"),
     reportPath,
+    rolloutConcurrency,
     kubeconfig,
     kubeconfigSpecified,
     errors,
@@ -158,6 +178,12 @@ export function parseArgs(argv) {
   };
 }
 
+function parseBoundedInteger(value, fallback, min, max) {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  if (!Number.isInteger(parsed)) return fallback;
+  return Math.min(Math.max(parsed, min), max);
+}
+
 function addBlocker(blockers, type, scope, summary) {
   const key = `${type}::${scope}`;
   if (!blockers.some((blocker) => `${blocker.type}::${blocker.scope}` === key)) {
@@ -169,6 +195,21 @@ function oneLine(value) {
   return String(value).replace(/\s+/g, " ").trim();
 }
 
+async function mapWithConcurrency(items, concurrency, worker) {
+  if (items.length === 0) return [];
+  const results = new Array(items.length);
+  let nextIndex = 0;
+  const workerCount = Math.min(Math.max(concurrency, 1), items.length);
+  await Promise.all(Array.from({ length: workerCount }, async () => {
+    while (nextIndex < items.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      results[index] = await worker(items[index], index);
+    }
+  }));
+  return results;
+}
+
 async function readJson(relativePath, blockers) {
   try {
     return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8"));
@@ -675,6 +716,146 @@ function parseImageTag(image) {
   return image.slice(colonIndex + 1);
 }
 
+function parseTaggedImageReference(image) {
+  if (typeof image !== "string" || image.includes("@")) return null;
+  const firstSlash = image.indexOf("/");
+  const lastSlash = image.lastIndexOf("/");
+  const colonIndex = image.lastIndexOf(":");
+  if (firstSlash <= 0 || colonIndex <= lastSlash) return null;
+  return {
+    image,
+    registry: image.slice(0, firstSlash),
+    repository: image.slice(firstSlash + 1, colonIndex),
+    tag: image.slice(colonIndex + 1)
+  };
+}
+
+function registryManifestUrlForImage(image) {
+  const parsed = parseTaggedImageReference(image);
+  if (!parsed) return null;
+  const repositoryPath = parsed.repository.split("/").map((part) => encodeURIComponent(part)).join("/");
+  return {
+    ...parsed,
+    url: `http://${parsed.registry}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}`
+  };
+}
+
+function registryAcceptHeader() {
+  return [
+    "application/vnd.docker.distribution.manifest.v2+json",
+    "application/vnd.oci.image.manifest.v1+json",
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.oci.image.index.v1+json",
+    "application/vnd.docker.distribution.manifest.v1+json"
+  ].join(", ");
+}
+
+function httpRequestText(url, { method = "GET", headers = {}, timeoutMs = 15000 } = {}) {
+  return new Promise((resolve, reject) => {
+    const request = httpRequest(url, { method, headers, timeout: timeoutMs }, (response) => {
+      response.setEncoding("utf8");
+      let body = "";
+      response.on("data", (chunk) => {
+        body += chunk;
+        if (body.length > 4096) {
+          request.destroy(new Error("registry manifest response exceeded 4096 bytes during CD verification"));
+        }
+      });
+      response.on("end", () => resolve({
+        statusCode: response.statusCode ?? 0,
+        headers: response.headers,
+        body
+      }));
+    });
+    request.on("timeout", () => {
+      request.destroy(new Error(`timeout after ${timeoutMs}ms`));
+    });
+    request.on("error", reject);
+    request.end();
+  });
+}
+
+async function verifyCatalogRegistryManifest(service, blockers) {
+  const image = service.image;
+  const expectedDigest = service.digest;
+  const parsed = registryManifestUrlForImage(image);
+  const base = {
+    serviceId: service.serviceId,
+    image,
+    imageTag: service.imageTag ?? parseImageTag(image),
+    expectedDigest,
+    url: parsed?.url ?? null
+  };
+
+  if (!parsed) {
+    const reason = `${service.serviceId} image is not a tagged registry image`;
+    addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason);
+    return { ...base, status: "blocked", reason };
+  }
+  if (!digestPattern.test(expectedDigest ?? "")) {
+    const reason = `${service.serviceId} catalog digest is not an immutable sha256 digest`;
+    addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason);
+    return { ...base, status: "blocked", reason };
+  }
+
+  try {
+    const response = await httpRequestText(parsed.url, {
+      method: "HEAD",
+      headers: {
+        Accept: registryAcceptHeader()
+      },
+      timeoutMs: 15000
+    });
+    const observedDigest = String(response.headers["docker-content-digest"] ?? "").trim();
+    const okStatus = response.statusCode >= 200 && response.statusCode < 300;
+    const digestMatches = observedDigest === expectedDigest;
+    if (!okStatus || !digestMatches) {
+      const reason = !okStatus
+        ? `${service.serviceId} registry manifest returned HTTP ${response.statusCode}`
+        : `${service.serviceId} registry digest mismatch: expected ${expectedDigest}, observed ${observedDigest || "missing"}`;
+      addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason);
+      return {
+        ...base,
+        status: "blocked",
+        reason,
+        httpStatus: response.statusCode,
+        observedDigest,
+        digestMatches
+      };
+    }
+    return {
+      ...base,
+      status: "pass",
+      httpStatus: response.statusCode,
+      observedDigest,
+      digestMatches: true
+    };
+  } catch (error) {
+    const reason = `${service.serviceId} registry manifest read failed: ${oneLine(error.message)}`;
+    addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason);
+    return {
+      ...base,
+      status: "blocked",
+      reason
+    };
+  }
+}
+
+async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency) {
+  const services = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false);
+  const results = await mapWithConcurrency(services, concurrency, async (service) =>
+    verifyCatalogRegistryManifest(service, blockers)
+  );
+  return {
+    status: results.every((result) => result.status === "pass") ? "pass" : "blocked",
+    source: "registry-manifest",
+    releaseGate: true,
+    serviceCount: results.length,
+    concurrency,
+    results
+  };
+}
+
 function expectedRuntimeIdentityEnv(artifact) {
   return {
     HWLAB_COMMIT_ID: artifact.sourceCommitId,
@@ -1675,9 +1856,9 @@ async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads,
   return cleanups;
 }
 
-async function executeDevTemplateJobReplacements(kubectl, replacements, blockers) {
+async function executeDevTemplateJobReplacements(kubectl, replacements, blockers, concurrency = defaultCdConcurrency) {
   const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned");
-  for (const replacement of planned) {
+  await mapWithConcurrency(planned, concurrency, async (replacement) => {
     const commandArgs = [
       "-n",
       replacement.namespace,
@@ -1694,17 +1875,17 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers
       replacement.result = "delete_failed";
       replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(commandOutput(result))}`;
       addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason);
-      continue;
+      return;
     }
     replacement.result = "deleted_pending_recreate";
     replacement.reason = "Live suspended template Job was deleted; kubectl apply must recreate it from the desired manifest.";
-  }
+  });
   return planned.length;
 }
 
-async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers) {
+async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers, concurrency = defaultCdConcurrency) {
   const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned");
-  for (const cleanup of planned) {
+  await mapWithConcurrency(planned, concurrency, async (cleanup) => {
     const commandArgs = [
       "-n",
       cleanup.namespace,
@@ -1721,14 +1902,107 @@ async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, bl
       cleanup.result = "delete_failed";
       cleanup.reason = `Failed to delete stale legacy simulator Deployment before apply: ${oneLine(commandOutput(result))}`;
       addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason);
-      continue;
+      return;
     }
     cleanup.result = "deleted_pending_apply";
     cleanup.reason = "Live stale legacy simulator Deployment was deleted; kubectl apply must leave the desired indexed StatefulSet in place.";
-  }
+  });
   return planned.length;
 }
 
+function rolloutApiName(kind) {
+  if (kind === "Deployment") return "deployment";
+  if (kind === "StatefulSet") return "statefulset";
+  return null;
+}
+
+function rolloutTargetsFromWorkloads(workloads) {
+  return listItems(workloads)
+    .map((item) => {
+      const apiName = rolloutApiName(item?.kind);
+      if (!apiName) return null;
+      return {
+        kind: item.kind,
+        apiName,
+        name: item?.metadata?.name ?? "unknown",
+        namespace: item?.metadata?.namespace ?? namespace,
+        replicas: replicaPlan(item),
+        serviceIds: uniqueStrings(containersFor(item).map((container) => serviceIdFor(item, container)))
+      };
+    })
+    .filter(Boolean);
+}
+
+async function verifyRolloutTarget(kubectl, target, blockers) {
+  const commandArgs = [
+    "-n",
+    target.namespace,
+    "rollout",
+    "status",
+    `${target.apiName}/${target.name}`,
+    "--timeout=180s"
+  ];
+  const base = {
+    ...target,
+    command: kubectlCommand(kubectl, commandArgs)
+  };
+  if (kubectl.status !== "ready") {
+    addBlocker(blockers, "environment_blocker", `rollout-status-${target.name}`, kubectl.reason);
+    return {
+      ...base,
+      status: "blocked",
+      reason: kubectl.reason
+    };
+  }
+  const result = await kubectlResult(kubectl, commandArgs, 190000);
+  if (!result.ok) {
+    const reason = oneLine(commandOutput(result));
+    addBlocker(blockers, "runtime_blocker", `rollout-status-${target.name}`, reason);
+    return {
+      ...base,
+      status: "blocked",
+      reason,
+      stdout: result.redactedStdout ?? result.stdout,
+      stderr: result.redactedStderr ?? result.stderr
+    };
+  }
+  return {
+    ...base,
+    status: "pass",
+    stdout: result.redactedStdout ?? result.stdout,
+    stderr: result.redactedStderr ?? result.stderr
+  };
+}
+
+async function verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep) {
+  const targets = rolloutTargetsFromWorkloads(workloads);
+  if (!args.apply) {
+    return {
+      status: "not_run",
+      reason: "dry-run mode",
+      concurrency: args.rolloutConcurrency,
+      targets
+    };
+  }
+  if (applyStep.status !== "pass") {
+    return {
+      status: "not_run",
+      reason: "apply did not complete",
+      concurrency: args.rolloutConcurrency,
+      targets
+    };
+  }
+  const results = await mapWithConcurrency(targets, args.rolloutConcurrency, async (target) =>
+    verifyRolloutTarget(kubectl, target, blockers)
+  );
+  return {
+    status: results.every((result) => result.status === "pass") ? "pass" : "blocked",
+    concurrency: args.rolloutConcurrency,
+    targetCount: targets.length,
+    results
+  };
+}
+
 function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
   const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName);
   if (result.ok || plannedNames.length === 0) return false;
@@ -1758,8 +2032,8 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements, le
   const replacementsNeeded = templateJobReplacements.filter((replacement) => replacement.replace);
 
   if (args.apply) {
-    const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers);
-    const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers);
+    const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers, args.rolloutConcurrency);
+    const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers, args.rolloutConcurrency);
     mutationAttempted = replaceCount > 0 || cleanupCount > 0;
     if (blockers.length > 0) {
       return {
@@ -1858,6 +2132,7 @@ export async function runDevDeployApply(argv, io = {}) {
   const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId);
 
   const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers);
+  const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency);
   const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers);
   const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers);
   const codeAgentProviderDesiredState = inspectCodeAgentProviderDesiredState(deploy, workloads);
@@ -1886,33 +2161,41 @@ export async function runDevDeployApply(argv, io = {}) {
     templateJobReplacements,
     legacySimulatorDeploymentCleanups
   );
-  const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass"
-    ? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
+  let rolloutStatusAfterApply;
+  let cloudWebRolloutAfterApply;
+  let codeAgentProviderLiveAfterApply;
+  if (args.apply && applyStep.status === "pass") {
+    [rolloutStatusAfterApply, cloudWebRolloutAfterApply, codeAgentProviderLiveAfterApply] = await Promise.all([
+      verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep),
+      observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
         observationPhase: "after_apply",
         blockRuntimeEnvDrift: true
-      })
-    : skippedDeploymentRolloutObservation(
-        kubectl,
-        deploy,
-        catalog,
-        "hwlab-cloud-web",
-        "after_apply",
-        args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification"
-      );
-  const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply;
-  const codeAgentProviderLiveAfterApply = args.apply && applyStep.status === "pass"
-    ? await observeCodeAgentProviderLiveDeployment(kubectl, blockers, {
+      }),
+      observeCodeAgentProviderLiveDeployment(kubectl, blockers, {
         phase: "after_apply",
         blockOnMismatch: true
       })
-    : {
-        phase: "after_apply",
-        status: "not_run",
-        reason: args.apply ? "apply did not complete" : "dry-run mode",
-        secretValuesRead: false,
-        kubernetesSecretDataRead: false,
-        valuesRedacted: true
-      };
+    ]);
+  } else {
+    rolloutStatusAfterApply = await verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep);
+    cloudWebRolloutAfterApply = skippedDeploymentRolloutObservation(
+      kubectl,
+      deploy,
+      catalog,
+      "hwlab-cloud-web",
+      "after_apply",
+      args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification"
+    );
+    codeAgentProviderLiveAfterApply = {
+      phase: "after_apply",
+      status: "not_run",
+      reason: args.apply ? "apply did not complete" : "dry-run mode",
+      secretValuesRead: false,
+      kubernetesSecretDataRead: false,
+      valuesRedacted: true
+    };
+  }
+  const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply;
   const status = blockers.length > 0 ? "blocked" : "pass";
   const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence);
   const workloadPlan = buildWorkloadPlan(workloads);
@@ -1964,6 +2247,7 @@ export async function runDevDeployApply(argv, io = {}) {
       evidence: [
         `artifact services checked: ${artifactEvidence.length}`,
         `expected artifact commit: ${artifactPlan.expectedArtifactCommit}`,
+        `registry manifests: ${registryManifests.status} (${registryManifests.serviceCount} services)`,
         `namespace: ${namespace}`,
         `workloads planned: ${workloadPlan.length}`,
         `kubectl executor: ${kubectl.executor ?? "missing"}`,
@@ -1972,6 +2256,7 @@ export async function runDevDeployApply(argv, io = {}) {
         `live health: ${liveProbe.status}`,
         `code agent provider desired-state: ${codeAgentProviderDesiredState.status}`,
         `code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`,
+        `rollout status concurrency: ${args.rolloutConcurrency}`,
         `template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`,
         `legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`
       ],
@@ -1980,7 +2265,7 @@ export async function runDevDeployApply(argv, io = {}) {
     devPreconditions: {
       status,
       requirements: [
-        "DEV artifact catalog must contain CI publish evidence and registry digests",
+        "DEV artifact catalog must contain CI publish expectations and registry digests; CD must verify those tag/digest pairs directly against registry manifests",
         "kubectl must be available for D601 hwlab-dev and must not target PROD",
         "hwlab-cloud-api /health/live must report serviceId, dev environment, and DB env readiness without exposing secret values",
         "hwlab-cloud-api must declare and preserve OPENAI_API_KEY from hwlab-code-agent-provider/openai-api-key plus the DEV Code Agent egress proxy env without reading Secret values",
@@ -2009,6 +2294,7 @@ export async function runDevDeployApply(argv, io = {}) {
       cloudWebRollout,
       cloudWebRolloutBeforeApply,
       cloudWebRolloutAfterApply,
+      rolloutStatusAfterApply,
       codeAgentProvider: {
         desiredState: codeAgentProviderDesiredState,
         liveBeforeApply: codeAgentProviderLiveBeforeApply,
@@ -2024,6 +2310,7 @@ export async function runDevDeployApply(argv, io = {}) {
       rollbackHint: buildRollbackHint(kubectl, workloads),
       remainingBlockers,
       artifactEvidence,
+      registryManifests,
       cloudApiDb,
       k8sManifest,
       clusterObservation,

scripts/src/dev-deploy-apply.test.mjs

@@ -15,8 +15,10 @@ import {
   resolveDevKubeconfigSelection
 } from "./dev-deploy-apply.mjs";
 import {
+  devArtifactProducer,
   devCdTransactionGuardFailure,
   hasDevCdTransactionEnv,
+  requireDevArtifactSideEffectEnv,
   requireDevCdTransactionForSideEffect
 } from "./dev-cd-transaction-guard.mjs";
 
@@ -141,6 +143,32 @@ test("DEV CD transaction guard rejects direct legacy side-effect calls", () => {
   );
 });
 
+test("DEV artifact publish guard accepts CI artifact identity without CD lock", () => {
+  const ciEnv = {
+    HWLAB_CI_ARTIFACT_RUN_ID: "ci-artifact-123",
+    HWLAB_CI_ARTIFACT_RUN_OWNER: "tekton"
+  };
+  assert.equal(requireDevArtifactSideEffectEnv({
+    env: ciEnv,
+    script: "scripts/dev-artifact-publish.mjs",
+    mode: "--publish"
+  }), null);
+  assert.deepEqual(devArtifactProducer(ciEnv), {
+    kind: "ci-artifact",
+    runId: "ci-artifact-123",
+    owner: "tekton"
+  });
+
+  const failure = requireDevArtifactSideEffectEnv({
+    env: {},
+    script: "scripts/dev-artifact-publish.mjs",
+    mode: "--publish"
+  });
+  assert.equal(failure.error, "cd-transaction-required");
+  assert.match(failure.requiredEnv, /HWLAB_CI_ARTIFACT_RUN_ID/u);
+  assert.match(failure.entrypoint, /HWLAB_CI_ARTIFACT_RUN_ID/u);
+});
+
 test("matching allowlisted suspended template Job image does not replace", () => {
   const image = "127.0.0.1:5000/hwlab/hwlab-cli:73b379f";
   const decision = decideDevTemplateJobReplacement({