fix: verify dev registry manifests from runner pods

docs/reference/deployment-publish.md

@@ -68,6 +68,15 @@ worker 并立即返回有界 JSON。`status` 仅读 `state.json`，`logs` 默认
 渐进查看。任何正式 CLI 调用都不得因为 Docker build、registry push、
 `kubectl wait --timeout` 或 rollout status 在前台长时间占用上下文。
 
+runner pod 内执行 CD 时，`127.0.0.1:5000` 指向 pod 自身，不等于 D601 host 的
+loopback registry；但 image reference 必须继续保持 `127.0.0.1:5000/...`，因为
+k3s/containerd 拉镜像和 CI push 使用的是 host 视角。CD registry manifest 校验
+默认先走直接 HTTP；如果 pod loopback 被拒绝，则用 Docker host-network helper
+（`docker run --rm --pull=never --network host`）读取同一个 registry manifest。
+正式命令仍必须通过非阻塞入口提交；如需显式 registry HTTP 入口，可在提交时加
+`--registry-manifest-base-url <url>`，该参数只改变 manifest 校验 URL，不改变
+`deploy.json` 或 workload 里的镜像引用。
+
 `scripts/dev-artifact-publish.mjs`、`scripts/dev-cd-apply.mjs` 和其他内部脚本仍
 保留为 worker/backend：它们可以在后台 job 内执行长流程，但不能作为指挥面常规
 入口直接运行。临时手动排障可以直接调用底层脚本，但排障结论必须回填到正式

scripts/src/dev-cd-apply.mjs

@@ -79,6 +79,7 @@ export function parseArgs(argv) {
     targetRef: "origin/main",
     kubeconfig: null,
     kubeconfigSpecified: false,
+    registryManifestBaseUrl: null,
     lockName: defaultLockName,
     targetNamespace: defaultNamespace,
     ttlSeconds: defaultTtlSeconds,
@@ -135,6 +136,12 @@ export function parseArgs(argv) {
       args.kubeconfig = arg.slice("--kubeconfig=".length);
       args.kubeconfigSpecified = true;
       args.flags.add("--kubeconfig");
+    } else if (arg === "--registry-manifest-base-url") {
+      args.registryManifestBaseUrl = readOption(argv, ++index, arg);
+      args.flags.add(arg);
+    } else if (arg.startsWith("--registry-manifest-base-url=")) {
+      args.registryManifestBaseUrl = arg.slice("--registry-manifest-base-url=".length);
+      args.flags.add("--registry-manifest-base-url");
     } else if (arg === "--lock-name") {
       args.lockName = readOption(argv, ++index, arg);
       args.flags.add(arg);
@@ -2740,7 +2747,8 @@ export async function runDevCdApply(argv, io = {}) {
           "--confirmed-non-production",
           "--report",
           deployApplyReportPath,
-          ...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : [])
+          ...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : []),
+          ...(args.registryManifestBaseUrl ? ["--registry-manifest-base-url", args.registryManifestBaseUrl] : [])
         ],
         timeoutMs: 20 * 60 * 1000,
         reportPath: deployApplyReportPath

scripts/src/dev-deploy-apply.mjs

@@ -27,6 +27,10 @@ const healthPath = "/health/live";
 const defaultD601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
 const devKubeconfigEnvName = "HWLAB_DEV_KUBECONFIG";
 const kubeconfigEnvName = "KUBECONFIG";
+const registryManifestBaseUrlEnvName = "HWLAB_DEV_REGISTRY_MANIFEST_BASE_URL";
+const registryVerifyModeEnvName = "HWLAB_DEV_REGISTRY_VERIFY_MODE";
+const registryHostNetworkHelperImageEnvName = "HWLAB_DEV_REGISTRY_HOSTNET_HELPER_IMAGE";
+const defaultRegistryHostNetworkHelperImage = "unidesk-code-queue:d601";
 const devTemplateJobReplacementPolicy = {
   status: "active",
   namespace,
@@ -96,6 +100,7 @@ export function parseArgs(argv) {
   const errors = [];
   let kubeconfig = null;
   let kubeconfigSpecified = false;
+  let registryManifestBaseUrl = null;
   let reportPath = defaultReportPath;
   let rolloutConcurrency = defaultCdConcurrency;
 
@@ -160,6 +165,25 @@ export function parseArgs(argv) {
       rolloutConcurrency = parseBoundedInteger(arg.slice("--rollout-concurrency=".length), defaultCdConcurrency, 1, 8);
       continue;
     }
+    if (arg === "--registry-manifest-base-url") {
+      flags.add("--registry-manifest-base-url");
+      const next = argv[index + 1];
+      if (typeof next !== "string" || next.startsWith("--")) {
+        errors.push("--registry-manifest-base-url requires a non-empty URL value");
+      } else {
+        registryManifestBaseUrl = next;
+        index += 1;
+      }
+      continue;
+    }
+    if (arg.startsWith("--registry-manifest-base-url=")) {
+      flags.add("--registry-manifest-base-url");
+      registryManifestBaseUrl = arg.slice("--registry-manifest-base-url=".length);
+      if (!registryManifestBaseUrl.trim()) {
+        errors.push("--registry-manifest-base-url requires a non-empty URL value");
+      }
+      continue;
+    }
     flags.add(arg);
   }
 
@@ -171,6 +195,7 @@ export function parseArgs(argv) {
     writeReport: flags.has("--report-output"),
     reportPath,
     rolloutConcurrency,
+    registryManifestBaseUrl,
     kubeconfig,
     kubeconfigSpecified,
     errors,
@@ -730,13 +755,33 @@ function parseTaggedImageReference(image) {
   };
 }
 
-function registryManifestUrlForImage(image) {
+function registryManifestBaseUrlForParsed(parsed, options = {}) {
+  const env = options.env ?? process.env;
+  const explicitBaseUrl = options.registryManifestBaseUrl ?? env?.[registryManifestBaseUrlEnvName] ?? null;
+  if (typeof explicitBaseUrl === "string" && explicitBaseUrl.trim().length > 0) {
+    return {
+      baseUrl: explicitBaseUrl.trim().replace(/\/+$/u, ""),
+      source: options.registryManifestBaseUrl ? "flag:--registry-manifest-base-url" : `env:${registryManifestBaseUrlEnvName}`
+    };
+  }
+  return {
+    baseUrl: `http://${parsed.registry}`,
+    source: "image-registry"
+  };
+}
+
+export function registryManifestUrlForImage(image, options = {}) {
   const parsed = parseTaggedImageReference(image);
   if (!parsed) return null;
   const repositoryPath = parsed.repository.split("/").map((part) => encodeURIComponent(part)).join("/");
+  const manifestBase = registryManifestBaseUrlForParsed(parsed, options);
   return {
     ...parsed,
-    url: `http://${parsed.registry}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}`
+    url: `${manifestBase.baseUrl}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}`,
+    registryAccess: {
+      baseUrl: manifestBase.baseUrl,
+      source: manifestBase.source
+    }
   };
 }
 
@@ -775,16 +820,126 @@ function httpRequestText(url, { method = "GET", headers = {}, timeoutMs = 15000
   });
 }
 
-async function verifyCatalogRegistryManifest(service, blockers) {
+function registryVerifyMode(options = {}) {
+  const env = options.env ?? process.env;
+  const value = String(options.registryVerifyMode ?? env?.[registryVerifyModeEnvName] ?? "auto").trim();
+  if (["auto", "direct", "docker-host-network"].includes(value)) return value;
+  return "auto";
+}
+
+function dockerHostNetworkHelperImage(options = {}) {
+  const env = options.env ?? process.env;
+  return String(
+    options.registryHostNetworkHelperImage ??
+    env?.[registryHostNetworkHelperImageEnvName] ??
+    defaultRegistryHostNetworkHelperImage
+  ).trim();
+}
+
+async function readRegistryManifestViaDockerHostNetwork(parsed, options = {}) {
+  const runCommand = options.runCommand ?? commandResult;
+  const helperImage = dockerHostNetworkHelperImage(options);
+  const helperScript = `
+const http = require("node:http");
+const url = process.argv[1];
+const accept = process.argv[2];
+const req = http.request(url, {
+  method: "HEAD",
+  headers: { Accept: accept },
+  timeout: 15000
+}, (response) => {
+  const digest = response.headers["docker-content-digest"] || null;
+  response.resume();
+  response.on("end", () => {
+    console.log(JSON.stringify({ statusCode: response.statusCode || 0, headers: { "docker-content-digest": digest } }));
+  });
+});
+req.on("timeout", () => req.destroy(new Error("timeout after 15000ms")));
+req.on("error", (error) => {
+  console.error(error.message);
+  process.exitCode = 1;
+});
+req.end();
+`.trim();
+  const args = [
+    "run",
+    "--rm",
+    "--pull=never",
+    "--network",
+    "host",
+    helperImage,
+    "node",
+    "-e",
+    helperScript,
+    parsed.url,
+    registryAcceptHeader()
+  ];
+  const result = await runCommand("docker", args, 60000);
+  if (!result.ok) {
+    throw new Error(
+      `docker host-network registry manifest read failed with ${result.code ?? "unknown"}: ${oneLine(result.stderr || result.stdout || "no output")}`
+    );
+  }
+  const parsedOutput = JSON.parse(result.stdout || "{}");
+  return {
+    statusCode: Number.parseInt(String(parsedOutput.statusCode ?? 0), 10),
+    headers: parsedOutput.headers ?? {},
+    body: "",
+    registryAccess: {
+      mode: "docker-host-network",
+      helperImage,
+      directUrl: parsed.url,
+      directAccessSource: parsed.registryAccess?.source ?? null
+    }
+  };
+}
+
+async function readRegistryManifest(parsed, options = {}) {
+  const mode = registryVerifyMode(options);
+  const httpReader = options.httpRequestText ?? httpRequestText;
+  if (mode === "docker-host-network") {
+    return readRegistryManifestViaDockerHostNetwork(parsed, options);
+  }
+  try {
+    const response = await httpReader(parsed.url, {
+      method: "HEAD",
+      headers: {
+        Accept: registryAcceptHeader()
+      },
+      timeoutMs: 15000
+    });
+    return {
+      ...response,
+      registryAccess: {
+        mode: "direct",
+        url: parsed.url,
+        source: parsed.registryAccess?.source ?? null
+      }
+    };
+  } catch (error) {
+    if (mode === "direct") throw error;
+    const fallback = await readRegistryManifestViaDockerHostNetwork(parsed, options);
+    return {
+      ...fallback,
+      registryAccess: {
+        ...fallback.registryAccess,
+        directError: oneLine(error.message)
+      }
+    };
+  }
+}
+
+export async function verifyCatalogRegistryManifest(service, blockers, options = {}) {
   const image = service.image;
   const expectedDigest = service.digest;
-  const parsed = registryManifestUrlForImage(image);
+  const parsed = registryManifestUrlForImage(image, options);
   const base = {
     serviceId: service.serviceId,
     image,
     imageTag: service.imageTag ?? parseImageTag(image),
     expectedDigest,
-    url: parsed?.url ?? null
+    url: parsed?.url ?? null,
+    registryAccess: parsed?.registryAccess ?? null
   };
 
   if (!parsed) {
@@ -799,13 +954,7 @@ async function verifyCatalogRegistryManifest(service, blockers) {
   }
 
   try {
-    const response = await httpRequestText(parsed.url, {
-      method: "HEAD",
-      headers: {
-        Accept: registryAcceptHeader()
-      },
-      timeoutMs: 15000
-    });
+    const response = await readRegistryManifest(parsed, options);
     const observedDigest = String(response.headers["docker-content-digest"] ?? "").trim();
     const okStatus = response.statusCode >= 200 && response.statusCode < 300;
     const digestMatches = observedDigest === expectedDigest;
@@ -820,7 +969,8 @@ async function verifyCatalogRegistryManifest(service, blockers) {
         reason,
         httpStatus: response.statusCode,
         observedDigest,
-        digestMatches
+        digestMatches,
+        registryAccess: response.registryAccess ?? base.registryAccess
       };
     }
     return {
@@ -828,7 +978,8 @@ async function verifyCatalogRegistryManifest(service, blockers) {
       status: "pass",
       httpStatus: response.statusCode,
       observedDigest,
-      digestMatches: true
+      digestMatches: true,
+      registryAccess: response.registryAccess ?? base.registryAccess
     };
   } catch (error) {
     const reason = `${service.serviceId} registry manifest read failed: ${oneLine(error.message)}`;
@@ -841,10 +992,10 @@ async function verifyCatalogRegistryManifest(service, blockers) {
   }
 }
 
-async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency) {
+async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency, options = {}) {
   const services = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false);
   const results = await mapWithConcurrency(services, concurrency, async (service) =>
-    verifyCatalogRegistryManifest(service, blockers)
+    verifyCatalogRegistryManifest(service, blockers, options)
   );
   return {
     status: results.every((result) => result.status === "pass") ? "pass" : "blocked",
@@ -852,6 +1003,7 @@ async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = d
     releaseGate: true,
     serviceCount: results.length,
     concurrency,
+    verifyMode: registryVerifyMode(options),
     results
   };
 }
@@ -2137,7 +2289,10 @@ export async function runDevDeployApply(argv, io = {}) {
   const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId);
 
   const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers);
-  const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency);
+  const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency, {
+    env: io.env ?? process.env,
+    registryManifestBaseUrl: args.registryManifestBaseUrl
+  });
   const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers);
   const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers);
   const codeAgentProviderDesiredState = inspectCodeAgentProviderDesiredState(deploy, workloads);

scripts/src/dev-deploy-apply.test.mjs

@@ -11,8 +11,10 @@ import {
   inspectCodeAgentProviderLiveDeployment,
   parseArgs,
   redactSensitiveText,
+  registryManifestUrlForImage,
   resolveApplySourceCommit,
-  resolveDevKubeconfigSelection
+  resolveDevKubeconfigSelection,
+  verifyCatalogRegistryManifest
 } from "./dev-deploy-apply.mjs";
 import {
   devArtifactProducer,
@@ -511,6 +513,60 @@ test("parseArgs reports a missing kubeconfig path as a blocking error", () => {
   assert.deepEqual(args.errors, ["--kubeconfig requires a non-empty path value"]);
 });
 
+test("registry manifest URL can be redirected without changing deploy image identity", () => {
+  const parsed = registryManifestUrlForImage("127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", {
+    registryManifestBaseUrl: "http://hwlab-registry-proxy.hwlab-dev.svc.cluster.local:5000/"
+  });
+
+  assert.equal(parsed.registry, "127.0.0.1:5000");
+  assert.equal(parsed.repository, "hwlab/hwlab-cloud-api");
+  assert.equal(parsed.url, "http://hwlab-registry-proxy.hwlab-dev.svc.cluster.local:5000/v2/hwlab/hwlab-cloud-api/manifests/abc1234");
+  assert.equal(parsed.registryAccess.source, "flag:--registry-manifest-base-url");
+});
+
+test("registry manifest verification falls back to docker host-network helper when pod loopback is blocked", async () => {
+  const expectedDigest = "sha256:2cc48ca2bd0e260d7ea6df1e1a791df6615a87da3d89f4b11103735a1e2d9b63";
+  const blockers = [];
+  const commands = [];
+  const result = await verifyCatalogRegistryManifest(
+    {
+      serviceId: "hwlab-cloud-api",
+      image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:fad9fc9",
+      imageTag: "fad9fc9",
+      digest: expectedDigest
+    },
+    blockers,
+    {
+      env: {
+        HWLAB_DEV_REGISTRY_HOSTNET_HELPER_IMAGE: "helper:local"
+      },
+      httpRequestText: async () => {
+        throw new Error("connect ECONNREFUSED 127.0.0.1:5000");
+      },
+      runCommand: async (command, args) => {
+        commands.push([command, args]);
+        return {
+          ok: true,
+          code: 0,
+          stdout: JSON.stringify({
+            statusCode: 200,
+            headers: { "docker-content-digest": expectedDigest }
+          }),
+          stderr: ""
+        };
+      }
+    }
+  );
+
+  assert.equal(result.status, "pass");
+  assert.equal(result.registryAccess.mode, "docker-host-network");
+  assert.equal(result.registryAccess.helperImage, "helper:local");
+  assert.match(result.registryAccess.directError, /ECONNREFUSED/u);
+  assert.deepEqual(blockers, []);
+  assert.equal(commands[0][0], "docker");
+  assert.deepEqual(commands[0][1].slice(0, 6), ["run", "--rm", "--pull=never", "--network", "host", "helper:local"]);
+});
+
 test("redaction removes common token and password material from kubectl output", () => {
   const output = redactSensitiveText(
     [

tools/hwlab-cli/lib/cicd-jobs.mjs

@@ -109,6 +109,8 @@ function buildJobSpec({ kind, flags, options, repoRoot, env, now }) {
     if (flags.has("--break-stale-lock")) args.push("--break-stale-lock");
     const concurrency = optionValue(options, "--concurrency");
     if (concurrency) envPatch.HWLAB_DEV_CD_CONCURRENCY = String(positiveInteger(concurrency, 4));
+    const registryManifestBaseUrl = optionValue(options, "--registry-manifest-base-url");
+    if (registryManifestBaseUrl) args.push("--registry-manifest-base-url", registryManifestBaseUrl);
   }
 
   if (optionValue(options, "--target-ref")) args.push("--target-ref", optionValue(options, "--target-ref"));