diff --git a/docs/reference/deployment-publish.md b/docs/reference/deployment-publish.md index 94cb88c0..b6e4f968 100644 --- a/docs/reference/deployment-publish.md +++ b/docs/reference/deployment-publish.md @@ -68,6 +68,15 @@ worker 并立即返回有界 JSON。`status` 仅读 `state.json`,`logs` 默认 渐进查看。任何正式 CLI 调用都不得因为 Docker build、registry push、 `kubectl wait --timeout` 或 rollout status 在前台长时间占用上下文。 +runner pod 内执行 CD 时,`127.0.0.1:5000` 指向 pod 自身,不等于 D601 host 的 +loopback registry;但 image reference 必须继续保持 `127.0.0.1:5000/...`,因为 +k3s/containerd 拉镜像和 CI push 使用的是 host 视角。CD registry manifest 校验 +默认先走直接 HTTP;如果 pod loopback 被拒绝,则用 Docker host-network helper +(`docker run --rm --pull=never --network host`)读取同一个 registry manifest。 +正式命令仍必须通过非阻塞入口提交;如需显式 registry HTTP 入口,可在提交时加 +`--registry-manifest-base-url `,该参数只改变 manifest 校验 URL,不改变 +`deploy.json` 或 workload 里的镜像引用。 + `scripts/dev-artifact-publish.mjs`、`scripts/dev-cd-apply.mjs` 和其他内部脚本仍 保留为 worker/backend:它们可以在后台 job 内执行长流程,但不能作为指挥面常规 入口直接运行。临时手动排障可以直接调用底层脚本,但排障结论必须回填到正式 diff --git a/scripts/src/dev-cd-apply.mjs b/scripts/src/dev-cd-apply.mjs index 070ae11e..c881a04c 100644 --- a/scripts/src/dev-cd-apply.mjs +++ b/scripts/src/dev-cd-apply.mjs @@ -79,6 +79,7 @@ export function parseArgs(argv) { targetRef: "origin/main", kubeconfig: null, kubeconfigSpecified: false, + registryManifestBaseUrl: null, lockName: defaultLockName, targetNamespace: defaultNamespace, ttlSeconds: defaultTtlSeconds, @@ -135,6 +136,12 @@ export function parseArgs(argv) { args.kubeconfig = arg.slice("--kubeconfig=".length); args.kubeconfigSpecified = true; args.flags.add("--kubeconfig"); + } else if (arg === "--registry-manifest-base-url") { + args.registryManifestBaseUrl = readOption(argv, ++index, arg); + args.flags.add(arg); + } else if (arg.startsWith("--registry-manifest-base-url=")) { + args.registryManifestBaseUrl = arg.slice("--registry-manifest-base-url=".length); + args.flags.add("--registry-manifest-base-url"); } else if (arg === "--lock-name") { args.lockName = readOption(argv, ++index, arg); args.flags.add(arg); @@ -2740,7 +2747,8 @@ export async function runDevCdApply(argv, io = {}) { "--confirmed-non-production", "--report", deployApplyReportPath, - ...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : []) + ...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : []), + ...(args.registryManifestBaseUrl ? ["--registry-manifest-base-url", args.registryManifestBaseUrl] : []) ], timeoutMs: 20 * 60 * 1000, reportPath: deployApplyReportPath diff --git a/scripts/src/dev-deploy-apply.mjs b/scripts/src/dev-deploy-apply.mjs index d3eeba5c..773757a0 100755 --- a/scripts/src/dev-deploy-apply.mjs +++ b/scripts/src/dev-deploy-apply.mjs @@ -27,6 +27,10 @@ const healthPath = "/health/live"; const defaultD601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml"; const devKubeconfigEnvName = "HWLAB_DEV_KUBECONFIG"; const kubeconfigEnvName = "KUBECONFIG"; +const registryManifestBaseUrlEnvName = "HWLAB_DEV_REGISTRY_MANIFEST_BASE_URL"; +const registryVerifyModeEnvName = "HWLAB_DEV_REGISTRY_VERIFY_MODE"; +const registryHostNetworkHelperImageEnvName = "HWLAB_DEV_REGISTRY_HOSTNET_HELPER_IMAGE"; +const defaultRegistryHostNetworkHelperImage = "unidesk-code-queue:d601"; const devTemplateJobReplacementPolicy = { status: "active", namespace, @@ -96,6 +100,7 @@ export function parseArgs(argv) { const errors = []; let kubeconfig = null; let kubeconfigSpecified = false; + let registryManifestBaseUrl = null; let reportPath = defaultReportPath; let rolloutConcurrency = defaultCdConcurrency; @@ -160,6 +165,25 @@ export function parseArgs(argv) { rolloutConcurrency = parseBoundedInteger(arg.slice("--rollout-concurrency=".length), defaultCdConcurrency, 1, 8); continue; } + if (arg === "--registry-manifest-base-url") { + flags.add("--registry-manifest-base-url"); + const next = argv[index + 1]; + if (typeof next !== "string" || next.startsWith("--")) { + errors.push("--registry-manifest-base-url requires a non-empty URL value"); + } else { + registryManifestBaseUrl = next; + index += 1; + } + continue; + } + if (arg.startsWith("--registry-manifest-base-url=")) { + flags.add("--registry-manifest-base-url"); + registryManifestBaseUrl = arg.slice("--registry-manifest-base-url=".length); + if (!registryManifestBaseUrl.trim()) { + errors.push("--registry-manifest-base-url requires a non-empty URL value"); + } + continue; + } flags.add(arg); } @@ -171,6 +195,7 @@ export function parseArgs(argv) { writeReport: flags.has("--report-output"), reportPath, rolloutConcurrency, + registryManifestBaseUrl, kubeconfig, kubeconfigSpecified, errors, @@ -730,13 +755,33 @@ function parseTaggedImageReference(image) { }; } -function registryManifestUrlForImage(image) { +function registryManifestBaseUrlForParsed(parsed, options = {}) { + const env = options.env ?? process.env; + const explicitBaseUrl = options.registryManifestBaseUrl ?? env?.[registryManifestBaseUrlEnvName] ?? null; + if (typeof explicitBaseUrl === "string" && explicitBaseUrl.trim().length > 0) { + return { + baseUrl: explicitBaseUrl.trim().replace(/\/+$/u, ""), + source: options.registryManifestBaseUrl ? "flag:--registry-manifest-base-url" : `env:${registryManifestBaseUrlEnvName}` + }; + } + return { + baseUrl: `http://${parsed.registry}`, + source: "image-registry" + }; +} + +export function registryManifestUrlForImage(image, options = {}) { const parsed = parseTaggedImageReference(image); if (!parsed) return null; const repositoryPath = parsed.repository.split("/").map((part) => encodeURIComponent(part)).join("/"); + const manifestBase = registryManifestBaseUrlForParsed(parsed, options); return { ...parsed, - url: `http://${parsed.registry}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}` + url: `${manifestBase.baseUrl}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}`, + registryAccess: { + baseUrl: manifestBase.baseUrl, + source: manifestBase.source + } }; } @@ -775,16 +820,126 @@ function httpRequestText(url, { method = "GET", headers = {}, timeoutMs = 15000 }); } -async function verifyCatalogRegistryManifest(service, blockers) { +function registryVerifyMode(options = {}) { + const env = options.env ?? process.env; + const value = String(options.registryVerifyMode ?? env?.[registryVerifyModeEnvName] ?? "auto").trim(); + if (["auto", "direct", "docker-host-network"].includes(value)) return value; + return "auto"; +} + +function dockerHostNetworkHelperImage(options = {}) { + const env = options.env ?? process.env; + return String( + options.registryHostNetworkHelperImage ?? + env?.[registryHostNetworkHelperImageEnvName] ?? + defaultRegistryHostNetworkHelperImage + ).trim(); +} + +async function readRegistryManifestViaDockerHostNetwork(parsed, options = {}) { + const runCommand = options.runCommand ?? commandResult; + const helperImage = dockerHostNetworkHelperImage(options); + const helperScript = ` +const http = require("node:http"); +const url = process.argv[1]; +const accept = process.argv[2]; +const req = http.request(url, { + method: "HEAD", + headers: { Accept: accept }, + timeout: 15000 +}, (response) => { + const digest = response.headers["docker-content-digest"] || null; + response.resume(); + response.on("end", () => { + console.log(JSON.stringify({ statusCode: response.statusCode || 0, headers: { "docker-content-digest": digest } })); + }); +}); +req.on("timeout", () => req.destroy(new Error("timeout after 15000ms"))); +req.on("error", (error) => { + console.error(error.message); + process.exitCode = 1; +}); +req.end(); +`.trim(); + const args = [ + "run", + "--rm", + "--pull=never", + "--network", + "host", + helperImage, + "node", + "-e", + helperScript, + parsed.url, + registryAcceptHeader() + ]; + const result = await runCommand("docker", args, 60000); + if (!result.ok) { + throw new Error( + `docker host-network registry manifest read failed with ${result.code ?? "unknown"}: ${oneLine(result.stderr || result.stdout || "no output")}` + ); + } + const parsedOutput = JSON.parse(result.stdout || "{}"); + return { + statusCode: Number.parseInt(String(parsedOutput.statusCode ?? 0), 10), + headers: parsedOutput.headers ?? {}, + body: "", + registryAccess: { + mode: "docker-host-network", + helperImage, + directUrl: parsed.url, + directAccessSource: parsed.registryAccess?.source ?? null + } + }; +} + +async function readRegistryManifest(parsed, options = {}) { + const mode = registryVerifyMode(options); + const httpReader = options.httpRequestText ?? httpRequestText; + if (mode === "docker-host-network") { + return readRegistryManifestViaDockerHostNetwork(parsed, options); + } + try { + const response = await httpReader(parsed.url, { + method: "HEAD", + headers: { + Accept: registryAcceptHeader() + }, + timeoutMs: 15000 + }); + return { + ...response, + registryAccess: { + mode: "direct", + url: parsed.url, + source: parsed.registryAccess?.source ?? null + } + }; + } catch (error) { + if (mode === "direct") throw error; + const fallback = await readRegistryManifestViaDockerHostNetwork(parsed, options); + return { + ...fallback, + registryAccess: { + ...fallback.registryAccess, + directError: oneLine(error.message) + } + }; + } +} + +export async function verifyCatalogRegistryManifest(service, blockers, options = {}) { const image = service.image; const expectedDigest = service.digest; - const parsed = registryManifestUrlForImage(image); + const parsed = registryManifestUrlForImage(image, options); const base = { serviceId: service.serviceId, image, imageTag: service.imageTag ?? parseImageTag(image), expectedDigest, - url: parsed?.url ?? null + url: parsed?.url ?? null, + registryAccess: parsed?.registryAccess ?? null }; if (!parsed) { @@ -799,13 +954,7 @@ async function verifyCatalogRegistryManifest(service, blockers) { } try { - const response = await httpRequestText(parsed.url, { - method: "HEAD", - headers: { - Accept: registryAcceptHeader() - }, - timeoutMs: 15000 - }); + const response = await readRegistryManifest(parsed, options); const observedDigest = String(response.headers["docker-content-digest"] ?? "").trim(); const okStatus = response.statusCode >= 200 && response.statusCode < 300; const digestMatches = observedDigest === expectedDigest; @@ -820,7 +969,8 @@ async function verifyCatalogRegistryManifest(service, blockers) { reason, httpStatus: response.statusCode, observedDigest, - digestMatches + digestMatches, + registryAccess: response.registryAccess ?? base.registryAccess }; } return { @@ -828,7 +978,8 @@ async function verifyCatalogRegistryManifest(service, blockers) { status: "pass", httpStatus: response.statusCode, observedDigest, - digestMatches: true + digestMatches: true, + registryAccess: response.registryAccess ?? base.registryAccess }; } catch (error) { const reason = `${service.serviceId} registry manifest read failed: ${oneLine(error.message)}`; @@ -841,10 +992,10 @@ async function verifyCatalogRegistryManifest(service, blockers) { } } -async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency) { +async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency, options = {}) { const services = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false); const results = await mapWithConcurrency(services, concurrency, async (service) => - verifyCatalogRegistryManifest(service, blockers) + verifyCatalogRegistryManifest(service, blockers, options) ); return { status: results.every((result) => result.status === "pass") ? "pass" : "blocked", @@ -852,6 +1003,7 @@ async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = d releaseGate: true, serviceCount: results.length, concurrency, + verifyMode: registryVerifyMode(options), results }; } @@ -2137,7 +2289,10 @@ export async function runDevDeployApply(argv, io = {}) { const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId); const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers); - const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency); + const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency, { + env: io.env ?? process.env, + registryManifestBaseUrl: args.registryManifestBaseUrl + }); const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers); const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers); const codeAgentProviderDesiredState = inspectCodeAgentProviderDesiredState(deploy, workloads); diff --git a/scripts/src/dev-deploy-apply.test.mjs b/scripts/src/dev-deploy-apply.test.mjs index 8b2228e4..6acc9c82 100644 --- a/scripts/src/dev-deploy-apply.test.mjs +++ b/scripts/src/dev-deploy-apply.test.mjs @@ -11,8 +11,10 @@ import { inspectCodeAgentProviderLiveDeployment, parseArgs, redactSensitiveText, + registryManifestUrlForImage, resolveApplySourceCommit, - resolveDevKubeconfigSelection + resolveDevKubeconfigSelection, + verifyCatalogRegistryManifest } from "./dev-deploy-apply.mjs"; import { devArtifactProducer, @@ -511,6 +513,60 @@ test("parseArgs reports a missing kubeconfig path as a blocking error", () => { assert.deepEqual(args.errors, ["--kubeconfig requires a non-empty path value"]); }); +test("registry manifest URL can be redirected without changing deploy image identity", () => { + const parsed = registryManifestUrlForImage("127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", { + registryManifestBaseUrl: "http://hwlab-registry-proxy.hwlab-dev.svc.cluster.local:5000/" + }); + + assert.equal(parsed.registry, "127.0.0.1:5000"); + assert.equal(parsed.repository, "hwlab/hwlab-cloud-api"); + assert.equal(parsed.url, "http://hwlab-registry-proxy.hwlab-dev.svc.cluster.local:5000/v2/hwlab/hwlab-cloud-api/manifests/abc1234"); + assert.equal(parsed.registryAccess.source, "flag:--registry-manifest-base-url"); +}); + +test("registry manifest verification falls back to docker host-network helper when pod loopback is blocked", async () => { + const expectedDigest = "sha256:2cc48ca2bd0e260d7ea6df1e1a791df6615a87da3d89f4b11103735a1e2d9b63"; + const blockers = []; + const commands = []; + const result = await verifyCatalogRegistryManifest( + { + serviceId: "hwlab-cloud-api", + image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:fad9fc9", + imageTag: "fad9fc9", + digest: expectedDigest + }, + blockers, + { + env: { + HWLAB_DEV_REGISTRY_HOSTNET_HELPER_IMAGE: "helper:local" + }, + httpRequestText: async () => { + throw new Error("connect ECONNREFUSED 127.0.0.1:5000"); + }, + runCommand: async (command, args) => { + commands.push([command, args]); + return { + ok: true, + code: 0, + stdout: JSON.stringify({ + statusCode: 200, + headers: { "docker-content-digest": expectedDigest } + }), + stderr: "" + }; + } + } + ); + + assert.equal(result.status, "pass"); + assert.equal(result.registryAccess.mode, "docker-host-network"); + assert.equal(result.registryAccess.helperImage, "helper:local"); + assert.match(result.registryAccess.directError, /ECONNREFUSED/u); + assert.deepEqual(blockers, []); + assert.equal(commands[0][0], "docker"); + assert.deepEqual(commands[0][1].slice(0, 6), ["run", "--rm", "--pull=never", "--network", "host", "helper:local"]); +}); + test("redaction removes common token and password material from kubectl output", () => { const output = redactSensitiveText( [ diff --git a/tools/hwlab-cli/lib/cicd-jobs.mjs b/tools/hwlab-cli/lib/cicd-jobs.mjs index 3a246852..79920195 100644 --- a/tools/hwlab-cli/lib/cicd-jobs.mjs +++ b/tools/hwlab-cli/lib/cicd-jobs.mjs @@ -109,6 +109,8 @@ function buildJobSpec({ kind, flags, options, repoRoot, env, now }) { if (flags.has("--break-stale-lock")) args.push("--break-stale-lock"); const concurrency = optionValue(options, "--concurrency"); if (concurrency) envPatch.HWLAB_DEV_CD_CONCURRENCY = String(positiveInteger(concurrency, 4)); + const registryManifestBaseUrl = optionValue(options, "--registry-manifest-base-url"); + if (registryManifestBaseUrl) args.push("--registry-manifest-base-url", registryManifestBaseUrl); } if (optionValue(options, "--target-ref")) args.push("--target-ref", optionValue(options, "--target-ref"));