From 358f73fe30563ede86fe11f28608e900d4cf65a8 Mon Sep 17 00:00:00 2001 From: lyon Date: Thu, 25 Jun 2026 09:49:28 +0800 Subject: [PATCH] fix: reuse current env hash registry tags --- scripts/artifact-publish.mjs | 8 +-- scripts/ci-plan.test.mjs | 95 ++++++++++++++++++++++++++++++++++++ scripts/src/ci-plan-lib.mjs | 73 +++++++++++++++++++++------ 3 files changed, 158 insertions(+), 18 deletions(-) diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index c62915d9..7b0a3154 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -1786,8 +1786,8 @@ function serviceImageCatalogProvenance(catalogService) { } function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commitId }) { - const environmentImage = catalogService?.environmentImage ?? service.environmentImage ?? null; - const environmentDigest = catalogService?.environmentDigest ?? service.environmentDigest ?? "not_published"; + const environmentImage = service.environmentImage ?? catalogService?.environmentImage ?? null; + const environmentDigest = service.environmentDigest ?? catalogService?.environmentDigest ?? "not_published"; const imageTag = imageTagFromReference(environmentImage); const bootCommit = service.codeChanged === true || service.envChanged === true ? service.bootCommit ?? commitId @@ -1813,7 +1813,7 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit buildCreatedAt: catalogService?.buildCreatedAt ?? null, buildSource: catalogService?.buildSource ?? null, buildBackend: "reused-env-catalog", - repositoryDigest: catalogService?.repositoryDigest ?? repositoryDigestFor(environmentImage, environmentDigest), + repositoryDigest: service.repositoryDigest ?? repositoryDigestFor(environmentImage, environmentDigest) ?? catalogService?.repositoryDigest ?? null, bootEnvEvidence: bootEnvEvidence({ service: { ...service, bootCommit }, commitId: bootCommit, environmentImage, environmentDigest }) }; if (!environmentImage || !imageTag || !shaDigestPattern.test(environmentDigest)) { @@ -1827,7 +1827,7 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit return { ...base, status: "reused", - reusedFrom: service.reuse?.reusedFrom ?? catalogService?.environmentInputHash ?? catalogService?.componentInputHash ?? imageTag, + reusedFrom: service.reuse?.reusedFrom ?? service.environmentInputHash ?? catalogService?.environmentInputHash ?? catalogService?.componentInputHash ?? imageTag, notPublishedReason: null }; } diff --git a/scripts/ci-plan.test.mjs b/scripts/ci-plan.test.mjs index d73a3ddf..44b1d504 100644 --- a/scripts/ci-plan.test.mjs +++ b/scripts/ci-plan.test.mjs @@ -184,6 +184,50 @@ test("v03 planner rebuilds env image when catalog env input hash is stale", asyn assert.deepEqual(plan.buildServices, ["hwlab-cloud-api"]); }); +test("v03 planner reuses current env hash tag when registry already has it", async () => { + const repo = await createFixtureRepo(); + const initialSha = (await git(repo, ["rev-parse", "HEAD"])).stdout.trim(); + const initialPlan = await createCiPlan({ + repoRoot: repo, + lane: "v03", + baseRef: "HEAD", + targetRef: initialSha, + artifactCatalogPath: "deploy/artifact-catalog.v03.json", + services: ["hwlab-cloud-api"] + }); + const catalog = createCatalogFixture("ready", catalogOptionsFromPlan(initialPlan, { sourceCommitId: initialSha })); + catalog.services.find((service) => service.serviceId === "hwlab-cloud-api").environmentInputHash = "stale-environment-input"; + await writeFile(path.join(repo, "deploy/artifact-catalog.v03.json"), JSON.stringify(catalog, null, 2)); + await git(repo, ["add", "deploy/artifact-catalog.v03.json"]); + await git(repo, ["commit", "-m", "seed stale v03 env catalog"]); + + const digest = `sha256:${"2".repeat(64)}`; + let probeRequest = null; + const plan = await createCiPlan({ + repoRoot: repo, + lane: "v03", + baseRef: "HEAD~1", + targetRef: "HEAD", + artifactCatalogPath: "deploy/artifact-catalog.v03.json", + services: ["hwlab-cloud-api"], + verifyReuseRegistry: true, + reuseRegistryProbe: async (request) => { + probeRequest = request; + return { status: "present", method: "HEAD", statusCode: 200, digest }; + } + }); + const cloudApi = plan.services.find((service) => service.serviceId === "hwlab-cloud-api"); + assert.equal(cloudApi.environmentInputChanged, true); + assert.equal(cloudApi.envChanged, false); + assert.equal(cloudApi.buildRequired, false); + assert.equal(cloudApi.environmentDigest, digest); + assert.equal(cloudApi.reuse.status, "ready"); + assert.equal(cloudApi.reuse.reusedFrom, cloudApi.environmentInputHash); + assert.equal(probeRequest.digest, null); + assert.equal(probeRequest.image, `127.0.0.1:5000/hwlab/hwlab-cloud-api-env:env-${cloudApi.environmentInputHash.slice(0, 12)}`); + assert.deepEqual(plan.buildServices, []); +}); + test("v03 planner rebuilds env image when target registry lacks reused digest", async () => { const repo = await createFixtureRepo(); const initialSha = (await git(repo, ["rev-parse", "HEAD"])).stdout.trim(); @@ -705,6 +749,57 @@ test("v02 planner reads env reuse declarations and recipe from deploy config", a assert.equal(recipe.launcherInputPaths.includes("custom/env-tool.mjs"), true); }); +test("planner sanitizes k8s native download no_proxy to internal targets", async () => { + const repo = await createFixtureRepo(); + const previousEnv = { + HWLAB_TEKTON_PIPELINERUN: process.env.HWLAB_TEKTON_PIPELINERUN, + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + NO_PROXY: process.env.NO_PROXY, + no_proxy: process.env.no_proxy + }; + process.env.HWLAB_TEKTON_PIPELINERUN = "test-pipelinerun"; + process.env.HTTP_PROXY = "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808"; + process.env.HTTPS_PROXY = "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808"; + process.env.NO_PROXY = [ + "127.0.0.1", + "localhost", + "10.43.0.0/16", + "git-mirror-http.devops-infra.svc.cluster.local", + "deb.debian.org", + ".debian.org", + "goproxy.cn", + ".goproxy.cn", + "hyueapi.com", + ".hyueapi.com" + ].join(","); + delete process.env.no_proxy; + try { + const plan = await createCiPlan({ + repoRoot: repo, + lane: "v03", + baseRef: "HEAD", + targetRef: "HEAD", + artifactCatalogPath: "deploy/artifact-catalog.v03.json", + services: ["hwlab-cloud-api"] + }); + const noProxy = plan.services[0].envReuseRecipe.downloadStack.noProxy; + assert.deepEqual(noProxy, [ + "127.0.0.1", + "localhost", + "10.43.0.0/16", + "git-mirror-http.devops-infra.svc.cluster.local", + "hyueapi.com", + ".hyueapi.com" + ]); + } finally { + for (const [key, value] of Object.entries(previousEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +}); + test("v03 planner reads env reuse declarations and catalog from v03 lane config", async () => { const repo = await createFixtureRepo(); diff --git a/scripts/src/ci-plan-lib.mjs b/scripts/src/ci-plan-lib.mjs index 6d0f1343..673795c0 100644 --- a/scripts/src/ci-plan-lib.mjs +++ b/scripts/src/ci-plan-lib.mjs @@ -130,11 +130,10 @@ export async function createCiPlan(options = {}) { const codeInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, codeInputPaths, { skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath) }) : null; - const environmentImage = envReuse ? environmentImageFromCatalog(model.serviceId, catalogRecord) : null; - const environmentDigest = envReuse ? environmentDigestFromCatalog(catalogRecord) : null; - const environmentReady = envReuse && /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? "") && Boolean(environmentImage); + const environmentImage = envReuse ? envReuseImageRef(registryPrefix, model.serviceId, environmentInputHash) : null; + let environmentDigest = envReuse ? environmentDigestFromCatalogForHash(catalogRecord, environmentInputHash) : null; const environmentInputChanged = envReuse ? catalogRecord?.environmentInputHash !== environmentInputHash : null; - const reuseRegistry = envReuse && reuseRegistryProbe && environmentReady + const reuseRegistry = envReuse && reuseRegistryProbe && environmentImage ? await reuseRegistryProbe({ serviceId: model.serviceId, image: environmentImage, @@ -143,8 +142,13 @@ export async function createCiPlan(options = {}) { timeoutMs: Number.isFinite(registryProbeTimeoutMs) ? registryProbeTimeoutMs : 3000 }) : null; + if (reuseRegistry?.status === "present" && /^sha256:[a-f0-9]{64}$/u.test(reuseRegistry.digest ?? "")) { + environmentDigest = reuseRegistry.digest; + } const reuseRegistryUnavailable = Boolean(reuseRegistry && reuseRegistry.status !== "present"); - const envChanged = envReuse ? Boolean(relevantEnvMatches.length > 0 || !environmentReady || environmentInputChanged || reuseRegistryUnavailable) : null; + const digestReady = /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? ""); + const environmentReady = envReuse && Boolean(environmentImage) && (reuseRegistryProbe ? reuseRegistry?.status === "present" && digestReady : !environmentInputChanged && digestReady); + const envChanged = envReuse ? Boolean(relevantEnvMatches.length > 0 || !environmentReady || reuseRegistryUnavailable) : null; const codeChanged = envReuse ? relevantCodeMatches.length > 0 || catalogRecord?.codeInputHash !== codeInputHash : null; const componentInputPaths = uniqueSorted([ ...model.componentPaths, @@ -188,7 +192,7 @@ export async function createCiPlan(options = {}) { } : catalogRecord; const componentInputChanged = !envReuse && catalogComponentProvenance?.safeToReuse !== true; const catalogReuse = envReuse - ? envReuseCandidate(catalogRecord, envChanged) + ? envReuseCandidate({ catalogRecord, envChanged, environmentImage, environmentDigest, environmentInputHash }) : reuseCandidate(catalogRecordForReuse, imageRelevantChangedPaths.length > 0, componentInputChanged); const affected = envReuse ? Boolean(envChanged || codeChanged || runtimeConfigChanged) @@ -494,7 +498,7 @@ function effectiveDownloadStackConfig(value, env = process.env) { if (!configured || !usesK8sNativeDownloadProxy(env)) return configured; const httpProxy = normalizeDeclarationString(env.HTTP_PROXY) || normalizeDeclarationString(env.http_proxy) || configured.httpProxy; const httpsProxy = normalizeDeclarationString(env.HTTPS_PROXY) || normalizeDeclarationString(env.https_proxy) || httpProxy || configured.httpsProxy; - const noProxy = noProxyFromEnv(env) || configured.noProxy; + const noProxy = sanitizeK8sDownloadNoProxy(noProxyFromEnv(env) || configured.noProxy); return { ...configured, httpProxy, @@ -513,6 +517,22 @@ function noProxyFromEnv(env = process.env) { return text.split(",").map((item) => item.trim()).filter(Boolean); } +function sanitizeK8sDownloadNoProxy(values) { + return uniquePreserveOrder(normalizeStringList(values, []).filter(isK8sDownloadNoProxyEntry)); +} + +function isK8sDownloadNoProxyEntry(value) { + const item = String(value ?? "").trim().toLowerCase(); + if (!item || item === "*") return false; + if (["localhost", "::1", "[::1]", "hyueapi.com", ".hyueapi.com"].includes(item)) return true; + if (item.endsWith(".hyueapi.com")) return true; + if (item === ".svc" || item === ".svc.cluster.local" || item === ".cluster.local") return true; + if (item.endsWith(".svc") || item.endsWith(".svc.cluster.local") || item.endsWith(".cluster.local")) return true; + if (/^(127|10)\./u.test(item) || /^192\.168\./u.test(item) || /^172\.(1[6-9]|2\d|3[01])\./u.test(item)) return true; + if (/^(127|10)\.[0-9./:-]+$/u.test(item) || /^192\.168\.[0-9./:-]+$/u.test(item) || /^172\.(1[6-9]|2\d|3[01])\.[0-9./:-]+$/u.test(item)) return true; + return false; +} + function normalizeOptionalHttpUrl(value, label) { const text = normalizeDeclarationString(value); if (!text) return null; @@ -896,8 +916,16 @@ function text(value) { return String(value ?? "").trim(); } -function envReuseCandidate(catalogRecord, envChanged) { +function envReuseCandidate({ catalogRecord, envChanged, environmentImage, environmentDigest, environmentInputHash }) { if (envChanged) return { status: "not-reused", reason: "environment-affected" }; + if (environmentImage && /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? "")) { + return { + status: "ready", + image: environmentImage, + digest: environmentDigest, + reusedFrom: environmentInputHash ?? catalogRecord?.environmentInputHash ?? catalogRecord?.commitId ?? catalogRecord?.imageTag ?? null + }; + } if (!catalogRecord) return { status: "candidate-no-catalog", reason: "no-previous-artifact-record" }; const image = environmentImageFromCatalog(catalogRecord.serviceId, catalogRecord); const digest = environmentDigestFromCatalog(catalogRecord); @@ -923,6 +951,15 @@ function environmentDigestFromCatalog(catalogRecord) { return catalogRecord?.environmentDigest ?? (catalogRecord?.environmentImage ? catalogRecord?.digest : null) ?? null; } +function environmentDigestFromCatalogForHash(catalogRecord, environmentInputHash) { + return catalogRecord?.environmentInputHash === environmentInputHash ? environmentDigestFromCatalog(catalogRecord) : null; +} + +function envReuseImageRef(registryPrefix, serviceId, environmentInputHash) { + const tag = String(environmentInputHash || "unknown").slice(0, 12); + return `${registryPrefix}/${serviceId}-env:env-${tag}`; +} + function digestFromImageReference(image) { const match = String(image ?? "").match(/@(sha256:[a-f0-9]{64})$/u); return match ? match[1] : null; @@ -936,21 +973,22 @@ async function probeRegistryManifest({ image, digest, timeoutMs = 3000 }) { const request = registryManifestRequest(image, digest); if (!request) return { status: "invalid-reference", image, digest, reason: "registry-reference-unparseable" }; const result = await registryManifestHttpProbe(request, "HEAD", timeoutMs); - if (result.status === "present" || result.status === "missing") return { ...result, image, digest }; + if (result.status === "present" || result.status === "missing") return { ...result, image, digest: result.digest ?? digest ?? null }; const fallback = await registryManifestHttpProbe(request, "GET", timeoutMs); - return { ...fallback, image, digest }; + return { ...fallback, image, digest: fallback.digest ?? digest ?? null }; } function registryManifestRequest(image, digest) { const parsed = parseTaggedImage(image); - if (!parsed || !/^sha256:[a-f0-9]{64}$/u.test(digest ?? "")) return null; + const reference = /^sha256:[a-f0-9]{64}$/u.test(digest ?? "") ? digest : parsed?.tag; + if (!parsed || !reference) return null; const protocol = parsed.host === "127.0.0.1" || parsed.host.startsWith("127.") || parsed.host === "localhost" || parsed.host.includes(":5000") ? "http:" : "https:"; return { protocol, host: parsed.host, - path: `/v2/${parsed.repository}/manifests/${digest}` + path: `/v2/${parsed.repository}/manifests/${reference}` }; } @@ -964,7 +1002,8 @@ function parseTaggedImage(image) { if (!host || tagSeparator <= 0 || rest.includes("@")) return null; return { host, - repository: rest.slice(0, tagSeparator) + repository: rest.slice(0, tagSeparator), + tag: rest.slice(tagSeparator + 1) }; } @@ -990,7 +1029,13 @@ function registryManifestHttpProbe(request, method, timeoutMs) { res.resume(); res.on("end", () => { if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { - resolve({ status: "present", method, statusCode: res.statusCode }); + const digest = String(res.headers["docker-content-digest"] ?? "").trim(); + resolve({ + status: "present", + method, + statusCode: res.statusCode, + digest: /^sha256:[a-f0-9]{64}$/u.test(digest) ? digest : null + }); } else if (res.statusCode === 404) { resolve({ status: "missing", method, statusCode: res.statusCode }); } else {