fix: reuse current env hash registry tags

This commit is contained in:
lyon
2026-06-25 09:49:28 +08:00
parent fb201751d9
commit 358f73fe30
3 changed files with 158 additions and 18 deletions
+4 -4
View File
@@ -1786,8 +1786,8 @@ function serviceImageCatalogProvenance(catalogService) {
}
function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commitId }) {
const environmentImage = catalogService?.environmentImage ?? service.environmentImage ?? null;
const environmentDigest = catalogService?.environmentDigest ?? service.environmentDigest ?? "not_published";
const environmentImage = service.environmentImage ?? catalogService?.environmentImage ?? null;
const environmentDigest = service.environmentDigest ?? catalogService?.environmentDigest ?? "not_published";
const imageTag = imageTagFromReference(environmentImage);
const bootCommit = service.codeChanged === true || service.envChanged === true
? service.bootCommit ?? commitId
@@ -1813,7 +1813,7 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit
buildCreatedAt: catalogService?.buildCreatedAt ?? null,
buildSource: catalogService?.buildSource ?? null,
buildBackend: "reused-env-catalog",
repositoryDigest: catalogService?.repositoryDigest ?? repositoryDigestFor(environmentImage, environmentDigest),
repositoryDigest: service.repositoryDigest ?? repositoryDigestFor(environmentImage, environmentDigest) ?? catalogService?.repositoryDigest ?? null,
bootEnvEvidence: bootEnvEvidence({ service: { ...service, bootCommit }, commitId: bootCommit, environmentImage, environmentDigest })
};
if (!environmentImage || !imageTag || !shaDigestPattern.test(environmentDigest)) {
@@ -1827,7 +1827,7 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit
return {
...base,
status: "reused",
reusedFrom: service.reuse?.reusedFrom ?? catalogService?.environmentInputHash ?? catalogService?.componentInputHash ?? imageTag,
reusedFrom: service.reuse?.reusedFrom ?? service.environmentInputHash ?? catalogService?.environmentInputHash ?? catalogService?.componentInputHash ?? imageTag,
notPublishedReason: null
};
}
+95
View File
@@ -184,6 +184,50 @@ test("v03 planner rebuilds env image when catalog env input hash is stale", asyn
assert.deepEqual(plan.buildServices, ["hwlab-cloud-api"]);
});
test("v03 planner reuses current env hash tag when registry already has it", async () => {
const repo = await createFixtureRepo();
const initialSha = (await git(repo, ["rev-parse", "HEAD"])).stdout.trim();
const initialPlan = await createCiPlan({
repoRoot: repo,
lane: "v03",
baseRef: "HEAD",
targetRef: initialSha,
artifactCatalogPath: "deploy/artifact-catalog.v03.json",
services: ["hwlab-cloud-api"]
});
const catalog = createCatalogFixture("ready", catalogOptionsFromPlan(initialPlan, { sourceCommitId: initialSha }));
catalog.services.find((service) => service.serviceId === "hwlab-cloud-api").environmentInputHash = "stale-environment-input";
await writeFile(path.join(repo, "deploy/artifact-catalog.v03.json"), JSON.stringify(catalog, null, 2));
await git(repo, ["add", "deploy/artifact-catalog.v03.json"]);
await git(repo, ["commit", "-m", "seed stale v03 env catalog"]);
const digest = `sha256:${"2".repeat(64)}`;
let probeRequest = null;
const plan = await createCiPlan({
repoRoot: repo,
lane: "v03",
baseRef: "HEAD~1",
targetRef: "HEAD",
artifactCatalogPath: "deploy/artifact-catalog.v03.json",
services: ["hwlab-cloud-api"],
verifyReuseRegistry: true,
reuseRegistryProbe: async (request) => {
probeRequest = request;
return { status: "present", method: "HEAD", statusCode: 200, digest };
}
});
const cloudApi = plan.services.find((service) => service.serviceId === "hwlab-cloud-api");
assert.equal(cloudApi.environmentInputChanged, true);
assert.equal(cloudApi.envChanged, false);
assert.equal(cloudApi.buildRequired, false);
assert.equal(cloudApi.environmentDigest, digest);
assert.equal(cloudApi.reuse.status, "ready");
assert.equal(cloudApi.reuse.reusedFrom, cloudApi.environmentInputHash);
assert.equal(probeRequest.digest, null);
assert.equal(probeRequest.image, `127.0.0.1:5000/hwlab/hwlab-cloud-api-env:env-${cloudApi.environmentInputHash.slice(0, 12)}`);
assert.deepEqual(plan.buildServices, []);
});
test("v03 planner rebuilds env image when target registry lacks reused digest", async () => {
const repo = await createFixtureRepo();
const initialSha = (await git(repo, ["rev-parse", "HEAD"])).stdout.trim();
@@ -705,6 +749,57 @@ test("v02 planner reads env reuse declarations and recipe from deploy config", a
assert.equal(recipe.launcherInputPaths.includes("custom/env-tool.mjs"), true);
});
test("planner sanitizes k8s native download no_proxy to internal targets", async () => {
const repo = await createFixtureRepo();
const previousEnv = {
HWLAB_TEKTON_PIPELINERUN: process.env.HWLAB_TEKTON_PIPELINERUN,
HTTP_PROXY: process.env.HTTP_PROXY,
HTTPS_PROXY: process.env.HTTPS_PROXY,
NO_PROXY: process.env.NO_PROXY,
no_proxy: process.env.no_proxy
};
process.env.HWLAB_TEKTON_PIPELINERUN = "test-pipelinerun";
process.env.HTTP_PROXY = "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808";
process.env.HTTPS_PROXY = "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808";
process.env.NO_PROXY = [
"127.0.0.1",
"localhost",
"10.43.0.0/16",
"git-mirror-http.devops-infra.svc.cluster.local",
"deb.debian.org",
".debian.org",
"goproxy.cn",
".goproxy.cn",
"hyueapi.com",
".hyueapi.com"
].join(",");
delete process.env.no_proxy;
try {
const plan = await createCiPlan({
repoRoot: repo,
lane: "v03",
baseRef: "HEAD",
targetRef: "HEAD",
artifactCatalogPath: "deploy/artifact-catalog.v03.json",
services: ["hwlab-cloud-api"]
});
const noProxy = plan.services[0].envReuseRecipe.downloadStack.noProxy;
assert.deepEqual(noProxy, [
"127.0.0.1",
"localhost",
"10.43.0.0/16",
"git-mirror-http.devops-infra.svc.cluster.local",
"hyueapi.com",
".hyueapi.com"
]);
} finally {
for (const [key, value] of Object.entries(previousEnv)) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
});
test("v03 planner reads env reuse declarations and catalog from v03 lane config", async () => {
const repo = await createFixtureRepo();
+59 -14
View File
@@ -130,11 +130,10 @@ export async function createCiPlan(options = {}) {
const codeInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, codeInputPaths, {
skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath)
}) : null;
const environmentImage = envReuse ? environmentImageFromCatalog(model.serviceId, catalogRecord) : null;
const environmentDigest = envReuse ? environmentDigestFromCatalog(catalogRecord) : null;
const environmentReady = envReuse && /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? "") && Boolean(environmentImage);
const environmentImage = envReuse ? envReuseImageRef(registryPrefix, model.serviceId, environmentInputHash) : null;
let environmentDigest = envReuse ? environmentDigestFromCatalogForHash(catalogRecord, environmentInputHash) : null;
const environmentInputChanged = envReuse ? catalogRecord?.environmentInputHash !== environmentInputHash : null;
const reuseRegistry = envReuse && reuseRegistryProbe && environmentReady
const reuseRegistry = envReuse && reuseRegistryProbe && environmentImage
? await reuseRegistryProbe({
serviceId: model.serviceId,
image: environmentImage,
@@ -143,8 +142,13 @@ export async function createCiPlan(options = {}) {
timeoutMs: Number.isFinite(registryProbeTimeoutMs) ? registryProbeTimeoutMs : 3000
})
: null;
if (reuseRegistry?.status === "present" && /^sha256:[a-f0-9]{64}$/u.test(reuseRegistry.digest ?? "")) {
environmentDigest = reuseRegistry.digest;
}
const reuseRegistryUnavailable = Boolean(reuseRegistry && reuseRegistry.status !== "present");
const envChanged = envReuse ? Boolean(relevantEnvMatches.length > 0 || !environmentReady || environmentInputChanged || reuseRegistryUnavailable) : null;
const digestReady = /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? "");
const environmentReady = envReuse && Boolean(environmentImage) && (reuseRegistryProbe ? reuseRegistry?.status === "present" && digestReady : !environmentInputChanged && digestReady);
const envChanged = envReuse ? Boolean(relevantEnvMatches.length > 0 || !environmentReady || reuseRegistryUnavailable) : null;
const codeChanged = envReuse ? relevantCodeMatches.length > 0 || catalogRecord?.codeInputHash !== codeInputHash : null;
const componentInputPaths = uniqueSorted([
...model.componentPaths,
@@ -188,7 +192,7 @@ export async function createCiPlan(options = {}) {
} : catalogRecord;
const componentInputChanged = !envReuse && catalogComponentProvenance?.safeToReuse !== true;
const catalogReuse = envReuse
? envReuseCandidate(catalogRecord, envChanged)
? envReuseCandidate({ catalogRecord, envChanged, environmentImage, environmentDigest, environmentInputHash })
: reuseCandidate(catalogRecordForReuse, imageRelevantChangedPaths.length > 0, componentInputChanged);
const affected = envReuse
? Boolean(envChanged || codeChanged || runtimeConfigChanged)
@@ -494,7 +498,7 @@ function effectiveDownloadStackConfig(value, env = process.env) {
if (!configured || !usesK8sNativeDownloadProxy(env)) return configured;
const httpProxy = normalizeDeclarationString(env.HTTP_PROXY) || normalizeDeclarationString(env.http_proxy) || configured.httpProxy;
const httpsProxy = normalizeDeclarationString(env.HTTPS_PROXY) || normalizeDeclarationString(env.https_proxy) || httpProxy || configured.httpsProxy;
const noProxy = noProxyFromEnv(env) || configured.noProxy;
const noProxy = sanitizeK8sDownloadNoProxy(noProxyFromEnv(env) || configured.noProxy);
return {
...configured,
httpProxy,
@@ -513,6 +517,22 @@ function noProxyFromEnv(env = process.env) {
return text.split(",").map((item) => item.trim()).filter(Boolean);
}
function sanitizeK8sDownloadNoProxy(values) {
return uniquePreserveOrder(normalizeStringList(values, []).filter(isK8sDownloadNoProxyEntry));
}
function isK8sDownloadNoProxyEntry(value) {
const item = String(value ?? "").trim().toLowerCase();
if (!item || item === "*") return false;
if (["localhost", "::1", "[::1]", "hyueapi.com", ".hyueapi.com"].includes(item)) return true;
if (item.endsWith(".hyueapi.com")) return true;
if (item === ".svc" || item === ".svc.cluster.local" || item === ".cluster.local") return true;
if (item.endsWith(".svc") || item.endsWith(".svc.cluster.local") || item.endsWith(".cluster.local")) return true;
if (/^(127|10)\./u.test(item) || /^192\.168\./u.test(item) || /^172\.(1[6-9]|2\d|3[01])\./u.test(item)) return true;
if (/^(127|10)\.[0-9./:-]+$/u.test(item) || /^192\.168\.[0-9./:-]+$/u.test(item) || /^172\.(1[6-9]|2\d|3[01])\.[0-9./:-]+$/u.test(item)) return true;
return false;
}
function normalizeOptionalHttpUrl(value, label) {
const text = normalizeDeclarationString(value);
if (!text) return null;
@@ -896,8 +916,16 @@ function text(value) {
return String(value ?? "").trim();
}
function envReuseCandidate(catalogRecord, envChanged) {
function envReuseCandidate({ catalogRecord, envChanged, environmentImage, environmentDigest, environmentInputHash }) {
if (envChanged) return { status: "not-reused", reason: "environment-affected" };
if (environmentImage && /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? "")) {
return {
status: "ready",
image: environmentImage,
digest: environmentDigest,
reusedFrom: environmentInputHash ?? catalogRecord?.environmentInputHash ?? catalogRecord?.commitId ?? catalogRecord?.imageTag ?? null
};
}
if (!catalogRecord) return { status: "candidate-no-catalog", reason: "no-previous-artifact-record" };
const image = environmentImageFromCatalog(catalogRecord.serviceId, catalogRecord);
const digest = environmentDigestFromCatalog(catalogRecord);
@@ -923,6 +951,15 @@ function environmentDigestFromCatalog(catalogRecord) {
return catalogRecord?.environmentDigest ?? (catalogRecord?.environmentImage ? catalogRecord?.digest : null) ?? null;
}
function environmentDigestFromCatalogForHash(catalogRecord, environmentInputHash) {
return catalogRecord?.environmentInputHash === environmentInputHash ? environmentDigestFromCatalog(catalogRecord) : null;
}
function envReuseImageRef(registryPrefix, serviceId, environmentInputHash) {
const tag = String(environmentInputHash || "unknown").slice(0, 12);
return `${registryPrefix}/${serviceId}-env:env-${tag}`;
}
function digestFromImageReference(image) {
const match = String(image ?? "").match(/@(sha256:[a-f0-9]{64})$/u);
return match ? match[1] : null;
@@ -936,21 +973,22 @@ async function probeRegistryManifest({ image, digest, timeoutMs = 3000 }) {
const request = registryManifestRequest(image, digest);
if (!request) return { status: "invalid-reference", image, digest, reason: "registry-reference-unparseable" };
const result = await registryManifestHttpProbe(request, "HEAD", timeoutMs);
if (result.status === "present" || result.status === "missing") return { ...result, image, digest };
if (result.status === "present" || result.status === "missing") return { ...result, image, digest: result.digest ?? digest ?? null };
const fallback = await registryManifestHttpProbe(request, "GET", timeoutMs);
return { ...fallback, image, digest };
return { ...fallback, image, digest: fallback.digest ?? digest ?? null };
}
function registryManifestRequest(image, digest) {
const parsed = parseTaggedImage(image);
if (!parsed || !/^sha256:[a-f0-9]{64}$/u.test(digest ?? "")) return null;
const reference = /^sha256:[a-f0-9]{64}$/u.test(digest ?? "") ? digest : parsed?.tag;
if (!parsed || !reference) return null;
const protocol = parsed.host === "127.0.0.1" || parsed.host.startsWith("127.") || parsed.host === "localhost" || parsed.host.includes(":5000")
? "http:"
: "https:";
return {
protocol,
host: parsed.host,
path: `/v2/${parsed.repository}/manifests/${digest}`
path: `/v2/${parsed.repository}/manifests/${reference}`
};
}
@@ -964,7 +1002,8 @@ function parseTaggedImage(image) {
if (!host || tagSeparator <= 0 || rest.includes("@")) return null;
return {
host,
repository: rest.slice(0, tagSeparator)
repository: rest.slice(0, tagSeparator),
tag: rest.slice(tagSeparator + 1)
};
}
@@ -990,7 +1029,13 @@ function registryManifestHttpProbe(request, method, timeoutMs) {
res.resume();
res.on("end", () => {
if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
resolve({ status: "present", method, statusCode: res.statusCode });
const digest = String(res.headers["docker-content-digest"] ?? "").trim();
resolve({
status: "present",
method,
statusCode: res.statusCode,
digest: /^sha256:[a-f0-9]{64}$/u.test(digest) ? digest : null
});
} else if (res.statusCode === 404) {
resolve({ status: "missing", method, statusCode: res.statusCode });
} else {