mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: share env reuse image across ts services

Browse Source
This commit is contained in:
UniDesk Codex
2026-07-06 14:24:19 +08:00
parent fac4b7ead9
commit 4ae933eeef
5 changed files with 3673 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/artifact-publish.mjs
+60 -2
View File
@@ -1299,7 +1299,7 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC
}
async function buildEnvReuseService({ args, repo, commitId, service, buildCreatedAt }) {
const ref = envReuseImageRef(args.registryPrefix, service.serviceId, service.environmentInputHash);
const ref = service.environmentImage ?? envReuseImageRef(args.registryPrefix, service.serviceId, service.environmentInputHash);
const buildSource = `${repo}@env:${service.environmentInputHash}`;
const labels = [
["org.opencontainers.image.source", "https://github.com/pikasTech/HWLAB"],
@@ -1308,6 +1308,8 @@ async function buildEnvReuseService({ args, repo, commitId, service, buildCreate
["org.opencontainers.image.title", `${service.serviceId}-env`],
["hwlab.pikastech.local/runtime-mode", v02EnvReuseRuntimeMode],
["hwlab.pikastech.local/service-id", service.serviceId],
["hwlab.pikastech.local/env-artifact-group", service.envArtifactGroupId ?? ""],
["hwlab.pikastech.local/env-artifact-build-service", service.envArtifactBuildServiceId ?? service.serviceId],
["hwlab.pikastech.local/environment-input-hash", service.environmentInputHash],
["hwlab.pikastech.local/build-created-at", buildCreatedAt]
];
@@ -1323,7 +1325,7 @@ async function buildEnvReuseServiceWithBuildkit({ args, service, ref, buildCreat
const contextDir = await prepareEnvReuseBuildContext(tmpDir, normalizedRecipe, goCacheEnabled);
const metadataPath = path.join(tmpDir, "metadata.json");
const dockerfilePath = path.join(tmpDir, "Dockerfile");
const cacheRef = envReuseCacheRef(args.registryPrefix, service.serviceId);
const cacheRef = service.envArtifactCacheRef ?? envReuseCacheRef(args.registryPrefix, service.serviceId);
const registryOption = buildkitRegistryOption(args.registryPrefix);
const buildkitAddrArgs = args.buildkitAddr ? ["--addr", args.buildkitAddr] : [];
const startedAt = Date.now();
@@ -1432,6 +1434,12 @@ function withEnvReuseArtifactFields({ artifact, service, commitId, buildCreatedA
environmentDigest,
environmentInputHash: service.environmentInputHash ?? null,
codeInputHash: service.codeInputHash ?? null,
envArtifactGroupId: service.envArtifactGroupId ?? null,
envArtifactGroup: service.envArtifactGroup ?? null,
envArtifactBuildServiceId: service.envArtifactBuildServiceId ?? service.serviceId,
envArtifactSourceServiceId: service.envArtifactSourceServiceId ?? service.serviceId,
envArtifactCacheRef: service.envArtifactCacheRef ?? null,
sharedEnvBuildSkipped: service.sharedEnvBuildSkipped === true,
bootRepo: service.bootRepo ?? "git@github.com:pikasTech/HWLAB.git",
bootCommit,
bootSh: service.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`,
@@ -1928,6 +1936,36 @@ async function readExternalServiceArtifacts({ dir, services, commitId }) {
return { artifacts, blockers };
}
function artifactRecordFromSharedEnvArtifact({ service, sourceArtifact, commitId }) {
const environmentImage = sourceArtifact.environmentImage ?? sourceArtifact.image ?? service.environmentImage ?? null;
const environmentDigest = sourceArtifact.environmentDigest ?? sourceArtifact.digest ?? service.environmentDigest ?? "not_published";
const artifact = withEnvReuseArtifactFields({
artifact: {
...sourceArtifact,
...service,
serviceId: service.serviceId,
status: publishReadyStatuses.has(sourceArtifact.status) ? "reused" : sourceArtifact.status,
image: environmentImage,
imageTag: imageTagFromReference(environmentImage),
digest: environmentDigest,
repositoryDigest: repositoryDigestFor(environmentImage, environmentDigest),
buildBackend: "shared-env-artifact",
reusedFrom: sourceArtifact.serviceId ?? service.envArtifactSourceServiceId ?? service.envArtifactGroupId ?? null,
notPublishedReason: null
},
service,
commitId,
buildCreatedAt: sourceArtifact.buildCreatedAt ?? null,
buildSource: sourceArtifact.buildSource ?? null
});
return {
...artifact,
envArtifactSourceServiceId: sourceArtifact.serviceId ?? service.envArtifactSourceServiceId ?? null,
sharedEnvBuildSkipped: true,
ciReason: uniquePreserveOrder([...(Array.isArray(service.reason) ? service.reason : []), "shared-env-artifact-consumer"])
};
}
function serviceRequiresCurrentBuildArtifact(service) {
return service?.buildRequired === true || service?.ciBuildRequired === true;
}
@@ -2425,6 +2463,26 @@ async function main() {
if (externalArtifacts.has(service.serviceId)) {
return externalArtifacts.get(service.serviceId);
}
const sharedEnvSourceServiceId = service.envReuse === true && service.envArtifactSourceServiceId && service.envArtifactSourceServiceId !== service.serviceId
? service.envArtifactSourceServiceId
: null;
if (sharedEnvSourceServiceId && externalArtifacts.has(sharedEnvSourceServiceId)) {
const sharedArtifact = artifactRecordFromSharedEnvArtifact({
service,
sourceArtifact: externalArtifacts.get(sharedEnvSourceServiceId),
commitId
});
emit("artifact_reuse", {
serviceId: service.serviceId,
status: sharedArtifact.status,
image: sharedArtifact.image,
digest: sharedArtifact.digest,
reusedFrom: sharedArtifact.reusedFrom ?? null,
envArtifactGroupId: sharedArtifact.envArtifactGroupId ?? null,
envArtifactSourceServiceId: sharedArtifact.envArtifactSourceServiceId ?? null
});
return sharedArtifact;
}
if (!service.artifactRequired || (buildServiceIds.has(service.serviceId) && serviceRequiresCurrentBuildArtifact(service))) {
return artifactRecord({
args,
scripts/gitops-render.mjs
+2 -1
View File
@@ -2107,12 +2107,13 @@ fs.writeFileSync("/workspace/source/affected-services.json", JSON.stringify({
buildServices: plan.buildServices || entries.filter((entry) => entry.buildRequired).map((entry) => entry.serviceId),
reusedServices: plan.reusedServices || [],
buildSkippedCount: plan.buildSkippedCount || 0,
envArtifactGroups: plan.envArtifactGroups || [],
changedPathSummary: plan.changedPathSummary || null,
ciCdPlan: plan.ciCdPlan || null,
services: plan.services || [],
entries
}, null, 2) + String.fromCharCode(10));
console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0 }));
console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0, envArtifactGroups: plan.envArtifactGroups || [] }));
NODE
`;
}
scripts/src/ci-plan-lib.mjs
+97 -4
View File
@@ -72,7 +72,8 @@ export async function createCiPlan(options = {}) {
const artifactCatalogPath = options.artifactCatalogPath ?? defaultArtifactCatalogPath(laneConfig, lane);
const artifactCatalog = await readStructuredFileIfPresent(repoRoot, artifactCatalogPath, null);
const runtimeReuseConfigPath = options.runtimeReuseConfigPath ?? "gitops/reuse.ymal";
const runtimeReuseByService = runtimeReuseServices(await readStructuredFileIfPresent(repoRoot, runtimeReuseConfigPath, null));
const runtimeReuseConfig = await readStructuredFileIfPresent(repoRoot, runtimeReuseConfigPath, null);
const runtimeReuseByService = runtimeReuseServices(runtimeReuseConfig);
const sourceCommitId = await gitValue(repoRoot, ["rev-parse", targetRef]);
const shortCommitId = await gitValue(repoRoot, ["rev-parse", "--short=7", targetRef]);
const changedPaths = await changedPathsBetween(repoRoot, baseRef, targetRef);
@@ -82,6 +83,7 @@ export async function createCiPlan(options = {}) {
const envReuseRecipe = envReuseRecipeForLane(deployJson, lane);
const componentModels = componentModelsForServices(serviceIdResolution.serviceIds, laneConfig, lane);
const envReuseServices = enabledEnvReuseServices(deployJson, lane, serviceIdResolution.serviceIds);
const envArtifactGroupByService = envArtifactGroupsByService(runtimeReuseConfig, serviceIdResolution.serviceIds);
const globalChange = classifyGlobalChange(normalizedChangedPaths);
const hasGoService = componentModels.some((model) => model.runtimeKind === "go-service");
const dockerfileHash = await hashGitPaths(repoRoot, targetRef, [
@@ -99,6 +101,7 @@ export async function createCiPlan(options = {}) {
const runtimeDepMatches = rawRuntimeDepMatches.filter((item) => semanticRuntimeDepPaths.has(item));
const buildSystemMatches = matchingPaths(normalizedChangedPaths, model.buildSystemPaths);
const envReuse = envReuseServices.has(model.serviceId);
const envArtifactGroup = envReuse ? envArtifactGroupByService.get(model.serviceId) ?? null : null;
const bootSh = envReuse ? bootShForService(deployJson, model.serviceId, lane) : null;
const serviceEnvReuseRecipe = envReuse ? envReuseRecipeForService(envReuseRecipe.publicRecipe, model) : null;
const runtimeConfigChanged = envReuse ? await serviceRuntimeConfigChanged(repoRoot, deployConfigPath, baseRef, targetRef, lane, model.serviceId) : null;
@@ -132,7 +135,7 @@ export async function createCiPlan(options = {}) {
const codeInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, codeInputPaths, {
skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath)
}) : null;
const currentEnvironmentImage = envReuse ? envReuseImageRef(registryPrefix, model.serviceId, environmentInputHash) : null;
const currentEnvironmentImage = envReuse ? envReuseImageRef(registryPrefix, model.serviceId, environmentInputHash, envArtifactGroup) : null;
const catalogEnvironmentImage = envReuse ? environmentImageFromCatalog(model.serviceId, catalogRecord) : null;
const catalogEnvironmentDigest = envReuse ? environmentDigestFromCatalog(catalogRecord) : null;
const catalogEnvironmentSourceCommitId = envReuse ? text(catalogRecord?.sourceCommitId) : "";
@@ -262,6 +265,17 @@ export async function createCiPlan(options = {}) {
runtimeDeps: model.runtimeDeps,
buildSystemPaths: model.buildSystemPaths,
envReuseRecipe: serviceEnvReuseRecipe,
envArtifactGroup: envArtifactGroup ? {
id: envArtifactGroup.id,
buildService: envArtifactGroup.buildService,
imageRepository: envArtifactGroup.imageRepository,
cacheRepository: envArtifactGroup.cacheRepository,
services: envArtifactGroup.services
} : null,
envArtifactGroupId: envArtifactGroup?.id ?? null,
envArtifactBuildServiceId: envArtifactGroup?.buildService ?? null,
envArtifactSourceServiceId: envArtifactGroup?.buildService ?? model.serviceId,
envArtifactCacheRef: envArtifactGroup ? envReuseCacheRef(registryPrefix, envArtifactGroup) : null,
componentCommitId,
componentInputHash,
catalogSourceCommitId: catalogSourceCommitId || null,
@@ -305,6 +319,8 @@ export async function createCiPlan(options = {}) {
});
}
const envArtifactGroupPlans = applyEnvArtifactGroupBuildPlan(services);
const affectedServices = services.filter((service) => service.affected).map((service) => service.serviceId);
const buildServices = services.filter((service) => service.buildRequired).map((service) => service.serviceId);
const reusedServices = services.filter((service) => !service.buildRequired).map((service) => service.serviceId);
@@ -326,6 +342,7 @@ export async function createCiPlan(options = {}) {
serviceDeclarationSource: `deploy.lanes.${lane}.serviceDeclarations`,
envRecipeSource: `deploy.lanes.${lane}.envRecipe`,
envReuseServiceIds: [...envReuseServices],
envArtifactGroups: envArtifactGroupPlans,
v02EnvReuseServiceIds: lane === "v02" ? [...envReuseServices] : [],
defaultComponentLazyBuild: true,
envReuseCodeOnlyFastLane: true,
@@ -345,11 +362,13 @@ export async function createCiPlan(options = {}) {
buildServices,
reusedServices,
buildSkippedCount: services.length - buildServices.length,
envArtifactGroups: envArtifactGroupPlans,
services,
ciCdPlan: {
willBuild: buildServices,
willRollout: affectedServices,
willReuse: reusedServices,
envArtifactGroups: envArtifactGroupPlans,
willRunGitopsPromote: globalChange.gitopsOnly || affectedServices.length > 0,
noImageBuildReason: buildServices.length === 0
? (affectedServices.length > 0 ? "env-reuse-code-only-rollout" : globalChange.summary)
@@ -683,6 +702,75 @@ function runtimeReuseServices(config) {
}));
}
function envArtifactGroupsByService(config, allowedServiceIds) {
const allowed = new Set(allowedServiceIds);
const raw = config?.spec?.envArtifactGroups ?? config?.envArtifactGroups;
const entries = Array.isArray(raw)
? raw.map((value, index) => [String(value?.id ?? index), value])
: Object.entries(raw ?? {});
const byService = new Map();
for (const [id, value] of entries) {
if (value?.enabled === false) continue;
const groupId = String(value?.id ?? id ?? "").trim();
const services = uniqueSorted(normalizeStringList(value?.services, [])).filter((serviceId) => allowed.has(serviceId));
if (!groupId || services.length === 0) continue;
const configuredBuildService = String(value?.buildService ?? value?.sourceService ?? services[0] ?? "").trim();
const buildService = services.includes(configuredBuildService) ? configuredBuildService : services[0];
const group = {
id: groupId,
buildService,
services,
imageRepository: normalizeRepositoryPath(value?.imageRepository) ?? groupId,
cacheRepository: normalizeRepositoryPath(value?.cacheRepository) ?? `cache/${groupId}`
};
for (const serviceId of services) byService.set(serviceId, group);
}
return byService;
}
function normalizeRepositoryPath(value) {
const text = String(value ?? "").trim();
return text && /^[a-z0-9][a-z0-9._/-]*$/u.test(text) && !text.includes("..") ? text : null;
}
function applyEnvArtifactGroupBuildPlan(services) {
const groups = new Map();
for (const service of services) {
const group = service.envArtifactGroup;
if (!service.envReuse || !group?.id) continue;
if (!groups.has(group.id)) groups.set(group.id, { ...group, services: [] });
groups.get(group.id).services.push(service);
}
const plans = [];
for (const group of groups.values()) {
const members = group.services;
const buildNeeded = members.some((service) => service.buildRequired === true);
const source = members.find((service) => service.serviceId === group.buildService) ?? members.find((service) => service.buildRequired === true) ?? members[0];
for (const service of members) {
service.envArtifactSourceServiceId = source.serviceId;
service.envArtifactBuildServiceId = source.serviceId;
service.envArtifactGroupBuildRequired = buildNeeded && service.serviceId === source.serviceId;
if (buildNeeded && service.serviceId !== source.serviceId && service.buildRequired === true) {
service.buildRequired = false;
service.skipImageBuild = true;
service.sharedEnvBuildSkipped = true;
service.reason = uniqueSorted([...(Array.isArray(service.reason) ? service.reason : []), "shared-env-artifact-consumer"]);
}
}
plans.push({
id: group.id,
buildService: source.serviceId,
imageRepository: group.imageRepository,
cacheRepository: group.cacheRepository,
buildRequired: buildNeeded,
services: members.map((service) => service.serviceId),
buildServices: buildNeeded ? [source.serviceId] : [],
consumerServices: buildNeeded ? members.filter((service) => service.serviceId !== source.serviceId).map((service) => service.serviceId) : members.map((service) => service.serviceId)
});
}
return plans.sort((a, b) => a.id.localeCompare(b.id));
}
async function runtimeReuseDecisionForService({ repoRoot, baseRef, targetRef, serviceId, config }) {
const runtimeReuse = config?.runtimeReuse ?? null;
const envReuse = config?.envReuse ?? null;
@@ -1032,9 +1120,14 @@ function environmentDigestFromCatalogForHash(catalogRecord, environmentInputHash
return catalogRecord?.environmentInputHash === environmentInputHash ? environmentDigestFromCatalog(catalogRecord) : null;
}
function envReuseImageRef(registryPrefix, serviceId, environmentInputHash) {
function envReuseImageRef(registryPrefix, serviceId, environmentInputHash, group = null) {
const tag = String(environmentInputHash || "unknown").slice(0, 12);
return `${registryPrefix}/${serviceId}-env:env-${tag}`;
const repository = group?.imageRepository ?? `${serviceId}-env`;
return `${registryPrefix}/${repository}:env-${tag}`;
}
function envReuseCacheRef(registryPrefix, group) {
return `${registryPrefix}/${group.cacheRepository}`;
}
function digestFromImageReference(image) {