fix: share env reuse image across ts services

2026-07-06 14:24:19 +08:00
parent fac4b7ead9
commit 4ae933eeef
5 changed files with 3673 additions and 8 deletions
@@ -4,6 +4,20 @@ metadata:
  name: hwlab-v03-reuse
  ownerRepository: pikasTech/HWLAB
 spec:
+  envArtifactGroups:
+    hwlab-ts-runtime-env:
+      enabled: true
+      mode: env-reuse-git-mirror-checkout
+      buildService: hwlab-cloud-api
+      imageRepository: hwlab-ts-runtime-env
+      cacheRepository: cache/hwlab-ts-runtime-env
+      services:
+        - hwlab-cloud-api
+        - hwlab-project-management
+        - hwlab-cloud-web
+        - hwlab-gateway
+        - hwlab-edge-proxy
+        - hwlab-agent-skills
  services:
    hwlab-cloud-api:
      runtimeReuse:
@@ -1299,7 +1299,7 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC
 }

 async function buildEnvReuseService({ args, repo, commitId, service, buildCreatedAt }) {
-  const ref = envReuseImageRef(args.registryPrefix, service.serviceId, service.environmentInputHash);
+  const ref = service.environmentImage ?? envReuseImageRef(args.registryPrefix, service.serviceId, service.environmentInputHash);
  const buildSource = `${repo}@env:${service.environmentInputHash}`;
  const labels = [
    ["org.opencontainers.image.source", "https://github.com/pikasTech/HWLAB"],
@@ -1308,6 +1308,8 @@ async function buildEnvReuseService({ args, repo, commitId, service, buildCreate
    ["org.opencontainers.image.title", `${service.serviceId}-env`],
    ["hwlab.pikastech.local/runtime-mode", v02EnvReuseRuntimeMode],
    ["hwlab.pikastech.local/service-id", service.serviceId],
+    ["hwlab.pikastech.local/env-artifact-group", service.envArtifactGroupId ?? ""],
+    ["hwlab.pikastech.local/env-artifact-build-service", service.envArtifactBuildServiceId ?? service.serviceId],
    ["hwlab.pikastech.local/environment-input-hash", service.environmentInputHash],
    ["hwlab.pikastech.local/build-created-at", buildCreatedAt]
  ];
@@ -1323,7 +1325,7 @@ async function buildEnvReuseServiceWithBuildkit({ args, service, ref, buildCreat
  const contextDir = await prepareEnvReuseBuildContext(tmpDir, normalizedRecipe, goCacheEnabled);
  const metadataPath = path.join(tmpDir, "metadata.json");
  const dockerfilePath = path.join(tmpDir, "Dockerfile");
-  const cacheRef = envReuseCacheRef(args.registryPrefix, service.serviceId);
+  const cacheRef = service.envArtifactCacheRef ?? envReuseCacheRef(args.registryPrefix, service.serviceId);
  const registryOption = buildkitRegistryOption(args.registryPrefix);
  const buildkitAddrArgs = args.buildkitAddr ? ["--addr", args.buildkitAddr] : [];
  const startedAt = Date.now();
@@ -1432,6 +1434,12 @@ function withEnvReuseArtifactFields({ artifact, service, commitId, buildCreatedA
    environmentDigest,
    environmentInputHash: service.environmentInputHash ?? null,
    codeInputHash: service.codeInputHash ?? null,
+    envArtifactGroupId: service.envArtifactGroupId ?? null,
+    envArtifactGroup: service.envArtifactGroup ?? null,
+    envArtifactBuildServiceId: service.envArtifactBuildServiceId ?? service.serviceId,
+    envArtifactSourceServiceId: service.envArtifactSourceServiceId ?? service.serviceId,
+    envArtifactCacheRef: service.envArtifactCacheRef ?? null,
+    sharedEnvBuildSkipped: service.sharedEnvBuildSkipped === true,
    bootRepo: service.bootRepo ?? "git@github.com:pikasTech/HWLAB.git",
    bootCommit,
    bootSh: service.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`,
@@ -1928,6 +1936,36 @@ async function readExternalServiceArtifacts({ dir, services, commitId }) {
  return { artifacts, blockers };
 }

+function artifactRecordFromSharedEnvArtifact({ service, sourceArtifact, commitId }) {
+  const environmentImage = sourceArtifact.environmentImage ?? sourceArtifact.image ?? service.environmentImage ?? null;
+  const environmentDigest = sourceArtifact.environmentDigest ?? sourceArtifact.digest ?? service.environmentDigest ?? "not_published";
+  const artifact = withEnvReuseArtifactFields({
+    artifact: {
+      ...sourceArtifact,
+      ...service,
+      serviceId: service.serviceId,
+      status: publishReadyStatuses.has(sourceArtifact.status) ? "reused" : sourceArtifact.status,
+      image: environmentImage,
+      imageTag: imageTagFromReference(environmentImage),
+      digest: environmentDigest,
+      repositoryDigest: repositoryDigestFor(environmentImage, environmentDigest),
+      buildBackend: "shared-env-artifact",
+      reusedFrom: sourceArtifact.serviceId ?? service.envArtifactSourceServiceId ?? service.envArtifactGroupId ?? null,
+      notPublishedReason: null
+    },
+    service,
+    commitId,
+    buildCreatedAt: sourceArtifact.buildCreatedAt ?? null,
+    buildSource: sourceArtifact.buildSource ?? null
+  });
+  return {
+    ...artifact,
+    envArtifactSourceServiceId: sourceArtifact.serviceId ?? service.envArtifactSourceServiceId ?? null,
+    sharedEnvBuildSkipped: true,
+    ciReason: uniquePreserveOrder([...(Array.isArray(service.reason) ? service.reason : []), "shared-env-artifact-consumer"])
+  };
+}
+
 function serviceRequiresCurrentBuildArtifact(service) {
  return service?.buildRequired === true || service?.ciBuildRequired === true;
 }
@@ -2425,6 +2463,26 @@ async function main() {
      if (externalArtifacts.has(service.serviceId)) {
        return externalArtifacts.get(service.serviceId);
      }
+      const sharedEnvSourceServiceId = service.envReuse === true && service.envArtifactSourceServiceId && service.envArtifactSourceServiceId !== service.serviceId
+        ? service.envArtifactSourceServiceId
+        : null;
+      if (sharedEnvSourceServiceId && externalArtifacts.has(sharedEnvSourceServiceId)) {
+        const sharedArtifact = artifactRecordFromSharedEnvArtifact({
+          service,
+          sourceArtifact: externalArtifacts.get(sharedEnvSourceServiceId),
+          commitId
+        });
+        emit("artifact_reuse", {
+          serviceId: service.serviceId,
+          status: sharedArtifact.status,
+          image: sharedArtifact.image,
+          digest: sharedArtifact.digest,
+          reusedFrom: sharedArtifact.reusedFrom ?? null,
+          envArtifactGroupId: sharedArtifact.envArtifactGroupId ?? null,
+          envArtifactSourceServiceId: sharedArtifact.envArtifactSourceServiceId ?? null
+        });
+        return sharedArtifact;
+      }
      if (!service.artifactRequired || (buildServiceIds.has(service.serviceId) && serviceRequiresCurrentBuildArtifact(service))) {
        return artifactRecord({
          args,
@@ -2107,12 +2107,13 @@ fs.writeFileSync("/workspace/source/affected-services.json", JSON.stringify({
  buildServices: plan.buildServices || entries.filter((entry) => entry.buildRequired).map((entry) => entry.serviceId),
  reusedServices: plan.reusedServices || [],
  buildSkippedCount: plan.buildSkippedCount || 0,
+  envArtifactGroups: plan.envArtifactGroups || [],
  changedPathSummary: plan.changedPathSummary || null,
  ciCdPlan: plan.ciCdPlan || null,
  services: plan.services || [],
  entries
 }, null, 2) + String.fromCharCode(10));
-console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0 }));
+console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0, envArtifactGroups: plan.envArtifactGroups || [] }));
 NODE
 `;
 }
@@ -72,7 +72,8 @@ export async function createCiPlan(options = {}) {
  const artifactCatalogPath = options.artifactCatalogPath ?? defaultArtifactCatalogPath(laneConfig, lane);
  const artifactCatalog = await readStructuredFileIfPresent(repoRoot, artifactCatalogPath, null);
  const runtimeReuseConfigPath = options.runtimeReuseConfigPath ?? "gitops/reuse.ymal";
-  const runtimeReuseByService = runtimeReuseServices(await readStructuredFileIfPresent(repoRoot, runtimeReuseConfigPath, null));
+  const runtimeReuseConfig = await readStructuredFileIfPresent(repoRoot, runtimeReuseConfigPath, null);
+  const runtimeReuseByService = runtimeReuseServices(runtimeReuseConfig);
  const sourceCommitId = await gitValue(repoRoot, ["rev-parse", targetRef]);
  const shortCommitId = await gitValue(repoRoot, ["rev-parse", "--short=7", targetRef]);
  const changedPaths = await changedPathsBetween(repoRoot, baseRef, targetRef);
@@ -82,6 +83,7 @@ export async function createCiPlan(options = {}) {
  const envReuseRecipe = envReuseRecipeForLane(deployJson, lane);
  const componentModels = componentModelsForServices(serviceIdResolution.serviceIds, laneConfig, lane);
  const envReuseServices = enabledEnvReuseServices(deployJson, lane, serviceIdResolution.serviceIds);
+  const envArtifactGroupByService = envArtifactGroupsByService(runtimeReuseConfig, serviceIdResolution.serviceIds);
  const globalChange = classifyGlobalChange(normalizedChangedPaths);
  const hasGoService = componentModels.some((model) => model.runtimeKind === "go-service");
  const dockerfileHash = await hashGitPaths(repoRoot, targetRef, [
@@ -99,6 +101,7 @@ export async function createCiPlan(options = {}) {
    const runtimeDepMatches = rawRuntimeDepMatches.filter((item) => semanticRuntimeDepPaths.has(item));
    const buildSystemMatches = matchingPaths(normalizedChangedPaths, model.buildSystemPaths);
    const envReuse = envReuseServices.has(model.serviceId);
+    const envArtifactGroup = envReuse ? envArtifactGroupByService.get(model.serviceId) ?? null : null;
    const bootSh = envReuse ? bootShForService(deployJson, model.serviceId, lane) : null;
    const serviceEnvReuseRecipe = envReuse ? envReuseRecipeForService(envReuseRecipe.publicRecipe, model) : null;
    const runtimeConfigChanged = envReuse ? await serviceRuntimeConfigChanged(repoRoot, deployConfigPath, baseRef, targetRef, lane, model.serviceId) : null;
@@ -132,7 +135,7 @@ export async function createCiPlan(options = {}) {
    const codeInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, codeInputPaths, {
      skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath)
    }) : null;
-    const currentEnvironmentImage = envReuse ? envReuseImageRef(registryPrefix, model.serviceId, environmentInputHash) : null;
+    const currentEnvironmentImage = envReuse ? envReuseImageRef(registryPrefix, model.serviceId, environmentInputHash, envArtifactGroup) : null;
    const catalogEnvironmentImage = envReuse ? environmentImageFromCatalog(model.serviceId, catalogRecord) : null;
    const catalogEnvironmentDigest = envReuse ? environmentDigestFromCatalog(catalogRecord) : null;
    const catalogEnvironmentSourceCommitId = envReuse ? text(catalogRecord?.sourceCommitId) : "";
@@ -262,6 +265,17 @@ export async function createCiPlan(options = {}) {
      runtimeDeps: model.runtimeDeps,
      buildSystemPaths: model.buildSystemPaths,
      envReuseRecipe: serviceEnvReuseRecipe,
+      envArtifactGroup: envArtifactGroup ? {
+        id: envArtifactGroup.id,
+        buildService: envArtifactGroup.buildService,
+        imageRepository: envArtifactGroup.imageRepository,
+        cacheRepository: envArtifactGroup.cacheRepository,
+        services: envArtifactGroup.services
+      } : null,
+      envArtifactGroupId: envArtifactGroup?.id ?? null,
+      envArtifactBuildServiceId: envArtifactGroup?.buildService ?? null,
+      envArtifactSourceServiceId: envArtifactGroup?.buildService ?? model.serviceId,
+      envArtifactCacheRef: envArtifactGroup ? envReuseCacheRef(registryPrefix, envArtifactGroup) : null,
      componentCommitId,
      componentInputHash,
      catalogSourceCommitId: catalogSourceCommitId || null,
@@ -305,6 +319,8 @@ export async function createCiPlan(options = {}) {
    });
  }

+  const envArtifactGroupPlans = applyEnvArtifactGroupBuildPlan(services);
+
  const affectedServices = services.filter((service) => service.affected).map((service) => service.serviceId);
  const buildServices = services.filter((service) => service.buildRequired).map((service) => service.serviceId);
  const reusedServices = services.filter((service) => !service.buildRequired).map((service) => service.serviceId);
@@ -326,6 +342,7 @@ export async function createCiPlan(options = {}) {
      serviceDeclarationSource: `deploy.lanes.${lane}.serviceDeclarations`,
      envRecipeSource: `deploy.lanes.${lane}.envRecipe`,
      envReuseServiceIds: [...envReuseServices],
+      envArtifactGroups: envArtifactGroupPlans,
      v02EnvReuseServiceIds: lane === "v02" ? [...envReuseServices] : [],
      defaultComponentLazyBuild: true,
      envReuseCodeOnlyFastLane: true,
@@ -345,11 +362,13 @@ export async function createCiPlan(options = {}) {
    buildServices,
    reusedServices,
    buildSkippedCount: services.length - buildServices.length,
+    envArtifactGroups: envArtifactGroupPlans,
    services,
    ciCdPlan: {
      willBuild: buildServices,
      willRollout: affectedServices,
      willReuse: reusedServices,
+      envArtifactGroups: envArtifactGroupPlans,
      willRunGitopsPromote: globalChange.gitopsOnly || affectedServices.length > 0,
      noImageBuildReason: buildServices.length === 0
        ? (affectedServices.length > 0 ? "env-reuse-code-only-rollout" : globalChange.summary)
@@ -683,6 +702,75 @@ function runtimeReuseServices(config) {
  }));
 }

+function envArtifactGroupsByService(config, allowedServiceIds) {
+  const allowed = new Set(allowedServiceIds);
+  const raw = config?.spec?.envArtifactGroups ?? config?.envArtifactGroups;
+  const entries = Array.isArray(raw)
+    ? raw.map((value, index) => [String(value?.id ?? index), value])
+    : Object.entries(raw ?? {});
+  const byService = new Map();
+  for (const [id, value] of entries) {
+    if (value?.enabled === false) continue;
+    const groupId = String(value?.id ?? id ?? "").trim();
+    const services = uniqueSorted(normalizeStringList(value?.services, [])).filter((serviceId) => allowed.has(serviceId));
+    if (!groupId || services.length === 0) continue;
+    const configuredBuildService = String(value?.buildService ?? value?.sourceService ?? services[0] ?? "").trim();
+    const buildService = services.includes(configuredBuildService) ? configuredBuildService : services[0];
+    const group = {
+      id: groupId,
+      buildService,
+      services,
+      imageRepository: normalizeRepositoryPath(value?.imageRepository) ?? groupId,
+      cacheRepository: normalizeRepositoryPath(value?.cacheRepository) ?? `cache/${groupId}`
+    };
+    for (const serviceId of services) byService.set(serviceId, group);
+  }
+  return byService;
+}
+
+function normalizeRepositoryPath(value) {
+  const text = String(value ?? "").trim();
+  return text && /^[a-z0-9][a-z0-9._/-]*$/u.test(text) && !text.includes("..") ? text : null;
+}
+
+function applyEnvArtifactGroupBuildPlan(services) {
+  const groups = new Map();
+  for (const service of services) {
+    const group = service.envArtifactGroup;
+    if (!service.envReuse || !group?.id) continue;
+    if (!groups.has(group.id)) groups.set(group.id, { ...group, services: [] });
+    groups.get(group.id).services.push(service);
+  }
+  const plans = [];
+  for (const group of groups.values()) {
+    const members = group.services;
+    const buildNeeded = members.some((service) => service.buildRequired === true);
+    const source = members.find((service) => service.serviceId === group.buildService) ?? members.find((service) => service.buildRequired === true) ?? members[0];
+    for (const service of members) {
+      service.envArtifactSourceServiceId = source.serviceId;
+      service.envArtifactBuildServiceId = source.serviceId;
+      service.envArtifactGroupBuildRequired = buildNeeded && service.serviceId === source.serviceId;
+      if (buildNeeded && service.serviceId !== source.serviceId && service.buildRequired === true) {
+        service.buildRequired = false;
+        service.skipImageBuild = true;
+        service.sharedEnvBuildSkipped = true;
+        service.reason = uniqueSorted([...(Array.isArray(service.reason) ? service.reason : []), "shared-env-artifact-consumer"]);
+      }
+    }
+    plans.push({
+      id: group.id,
+      buildService: source.serviceId,
+      imageRepository: group.imageRepository,
+      cacheRepository: group.cacheRepository,
+      buildRequired: buildNeeded,
+      services: members.map((service) => service.serviceId),
+      buildServices: buildNeeded ? [source.serviceId] : [],
+      consumerServices: buildNeeded ? members.filter((service) => service.serviceId !== source.serviceId).map((service) => service.serviceId) : members.map((service) => service.serviceId)
+    });
+  }
+  return plans.sort((a, b) => a.id.localeCompare(b.id));
+}
+
 async function runtimeReuseDecisionForService({ repoRoot, baseRef, targetRef, serviceId, config }) {
  const runtimeReuse = config?.runtimeReuse ?? null;
  const envReuse = config?.envReuse ?? null;
@@ -1032,9 +1120,14 @@ function environmentDigestFromCatalogForHash(catalogRecord, environmentInputHash
  return catalogRecord?.environmentInputHash === environmentInputHash ? environmentDigestFromCatalog(catalogRecord) : null;
 }

-function envReuseImageRef(registryPrefix, serviceId, environmentInputHash) {
+function envReuseImageRef(registryPrefix, serviceId, environmentInputHash, group = null) {
  const tag = String(environmentInputHash || "unknown").slice(0, 12);
-  return `${registryPrefix}/${serviceId}-env:env-${tag}`;
+  const repository = group?.imageRepository ?? `${serviceId}-env`;
+  return `${registryPrefix}/${repository}:env-${tag}`;
+}
+
+function envReuseCacheRef(registryPrefix, group) {
+  return `${registryPrefix}/${group.cacheRepository}`;
 }

 function digestFromImageReference(image) {