Merge pull request #2767 from pikasTech/fix/workbench-kafka-sse-l2

修复生产首次物化共享环境制品

.tekton/hwlab-nc01-production-pac.yaml

@@ -1,18 +1,19 @@
 apiVersion: tekton.dev/v1
 kind: PipelineRun
 metadata:
-  generateName: hwlab-nc01-production-ci-poll-
+  name: "hwlab-nc01-prod-ci-{{ revision }}"
   namespace: hwlab-ci
   annotations:
     pipelinesascode.tekton.dev/on-event: "[push]"
     pipelinesascode.tekton.dev/on-target-branch: "[release]"
-    pipelinesascode.tekton.dev/on-cel-expression: "event == 'push' && target_branch == 'release' && node == 'NC01'"
+    pipelinesascode.tekton.dev/on-cel-expression: "event == 'push' && target_branch == 'release' && node == 'NC01' && body.head_commit.message.startsWith('unidesk-release-plan-v1.hwlab-nc01-production.')"
     pipelinesascode.tekton.dev/max-keep-runs: "8"
     unidesk.ai/owning-config-ref: "config/hwlab-node-lanes.yaml#lanes.production.targets.NC01"
-    unidesk.ai/effective-config-sha256: sha256:f0b827a59094630ba9a78a63c39a624613f93f87e685779732ebc38bd694af39
+    unidesk.ai/effective-config-sha256: sha256:15ec3bab90b30bda0a308bfb5ae9f83386f45f9175203ad351c610250e2f5814
     unidesk.ai/source-artifact-renderer: hwlab-runtime-lane
     unidesk.ai/source-artifact-mode: remote-pipeline-annotation
     pipelinesascode.tekton.dev/pipeline: ci/pipelines/hwlab-nc01-production-ci-image-publish.yaml
+    unidesk.ai/ci-dedupe-mode: deterministic-kubernetes-name
     hwlab.pikastech.local/ci-contract: tekton-native-primitive-tasks
     hwlab.pikastech.local/download-profile: jd01-node-default
     hwlab.pikastech.local/gitops-branch: release-gitops
@@ -39,6 +40,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       securityContext:
         fsGroup: 1000
+      priorityClassName: unidesk-ci-batch
   pipelineRef:
     name: hwlab-nc01-production-ci-image-publish
   workspaces:
@@ -50,16 +52,23 @@ spec:
           resources:
             requests:
               storage: 8Gi
-    - name: git-ssh
+    - name: gitea-auth
       secret:
-        secretName: hwlab-git-ssh
+        secretName: pac-gitea-hwlab-nc01-v03
+        items:
+          - key: token
+            path: token
   params:
     - name: git-url
       value: "{{ repo_url }}"
     - name: git-read-url
       value: "{{ git_read_url }}"
+    - name: gitops-read-url
+      value: "{{ git_read_url }}"
     - name: git-write-url
-      value: "{{ git_write_url }}"
+      value: "{{ git_read_url }}"
+    - name: gitops-username
+      value: "{{ gitops_username }}"
     - name: source-branch
       value: release
     - name: gitops-branch
@@ -77,10 +86,18 @@ spec:
     - name: registry-prefix
       value: 127.0.0.1:5000/hwlab
     - name: services
-      value: "hwlab-cloud-api,hwlab-workbench-runtime,hwlab-user-billing,hwlab-project-management,hwlab-tasktree-api,hwlab-tasktree-worker,hwlab-harnessrl-api,hwlab-harnessrl-worker,hwlab-workbench-api,hwlab-workbench-worker,hwlab-cloud-web,hwlab-gateway,hwlab-edge-proxy,hwlab-agent-skills"
+      value: "hwlab-cloud-api,hwlab-workbench-runtime,hwlab-user-billing,hwlab-project-management,hwlab-tasktree-api,hwlab-tasktree-worker,hwlab-harnessrl-api,hwlab-harnessrl-worker,hwlab-workbench-api,hwlab-workbench-worker,hwlab-hwpod-api,hwlab-hwpod-worker,hwlab-hwpod-web,hwlab-cloud-web,hwlab-gateway,hwlab-edge-proxy,hwlab-agent-skills"
     - name: base-image
       value: 127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim
     - name: build-cache-mode
       value: registry
     - name: runtime-ready-timeout-ms
       value: "60000"
+    - name: release-base-commit
+      value: "{{ body.before }}"
+    - name: release-target
+      value: NC01
+    - name: release-consumer
+      value: hwlab-nc01-production
+    - name: reviewed-plan
+      value: "{{ body.head_commit.message }}"

scripts/ci-plan.test.mjs

@@ -915,6 +915,108 @@ test("artifact publish ignores stale external reports for envreuse services that
   assert.equal(result.services[0].sourceCommitId, currentCommit);
 });
 
+test("artifact publish derives a shared env consumer from its source report when the runtime catalog is missing", async () => {
+  const currentCommit = (await git(process.cwd(), ["rev-parse", "HEAD"])).stdout.trim();
+  const digest = `sha256:${"5".repeat(64)}`;
+  const tmp = await mkdtemp(path.join(os.tmpdir(), "hwlab-artifact-first-materialization-"));
+  const reportDir = path.join(tmp, "service-results");
+  await mkdir(reportDir, { recursive: true });
+  const image = "127.0.0.1:5000/hwlab/hwlab-ts-runtime-env:env-first";
+  const ciPlanPath = path.join(tmp, "ci-plan.json");
+  const missingCatalogPath = path.join(tmp, "artifact-catalog.production.json");
+  const sourceServiceId = "hwlab-cloud-api";
+  const consumerServiceId = "hwlab-project-management";
+  const sharedFields = {
+    runtimeMode: "env-reuse-gitea-checkout",
+    envReuse: true,
+    affected: true,
+    envChanged: true,
+    codeChanged: true,
+    environmentImage: image,
+    environmentDigest: null,
+    environmentInputHash: "e".repeat(64),
+    bootRepo: "git@github.com:pikasTech/HWLAB.git",
+    bootCommit: currentCommit,
+    envArtifactGroupId: "hwlab-ts-runtime-env",
+    envArtifactSourceServiceId: sourceServiceId
+  };
+  await writeFile(ciPlanPath, JSON.stringify({
+    planVersion: "v1",
+    sourceCommitId: currentCommit,
+    shortCommitId: currentCommit.slice(0, 12),
+    baseRef: "HEAD~1",
+    targetRef: "HEAD",
+    compatibility: { lane: "production" },
+    changedPathSummary: {},
+    imageBuildRequired: true,
+    affectedServices: [sourceServiceId, consumerServiceId],
+    buildServices: [sourceServiceId],
+    rolloutServices: [sourceServiceId, consumerServiceId],
+    reusedServices: [],
+    buildSkippedCount: 1,
+    ciCdPlan: {},
+    services: [{
+      serviceId: sourceServiceId,
+      ...sharedFields,
+      buildRequired: true,
+      envArtifactGroupBuildRequired: true,
+      bootSh: `deploy/runtime/boot/${sourceServiceId}.sh`
+    }, {
+      serviceId: consumerServiceId,
+      ...sharedFields,
+      buildRequired: false,
+      sharedEnvBuildSkipped: true,
+      bootSh: `deploy/runtime/boot/${consumerServiceId}.sh`,
+      reason: ["shared-env-artifact-consumer"]
+    }]
+  }, null, 2));
+  await writeFile(path.join(reportDir, `${sourceServiceId}.json`), JSON.stringify({
+    artifactPublish: {
+      services: [{
+        serviceId: sourceServiceId,
+        status: "published",
+        sourceCommitId: currentCommit,
+        image,
+        imageTag: "env-first",
+        digest,
+        repositoryDigest: `${image.slice(0, image.lastIndexOf(":"))}@${digest}`,
+        runtimeMode: "env-reuse-gitea-checkout",
+        envReuse: true,
+        environmentImage: image,
+        environmentDigest: digest,
+        environmentInputHash: "e".repeat(64),
+        bootCommit: currentCommit
+      }]
+    }
+  }, null, 2));
+
+  const { stdout } = await execFileAsync(process.execPath, [
+    "scripts/artifact-publish.mjs",
+    "--build",
+    "--lane", "production",
+    "--services", `${sourceServiceId},${consumerServiceId}`,
+    "--deploy-config", "deploy/deploy.yaml",
+    "--catalog-path", missingCatalogPath,
+    "--ci-plan-path", ciPlanPath,
+    "--external-service-report-dir", reportDir,
+    "--registry-prefix", "127.0.0.1:5000/hwlab",
+    "--no-report"
+  ], {
+    cwd: process.cwd(),
+    env: { ...process.env, HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY: "false" }
+  });
+  const result = JSON.parse(stdout);
+  assert.equal(result.status, "built");
+  assert.deepEqual(result.blockers, []);
+  const source = result.services.find((service) => service.serviceId === sourceServiceId);
+  const consumer = result.services.find((service) => service.serviceId === consumerServiceId);
+  assert.equal(source.status, "published");
+  assert.equal(consumer.status, "reused");
+  assert.equal(consumer.digest, digest);
+  assert.equal(consumer.reusedFrom, sourceServiceId);
+  assert.equal(consumer.buildBackend, "shared-env-artifact");
+});
+
 test("artifact publish accepts stale reused envreuse external reports for unchanged services in buildServices", async () => {
   const currentCommit = (await git(process.cwd(), ["rev-parse", "HEAD"])).stdout.trim();
   const oldCommit = "1".repeat(40);

scripts/gitops-render.test.ts

@@ -71,8 +71,8 @@ test("NC01 PaC artifact keeps the runtime GitOps file contract", async () => {
 
   assert.match(pacSource, new RegExp(`pipelinesascode\\.tekton\\.dev/pipeline: ${pipelinePath.replaceAll("/", "\\/")}`, "u"));
 
-  const gitopsPromote = taskByName(pipeline, "gitops-promote");
-  assert.deepEqual(gitopsPromote.taskSpec.volumes, [{
+  const releaseArtifacts = taskByName(pipeline, "release-artifacts");
+  assert.deepEqual(releaseArtifacts.taskSpec.volumes, [{
     name: "unidesk-runtime-gitops-scripts",
     configMap: {
       name: "hwlab-nc01-v03-ci-image-publish-runtime-gitops-scripts",
@@ -80,8 +80,8 @@ test("NC01 PaC artifact keeps the runtime GitOps file contract", async () => {
     }
   }]);
 
-  const promoteStep = gitopsPromote.taskSpec.steps.find((step) => step.name === "promote");
-  assert.ok(promoteStep, "missing gitops-promote promote step");
+  const promoteStep = releaseArtifacts.taskSpec.steps.find((step) => step.name === "promote");
+  assert.ok(promoteStep, "missing release-artifacts promote step");
   assert.equal(promoteStep.image, "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1");
   assert.deepEqual(promoteStep.volumeMounts, [{
     name: "unidesk-runtime-gitops-scripts",
@@ -236,7 +236,8 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
     const pipelineJson = JSON.parse(pipeline);
     assert.match(pipeline, /127\.0\.0\.1:5000\/hwlab\/hwlab-ci-node-tools:node22-alpine-bun-v1/u);
     assert.match(pipeline, /tar -C \/workspace\/source\/repo -cf - \. \| tar -C \/workspace\/service-work\/repo -xo -f -/u);
-    assert.match(pipeline, /node scripts\/run-bun\.mjs test cmd\/hwlab-codex-api-responses-forwarder\/main\.test\.ts/u);
+    assert.deepEqual(pipelineJson.spec.tasks.map((task) => task.name), ["plan-artifacts", "build-services", "release-artifacts"]);
+    assert.doesNotMatch(pipeline, /node scripts\/run-bun\.mjs test cmd\/hwlab-codex-api-responses-forwarder\/main\.test\.ts/u);
     assert.doesNotMatch(pipeline, /cmd\/hwlab-codex-api-responses-forwarder\/main\.mjs/u);
     const runtimeReadySource = await readFile(path.join(process.cwd(), "scripts/ci/runtime-ready.mjs"), "utf8");
     assert.doesNotMatch(runtimeReadySource, /argo-sync-health|healthStatus === "Healthy"/u);
@@ -249,7 +250,7 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
     assert.ok(!pipelineJson.spec.tasks.some((task) => task.name === "runtime-ready"));
 
     const planArtifacts = taskByName(pipelineJson, "plan-artifacts");
-    const planArtifactsScript = planArtifacts.taskSpec.steps[0].script;
+    const planArtifactsScript = planArtifacts.taskSpec.steps.find((step) => step.name === "plan")?.script ?? "";
     assert.match(planArtifactsScript, /node scripts\/ci\/restore-artifact-catalog\.mjs/u);
     assert.match(planArtifactsScript, /HWLAB_GIT_READ_URL="\$\(params\.gitops-read-url\)"/u);
     assert.match(planArtifactsScript, /HWLAB_SERVICES="\$\(params\.services\)"/u);
@@ -267,9 +268,9 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
       assert.ok(planArtifacts.taskSpec.params.some((param) => param.name === name));
     }
 
-    const gitopsPromote = taskByName(pipelineJson, "gitops-promote");
-    assert.ok(gitopsPromote.taskSpec.results.some((result) => result.name === "runtime-ready-required"));
-    const gitopsPromoteScript = gitopsPromote.taskSpec.steps[0].script;
+    const releaseArtifacts = taskByName(pipelineJson, "release-artifacts");
+    assert.ok(releaseArtifacts.taskSpec.results.some((result) => result.name === "runtime-ready-required"));
+    const gitopsPromoteScript = releaseArtifacts.taskSpec.steps.find((step) => step.name === "promote")?.script ?? "";
     assert.match(gitopsPromoteScript, /if \[ "\$runtime_lane" = "true" \]; then\s+printf 'false' > \/tekton\/results\/runtime-ready-required\s+else\s+printf 'true' > \/tekton\/results\/runtime-ready-required/u);
     assert.match(gitopsPromoteScript, /runtime-identity-only/u);
     assert.match(gitopsPromoteScript, /skipped-runtime-unchanged/u);
@@ -301,11 +302,11 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
     assert.doesNotMatch(gitopsPromoteScript, /gitops-promote-proxy-preflight/u);
     assert.doesNotMatch(gitopsPromoteScript, /for tool in node npm git ssh curl timeout/u);
 
-    const prepareSource = taskByName(pipelineJson, "prepare-source");
+    const prepareSource = planArtifacts;
     assert.ok(prepareSource.taskSpec.params.some((param) => param.name === "git-read-url"));
     assert.ok(prepareSource.taskSpec.params.some((param) => param.name === "gitops-read-url"));
     assert.ok(prepareSource.params.some((param) => param.name === "git-read-url" && param.value === "$(params.git-read-url)"));
-    const prepareSourceScript = prepareSource.taskSpec.steps[0].script;
+    const prepareSourceScript = prepareSource.taskSpec.steps.find((step) => step.name === "prepare-source")?.script ?? "";
     assert.match(prepareSourceScript, /git_read_url="\$\(params\.git-read-url\)"/u);
     assert.doesNotMatch(prepareSourceScript, /git-ssh-setup/u);
     assert.match(prepareSourceScript, /git_timed source-clone 180 git clone --branch "\$\(params\.source-branch\)" "\$git_read_url"/u);
@@ -321,9 +322,9 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
     assert.doesNotMatch(prepareSourceScript, /prepare-source-proxy-preflight/u);
     assert.doesNotMatch(prepareSourceScript, /dependency-curl-probe-start/u);
 
-    assert.doesNotMatch(planArtifacts.taskSpec.steps[0].script, /plan-artifacts-proxy-preflight/u);
-    const collectArtifacts = taskByName(pipelineJson, "collect-artifacts");
-    assert.doesNotMatch(collectArtifacts.taskSpec.steps[0].script, /collect-artifacts-proxy-preflight/u);
+    assert.doesNotMatch(planArtifactsScript, /plan-artifacts-proxy-preflight/u);
+    const collectArtifactsScript = releaseArtifacts.taskSpec.steps.find((step) => step.name === "collect")?.script ?? "";
+    assert.doesNotMatch(collectArtifactsScript, /collect-artifacts-proxy-preflight/u);
 
     const pipelineGitReadParam = pipelineJson.spec.params.find((param) => param.name === "git-read-url");
     assert.equal(pipelineGitReadParam?.default, "http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-HWLAB.git");
@@ -367,7 +368,7 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
     assert.deepEqual(buildServices.matrix, { params: [{ name: "service-id", value: retainedServiceIds.map((id) => `hwlab-${id.replace(/^hwlab-/u, "")}`) }] });
     assert.ok(!buildServices.params.some((param) => param.name === "service-id"));
 
-    const collectScript = collectArtifacts.taskSpec.steps[0].script;
+    const collectScript = collectArtifactsScript;
     assert.match(collectScript, /--external-service-report-dir \/workspace\/source\/service-results/u);
     assert.ok(
       buildServices.taskSpec.results.some((result) => result.name === "go-base-image-status"),
@@ -594,6 +595,16 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
   }
 });
 
+test("shared env consumer defers to the current source report before reading a prior catalog", async () => {
+  const script = await readFile("scripts/src/gitops-render/templates/per-service-publish.sh", "utf8");
+  assert.match(script, /reason: "shared-env-artifact-pending"/u);
+  assert.match(script, /planned\.envReuse === true && sharedEnvSourceServiceId && sharedEnvSourceServiceId !== serviceId && sharedEnvSource\.buildRequired === true/u);
+  assert.ok(
+    script.indexOf("shared-env-artifact-pending") < script.indexOf("fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH"),
+    "shared env consumers must defer to the source service report before reading a prior catalog"
+  );
+});
+
 test("v03 render keeps node identity as data instead of generated structure", async () => {
   const outDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-v03-render-test-"));
   const scratchDir = path.join(".tmp", `gitops-render-v03-${process.pid}-${Date.now()}`);
@@ -657,8 +668,8 @@ test("v03 render keeps node identity as data instead of generated structure", as
     assert.match(pipeline, /--external-service-report-dir \/workspace\/source\/service-results/u);
     assert.match(pipeline, /--ci-plan-path \/workspace\/source\/affected-services\.json/u);
     assert.ok(!pipelineJson.spec.tasks.some((task) => task.name === "runtime-ready"));
-    const gitopsPromote = taskByName(pipelineJson, "gitops-promote");
-    const gitopsPromoteScript = gitopsPromote.taskSpec.steps[0].script;
+    const releaseArtifacts = taskByName(pipelineJson, "release-artifacts");
+    const gitopsPromoteScript = releaseArtifacts.taskSpec.steps.find((step) => step.name === "promote")?.script ?? "";
     assert.match(gitopsPromoteScript, /if \[ "\$runtime_lane" = "true" \]; then\s+printf 'false' > \/tekton\/results\/runtime-ready-required\s+else\s+printf 'true' > \/tekton\/results\/runtime-ready-required/u);
     assert.match(gitopsPromoteScript, /delegated-post-flush-closeout/u);
     assert.ok(!generatedPaths.includes("runtime-v03/postgres.yaml"));

scripts/src/gitops-render/templates/per-service-publish.sh

@@ -24,10 +24,21 @@ if [ -s /workspace/source/affected-services.json ] && ! node -e 'const fs=requir
   node - "$(params.service-id)" <<'NODE'
 const fs = require("node:fs");
 const serviceId = process.argv[2];
-const catalog = JSON.parse(fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH, "utf8"));
 const plan = fs.existsSync("/workspace/source/affected-services.json") ? JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")) : {};
-const service = (catalog.services || []).find((item) => item.serviceId === serviceId) || {};
 const planned = (plan.services || []).find((item) => item.serviceId === serviceId) || {};
+const sharedEnvSourceServiceId = planned.envArtifactSourceServiceId || planned.envArtifactBuildServiceId || "";
+const sharedEnvSource = (plan.services || []).find((item) => item.serviceId === sharedEnvSourceServiceId) || {};
+if (planned.envReuse === true && sharedEnvSourceServiceId && sharedEnvSourceServiceId !== serviceId && sharedEnvSource.buildRequired === true) {
+  console.log(JSON.stringify({
+    event: "service-build-skip",
+    serviceId,
+    reason: "shared-env-artifact-pending",
+    envArtifactSourceServiceId: sharedEnvSourceServiceId
+  }));
+  process.exit(0);
+}
+const catalog = JSON.parse(fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH, "utf8"));
+const service = (catalog.services || []).find((item) => item.serviceId === serviceId) || {};
 const envReuse = planned.runtimeMode === "env-reuse-gitea-checkout" || service.runtimeMode === "env-reuse-gitea-checkout" || planned.envReuse === true || service.envReuse === true;
 const image = envReuse ? (service.environmentImage || service.image || "") : (service.image || "");
 const digest = envReuse ? (service.environmentDigest || service.digest || "not_published") : (service.digest || "not_published");