From 7fc45188fae8c451bc951ebf0d30da8664e1b1d6 Mon Sep 17 00:00:00 2001 From: pikastech Date: Tue, 21 Jul 2026 15:40:53 +0200 Subject: [PATCH] =?UTF-8?q?feat(cicd):=20=E5=BB=BA=E7=AB=8B=20Temporal=20P?= =?UTF-8?q?aC=20=E8=87=AA=E5=8A=A8=E4=BA=A4=E4=BB=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .tekton/platform-infra-temporal-nc01-pac.yaml | 55 ++++++++++++++ config/platform-infra/pipelines-as-code.yaml | 30 ++++++++ config/platform-infra/temporal.yaml | 33 +++++++++ .../publish-platform-infra-temporal-gitops.ts | 59 +++++++++++++++ ...platform-infra-pipelines-as-code-remote.sh | 16 +++-- .../src/platform-infra-pipelines-as-code.ts | 9 ++- scripts/src/platform-infra-temporal.ts | 71 ++++++++++++++++--- 7 files changed, 258 insertions(+), 15 deletions(-) create mode 100644 .tekton/platform-infra-temporal-nc01-pac.yaml create mode 100644 scripts/native/cicd/publish-platform-infra-temporal-gitops.ts diff --git a/.tekton/platform-infra-temporal-nc01-pac.yaml b/.tekton/platform-infra-temporal-nc01-pac.yaml new file mode 100644 index 00000000..154435d6 --- /dev/null +++ b/.tekton/platform-infra-temporal-nc01-pac.yaml @@ -0,0 +1,55 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: platform-infra-temporal-nc01- + annotations: + pipelinesascode.tekton.dev/on-event: "[push]" + pipelinesascode.tekton.dev/on-target-branch: "[master]" + pipelinesascode.tekton.dev/on-cel-expression: "event == 'push' && target_branch == 'master' && node == 'NC01'" + labels: + app.kubernetes.io/name: platform-infra-temporal-nc01-pac + app.kubernetes.io/part-of: temporal + unidesk.ai/source-commit: "{{revision}}" +spec: + timeouts: + pipeline: 600s + taskRunTemplate: + serviceAccountName: default + podTemplate: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + securityContext: + fsGroup: 1000 + pipelineSpec: + tasks: + - name: validate-and-publish + taskSpec: + volumes: + - name: workspace + emptyDir: + sizeLimit: 1Gi + steps: + - name: publish-gitops + image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: IfNotPresent + workingDir: /workspace + env: + - name: SOURCE_COMMIT + value: "{{revision}}" + - name: SOURCE_SNAPSHOT_PREFIX + value: "{{source_snapshot_prefix}}" + script: | + #!/bin/sh + set -eu + git clone --filter=blob:none --no-checkout http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git source + cd source + git fetch --depth=1 --filter=blob:none origin "+$SOURCE_SNAPSHOT_PREFIX/$SOURCE_COMMIT:refs/remotes/origin/unidesk-source-snapshot" + git checkout --detach "$SOURCE_COMMIT" + test "$(git rev-parse HEAD)" = "$SOURCE_COMMIT" + bun scripts/cli.ts platform-infra temporal plan --target NC01 + exec bun scripts/native/cicd/publish-platform-infra-temporal-gitops.ts \ + --source-commit "$SOURCE_COMMIT" \ + --worktree /workspace/temporal-gitops + volumeMounts: + - name: workspace + mountPath: /workspace diff --git a/config/platform-infra/pipelines-as-code.yaml b/config/platform-infra/pipelines-as-code.yaml index 1b9107bb..8bd0bbfc 100644 --- a/config/platform-infra/pipelines-as-code.yaml +++ b/config/platform-infra/pipelines-as-code.yaml @@ -487,6 +487,36 @@ consumers: path: deploy/gitops/unidesk-host destinationNamespace: unidesk automated: true + - id: platform-infra-temporal-nc01 + repositoryRef: sentinel-nc01-v03 + node: NC01 + lane: platform-infra + namespace: devops-infra + pipeline: platform-infra-temporal-nc01-pac + pipelineRunPrefix: platform-infra-temporal-nc01- + argoNamespace: argocd + argoApplication: platform-infra-temporal-nc01 + deliveryObservation: + runtimeEvidence: required + closeoutGitOpsMirrorFlush: true + closeoutGitOpsMirrorLane: v03 + materializationOnly: true + materializationPaths: + - config/platform-infra/temporal.yaml + - scripts/src/platform-infra-temporal.ts + - scripts/native/cicd/publish-platform-infra-temporal-gitops.ts + - .tekton/platform-infra-temporal-nc01-pac.yaml + params: + runtime_deployment: temporal + gitops_branch: temporal-nc01-gitops + gitops_manifest_path: deploy/gitops/platform-infra/temporal-nc01/resources.yaml + argoBootstrap: + project: default + repoUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/unidesk.git + targetRevision: temporal-nc01-gitops + path: deploy/gitops/platform-infra/temporal-nc01 + destinationNamespace: temporal + automated: true - id: platform-infra-gitea-nc01 repositoryRef: sentinel-nc01-v03 node: NC01 diff --git a/config/platform-infra/temporal.yaml b/config/platform-infra/temporal.yaml index de4d4890..6b667e42 100644 --- a/config/platform-infra/temporal.yaml +++ b/config/platform-infra/temporal.yaml @@ -7,6 +7,7 @@ metadata: relatedIssues: - 2358 - 2374 + - 2784 defaults: targetId: NC01 @@ -24,6 +25,38 @@ targets: createNamespace: true enabled: true +delivery: + enabled: true + repositoryRef: sentinel-nc01-v03 + sourceBranch: master + sourceSnapshotPrefix: refs/unidesk/snapshots/gitea-actions/unidesk-master-nc01 + pipeline: + name: platform-infra-temporal-nc01-pac + runPrefix: platform-infra-temporal-nc01- + namespace: devops-infra + serviceAccount: default + timeout: 600s + gitops: + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/unidesk.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/unidesk.git + branch: temporal-nc01-gitops + path: deploy/gitops/platform-infra/temporal-nc01 + manifestPath: deploy/gitops/platform-infra/temporal-nc01/resources.yaml + author: + name: unidesk-temporal-ci + email: unidesk-temporal-ci@unidesk.local + argo: + namespace: argocd + application: platform-infra-temporal-nc01 + project: default + destinationNamespace: temporal + automated: true + materializationPaths: + - config/platform-infra/temporal.yaml + - scripts/src/platform-infra-temporal.ts + - scripts/native/cicd/publish-platform-infra-temporal-gitops.ts + - .tekton/platform-infra-temporal-nc01-pac.yaml + database: configRef: config/platform-db/postgres-nc01.yaml sourceRef: platform-db/temporal-nc01-db.env diff --git a/scripts/native/cicd/publish-platform-infra-temporal-gitops.ts b/scripts/native/cicd/publish-platform-infra-temporal-gitops.ts new file mode 100644 index 00000000..7536b4c3 --- /dev/null +++ b/scripts/native/cicd/publish-platform-infra-temporal-gitops.ts @@ -0,0 +1,59 @@ +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; + +import { readTemporalConfig, renderTemporalGitOpsManifest } from "../../src/platform-infra-temporal"; + +function option(name: string): string | null { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] ?? null : null; +} + +function required(value: string | null, name: string): string { + if (value === null || value.length === 0) throw new Error(`${name} is required`); + return value; +} + +function run(command: string, args: string[], cwd: string, allowFailure = false) { + const result = spawnSync(command, args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + if (!allowFailure && result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed: ${result.stderr.trim()}`); + return result; +} + +const sourceCommit = required(option("--source-commit"), "--source-commit"); +const worktree = resolve(option("--worktree") ?? "/workspace/temporal-gitops"); +const dryRun = process.argv.includes("--dry-run"); +if (!/^[0-9a-f]{40}$/u.test(sourceCommit)) throw new Error("--source-commit must be a full Git commit SHA"); +const temporal = readTemporalConfig(); +if (!temporal.delivery.enabled) throw new Error("config/platform-infra/temporal.yaml#delivery.enabled must be true"); +const target = temporal.targets.find((item) => item.id === temporal.defaults.targetId && item.enabled); +if (target === undefined) throw new Error("Temporal default delivery target is not enabled"); +const manifest = renderTemporalGitOpsManifest(temporal, target); +if (manifest.includes("kind: Secret")) throw new Error("GitOps manifest must not contain Secret payloads"); +if (!manifest.includes("kind: Deployment") || !manifest.includes("kind: Service")) throw new Error("Temporal GitOps renderer did not produce workloads"); +if (dryRun) { + console.log(JSON.stringify({ ok: true, mutation: false, mode: "dry-run", sourceCommit, branch: temporal.delivery.gitops.branch, manifestPath: temporal.delivery.gitops.manifestPath, objectCount: manifest.split("\n---\n").length, secretValuesPrinted: false, valuesPrinted: false })); + process.exit(0); +} + +rmSync(worktree, { recursive: true, force: true }); +run("git", ["clone", "--no-checkout", temporal.delivery.gitops.readUrl, worktree], process.cwd()); +const fetched = run("git", ["fetch", "origin", temporal.delivery.gitops.branch], worktree, true).status === 0; +if (fetched) run("git", ["checkout", "-B", temporal.delivery.gitops.branch, `origin/${temporal.delivery.gitops.branch}`], worktree); +else { + run("git", ["checkout", "--orphan", temporal.delivery.gitops.branch], worktree); + run("git", ["rm", "-rf", "."], worktree, true); +} +const manifestPath = resolve(worktree, temporal.delivery.gitops.manifestPath); +if (!manifestPath.startsWith(`${worktree}/`)) throw new Error("delivery.gitops.manifestPath escaped worktree"); +mkdirSync(dirname(manifestPath), { recursive: true }); +writeFileSync(manifestPath, `${manifest.trim()}\n`, "utf8"); +run("git", ["add", temporal.delivery.gitops.manifestPath], worktree); +run("git", ["config", "user.name", temporal.delivery.gitops.author.name], worktree); +run("git", ["config", "user.email", temporal.delivery.gitops.author.email], worktree); +const changed = run("git", ["diff", "--cached", "--quiet"], worktree, true).status !== 0; +if (changed) { + run("git", ["commit", "-m", `chore(temporal): publish ${sourceCommit.slice(0, 12)}`], worktree); + run("git", ["push", temporal.delivery.gitops.writeUrl, `HEAD:${temporal.delivery.gitops.branch}`], worktree); +} +console.log(JSON.stringify({ ok: true, mutation: changed, sourceCommit, branch: temporal.delivery.gitops.branch, manifestPath: temporal.delivery.gitops.manifestPath, secretValuesPrinted: false, valuesPrinted: false })); diff --git a/scripts/src/platform-infra-pipelines-as-code-remote.sh b/scripts/src/platform-infra-pipelines-as-code-remote.sh index b8c97d8f..2ebce7d9 100644 --- a/scripts/src/platform-infra-pipelines-as-code-remote.sh +++ b/scripts/src/platform-infra-pipelines-as-code-remote.sh @@ -304,10 +304,14 @@ NODE const fs = require('node:fs'); const changedPaths = fs.readFileSync(process.argv[2], 'utf8').split('\n').filter(Boolean); const domain = JSON.parse(fs.readFileSync(process.argv[3], 'utf8')); +const materializationPaths = JSON.parse(process.env.UNIDESK_PAC_MANUAL_MATERIALIZATION_PATHS_JSON || '[]'); +const effectiveChangedPaths = materializationPaths.length === 0 + ? changedPaths + : changedPaths.filter((changedPath) => materializationPaths.some((inputPath) => changedPath === inputPath || changedPath.startsWith(`${inputPath}/`))); const configuredImages = JSON.parse(process.env.UNIDESK_PAC_MANUAL_IMAGE_REPOSITORIES_JSON || '[]'); -const affectedServices = domain?.affectedServices || (changedPaths.length > 0 ? [process.env.UNIDESK_PAC_MANUAL_RUNTIME_SERVICE].filter(Boolean) : []); +const affectedServices = domain?.affectedServices || (effectiveChangedPaths.length > 0 ? [process.env.UNIDESK_PAC_MANUAL_RUNTIME_SERVICE].filter(Boolean) : []); const rolloutServices = domain?.rolloutServices || affectedServices; -const buildServices = domain?.buildServices || (changedPaths.length > 0 ? configuredImages : []); +const buildServices = domain?.buildServices || (effectiveChangedPaths.length > 0 ? configuredImages : []); const requestedServices = domain ? (domain.services || []) .filter((service) => (service.changedPaths || []).length > 0 @@ -315,7 +319,7 @@ const requestedServices = domain || (service.codeChangedPaths || []).length > 0 || service.runtimeConfigChanged === true) .map((service) => service.serviceId) - : (changedPaths.length > 0 ? [process.env.UNIDESK_PAC_MANUAL_RUNTIME_SERVICE].filter(Boolean) : []); + : (effectiveChangedPaths.length > 0 ? [process.env.UNIDESK_PAC_MANUAL_RUNTIME_SERVICE].filter(Boolean) : []); const expandedServices = affectedServices.filter((serviceId) => !requestedServices.includes(serviceId)); const envGroups = Array.isArray(domain?.envArtifactGroups) ? domain.envArtifactGroups : []; const envReuse = domain ? { @@ -329,8 +333,10 @@ const plan = { sourceCommit: process.env.SOURCE_COMMIT, baseCommit: process.env.BASE_COMMIT, sourceBranch: process.env.UNIDESK_PAC_MANUAL_SOURCE_BRANCH, - changedPathCount: changedPaths.length, - changedPaths, + changedPathCount: effectiveChangedPaths.length, + changedPaths: effectiveChangedPaths, + sourceChangedPathCount: changedPaths.length, + materializationPaths, artifactCatalog: domain?.artifactCatalog || null, rangeBaseline: { mode: process.env.BASE_SOURCE, diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index 4a6a045f..3689d655 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -253,6 +253,8 @@ interface PacConsumer { } | null; repositoryRef: string; params: Record; + materializationOnly: boolean; + materializationPaths: string[]; closeoutGitOpsMirrorFlush: boolean; closeoutGitOpsMirrorLane: "v02" | "v03" | null; sourceArtifact: PacSourceArtifactSpec | null; @@ -977,6 +979,10 @@ function parseConsumer(consumer: Record, path: string, defaultR deliveryProvenance, repositoryRef: typeof consumer.repositoryRef === "string" && consumer.repositoryRef.length > 0 ? consumer.repositoryRef : defaultRepositoryRef, params: consumer.params === undefined ? {} : stringRecord(y.objectField(consumer, "params", path), `${path}.params`), + materializationOnly: consumer.materializationOnly === undefined ? false : y.booleanField(consumer, "materializationOnly", path), + materializationPaths: consumer.materializationPaths === undefined + ? [] + : y.stringArrayField(consumer, "materializationPaths", path), closeoutGitOpsMirrorFlush: y.booleanField(consumer, "closeoutGitOpsMirrorFlush", path), closeoutGitOpsMirrorLane: consumer.closeoutGitOpsMirrorLane === undefined ? null : y.enumField(consumer, "closeoutGitOpsMirrorLane", path, ["v02", "v03"] as const), sourceArtifact: consumer.sourceArtifact === undefined ? null : parseSourceArtifact(y.objectField(consumer, "sourceArtifact", path), `${path}.sourceArtifact`), @@ -1613,7 +1619,7 @@ async function manualRelease(config: UniDeskConfig, action: "plan" | "trigger", if (consumer.node.toLowerCase() !== target.id.toLowerCase()) throw new Error(`consumer ${consumer.id} belongs to ${consumer.node}, not ${target.id}`); const repository = resolveRepository(pac, consumer.repositoryRef); const params = consumerParams(repository, consumer); - const imageRepositories = [...new Set(Object.entries(params) + const imageRepositories = consumer.materializationOnly ? [] : [...new Set(Object.entries(params) .filter(([key, value]) => key.endsWith("image_repository") && value.length > 0) .map(([, value]) => value))]; const hwlabLane = consumer.sourceArtifact?.renderer === "hwlab-runtime-lane" @@ -2472,6 +2478,7 @@ function remoteScript(action: "apply" | "status" | "history" | "diagnose-regress UNIDESK_PAC_MANUAL_IMAGE_REPOSITORIES_JSON: JSON.stringify(manualRelease?.imageRepositories ?? []), UNIDESK_PAC_MANUAL_RUNTIME_SERVICE: manualRelease?.runtimeService ?? "", UNIDESK_PAC_MANUAL_RENDERER: consumer.sourceArtifact?.renderer ?? "generic", + UNIDESK_PAC_MANUAL_MATERIALIZATION_PATHS_JSON: JSON.stringify(consumer.materializationPaths), UNIDESK_PAC_MANUAL_NODE_MODULES: manualRelease?.planningNodeModules ?? "", UNIDESK_PAC_MANUAL_ARTIFACT_CATALOG_PATH: manualRelease?.artifactCatalogPath ?? "", UNIDESK_PAC_MANUAL_GITOPS_READ_URL: manualRelease?.gitopsReadUrl ?? "", diff --git a/scripts/src/platform-infra-temporal.ts b/scripts/src/platform-infra-temporal.ts index 66a5028c..3946b1b1 100644 --- a/scripts/src/platform-infra-temporal.ts +++ b/scripts/src/platform-infra-temporal.ts @@ -13,6 +13,7 @@ import { parseOpsCommonOptions, readYamlRecord, shQuote, + stringArrayField, stringField, } from "./platform-infra-ops-library"; import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets"; @@ -59,6 +60,16 @@ export interface TemporalWebProbeSpec { export interface TemporalConfig { defaults: { targetId: string }; + delivery: { + enabled: boolean; + repositoryRef: string; + sourceBranch: string; + sourceSnapshotPrefix: string; + pipeline: { name: string; runPrefix: string; namespace: string; serviceAccount: string; timeout: string }; + gitops: { readUrl: string; writeUrl: string; branch: string; path: string; manifestPath: string; author: { name: string; email: string } }; + argo: { namespace: string; application: string; project: string; destinationNamespace: string; automated: boolean }; + materializationPaths: string[]; + }; images: { server: string; ui: string; authProxy: string; pullPolicy: string }; targets: TemporalTarget[]; database: { @@ -229,7 +240,11 @@ async function observe( }; } -function renderManifest(temporal: TemporalConfig, target: TemporalTarget, material: DatabaseMaterial | null): string { +export function renderTemporalGitOpsManifest(temporal: TemporalConfig, target: TemporalTarget): string { + return renderManifest(temporal, target, null, false); +} + +function renderManifest(temporal: TemporalConfig, target: TemporalTarget, material: DatabaseMaterial | null, includeDatabaseSecret = true): string { const server = temporal.runtime.server; const ui = temporal.runtime.ui; const secretData = material === null @@ -237,13 +252,7 @@ function renderManifest(temporal: TemporalConfig, target: TemporalTarget, materi : material; const encoded = (value: string) => Buffer.from(value, "utf8").toString("base64"); const namespace = target.namespace; - return `apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} - labels: - app.kubernetes.io/part-of: temporal ---- + const databaseSecret = includeDatabaseSecret ? `--- apiVersion: v1 kind: Secret metadata: @@ -257,7 +266,14 @@ data: password: ${encoded(secretData.password)} database: ${encoded(secretData.database)} visibilityDatabase: ${encoded(secretData.visibilityDatabase)} ---- +` : ""; + return `apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + app.kubernetes.io/part-of: temporal +${databaseSecret}--- apiVersion: v1 kind: Service metadata: @@ -554,6 +570,11 @@ function readDatabaseMaterial(temporal: TemporalConfig): DatabaseMaterial { export function readTemporalConfig(): TemporalConfig { const root = readYamlRecord(configPath, "platform-infra-temporal"); const defaults = recordField(root, "defaults", configLabel); + const delivery = recordField(root, "delivery", configLabel); + const deliveryPipeline = recordField(delivery, "pipeline", `${configLabel}.delivery`); + const deliveryGitops = recordField(delivery, "gitops", `${configLabel}.delivery`); + const deliveryGitopsAuthor = recordField(deliveryGitops, "author", `${configLabel}.delivery.gitops`); + const deliveryArgo = recordField(delivery, "argo", `${configLabel}.delivery`); const images = recordField(root, "images", configLabel); const database = recordField(root, "database", configLabel); const sourceKeys = recordField(database, "sourceKeys", `${configLabel}.database`); @@ -599,6 +620,38 @@ export function readTemporalConfig(): TemporalConfig { if (browserProxyMode !== "auto" && browserProxyMode !== "direct") throw new Error(`${configLabel}.webProbe.origin.browserProxyMode must be auto or direct`); return { defaults: { targetId: stringField(defaults, "targetId", `${configLabel}.defaults`) }, + delivery: { + enabled: booleanField(delivery, "enabled", `${configLabel}.delivery`), + repositoryRef: stringField(delivery, "repositoryRef", `${configLabel}.delivery`), + sourceBranch: stringField(delivery, "sourceBranch", `${configLabel}.delivery`), + sourceSnapshotPrefix: stringField(delivery, "sourceSnapshotPrefix", `${configLabel}.delivery`), + pipeline: { + name: stringField(deliveryPipeline, "name", `${configLabel}.delivery.pipeline`), + runPrefix: stringField(deliveryPipeline, "runPrefix", `${configLabel}.delivery.pipeline`), + namespace: stringField(deliveryPipeline, "namespace", `${configLabel}.delivery.pipeline`), + serviceAccount: stringField(deliveryPipeline, "serviceAccount", `${configLabel}.delivery.pipeline`), + timeout: stringField(deliveryPipeline, "timeout", `${configLabel}.delivery.pipeline`), + }, + gitops: { + readUrl: stringField(deliveryGitops, "readUrl", `${configLabel}.delivery.gitops`), + writeUrl: stringField(deliveryGitops, "writeUrl", `${configLabel}.delivery.gitops`), + branch: stringField(deliveryGitops, "branch", `${configLabel}.delivery.gitops`), + path: stringField(deliveryGitops, "path", `${configLabel}.delivery.gitops`), + manifestPath: stringField(deliveryGitops, "manifestPath", `${configLabel}.delivery.gitops`), + author: { + name: stringField(deliveryGitopsAuthor, "name", `${configLabel}.delivery.gitops.author`), + email: stringField(deliveryGitopsAuthor, "email", `${configLabel}.delivery.gitops.author`), + }, + }, + argo: { + namespace: stringField(deliveryArgo, "namespace", `${configLabel}.delivery.argo`), + application: stringField(deliveryArgo, "application", `${configLabel}.delivery.argo`), + project: stringField(deliveryArgo, "project", `${configLabel}.delivery.argo`), + destinationNamespace: stringField(deliveryArgo, "destinationNamespace", `${configLabel}.delivery.argo`), + automated: booleanField(deliveryArgo, "automated", `${configLabel}.delivery.argo`), + }, + materializationPaths: stringArrayField(delivery, "materializationPaths", `${configLabel}.delivery`), + }, images: { server: stringField(images, "server", `${configLabel}.images`), ui: stringField(images, "ui", `${configLabel}.images`),