diff --git a/docs/reference/g14-cicd-performance.md b/docs/reference/g14-cicd-performance.md index e4d992b4..beb670c8 100644 --- a/docs/reference/g14-cicd-performance.md +++ b/docs/reference/g14-cicd-performance.md @@ -22,6 +22,14 @@ G14 CI/CD 耗时必须按阶段测量，不能只看一个 PipelineRun 总时长 - `multi-component-build`：多个但不是全部服务镜像重建。 - `full-rebuild`：共享输入或 build-system 输入变化导致全部服务镜像重建。 +## Timing 输出合同 + +G14 Tekton 日志中的分段耗时统一使用 JSON 行事件：`event="g14-cicd-timing"`、`schemaVersion="v1"`、`stage`、`status`、`durationMs`，并尽量携带 `pipelineRun`、`taskRun`、`task`、`revision` 和 `serviceId`。后续性能对比只消费这些稳定字段，不从自然语言日志里反推耗时。 + +当前稳定 stage 名称：`source-clone`、`catalog-fetch`、`npm-ci`、`buildkit-context-transfer`、`dependency-install`、`cache-import`、`cache-export`、`gitops-render`、`gitops-clone`、`gitops-commit`、`gitops-push`、`argo-refresh`、`workload-ready`。 + +`argo-refresh` 以 `hwlab-dev` workload 的 `hwlab.pikastech.local/source-commit` 元数据收敛作为观测点，不要求 Tekton 跨 namespace patch Argo Application。`workload-ready` 以 `hwlab-dev` 的 Deployment/StatefulSet observed generation、ready replicas 和 updated replicas 为准；权限不足或超时必须用 `status` 明确输出，不得静默吞掉。 + ## 当前基线 当前 G14 pipeline 已具备关键架构优化：一分钟 source polling、组件级 lazy build、per-service fan-out、G14 本地 registry、本地 base image、BuildKit-only publish、GitOps/Argo 分离。剩余耗时主要集中在固定开销和镜像构建机制上。 diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index 5d37044d..a61e1c38 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -215,6 +215,109 @@ function emit(event, payload = {}) { process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...payload })}\n`); } +function ciTimingEnabled(env = process.env) { + return env.HWLAB_G14_CICD_TIMING === "1" || env.HWLAB_CI_TIMING === "1"; +} + +function emitCiTiming(payload) { + emit("g14-cicd-timing", { + schemaVersion: "v1", + source: "scripts/artifact-publish.mjs", + pipelineRun: process.env.HWLAB_TEKTON_PIPELINERUN ?? null, + taskRun: process.env.HWLAB_TEKTON_TASKRUN ?? null, + task: process.env.HWLAB_TEKTON_TASK ?? null, + revision: process.env.HWLAB_SOURCE_REVISION ?? null, + ...payload + }); +} + +function parseDurationMs(value) { + const text = String(value ?? "").trim(); + if (!text) return null; + let total = 0; + let matched = false; + for (const match of text.matchAll(/(\d+(?:\.\d+)?)(ms|s|m|h)/gu)) { + matched = true; + const amount = Number(match[1]); + const unit = match[2]; + if (unit === "ms") total += amount; + else if (unit === "s") total += amount * 1000; + else if (unit === "m") total += amount * 60 * 1000; + else if (unit === "h") total += amount * 60 * 60 * 1000; + } + return matched ? Math.round(total) : null; +} + +function parseBuildkitPlainTimings(text) { + const idToStage = new Map(); + const timings = new Map(); + const stageMatchers = [ + { stage: "buildkit-context-transfer", pattern: /load build context|transferring context/iu }, + { stage: "cache-import", pattern: /importing cache manifest/iu }, + { stage: "cache-export", pattern: /exporting cache/iu } + ]; + for (const rawLine of String(text ?? "").split("\n")) { + const line = rawLine.replace(/\u001b\[[0-9;]*m/gu, "").trim(); + const match = line.match(/^#(\d+)\s+(.*)$/u); + if (!match) continue; + const [, id, body] = match; + const identified = stageMatchers.find((item) => item.pattern.test(body)); + if (identified) idToStage.set(id, identified.stage); + const stage = idToStage.get(id); + if (!stage || timings.has(stage)) continue; + const durationMatch = body.match(/(?:DONE|done)\s+([0-9.]+(?:ms|s|m|h)(?:[0-9.]+(?:ms|s|m|h))*)/iu) ?? + body.match(/([0-9.]+(?:ms|s|m|h)(?:[0-9.]+(?:ms|s|m|h))*)\s+done/iu); + const durationMs = parseDurationMs(durationMatch?.[1]); + if (durationMs !== null) timings.set(stage, { stage, status: "succeeded", durationMs, source: "buildkit-plain-progress" }); + } + return timings; +} + +function parseDockerfileStepTimings(text) { + const timings = []; + for (const rawLine of String(text ?? "").split("\n")) { + const line = rawLine.replace(/\u001b\[[0-9;]*m/gu, ""); + const start = line.indexOf("{"); + const end = line.lastIndexOf("}"); + if (start < 0 || end <= start) continue; + let parsed; + try { + parsed = JSON.parse(line.slice(start, end + 1)); + } catch { + continue; + } + if (parsed?.event !== "g14-cicd-build-step-timing") continue; + timings.push({ + stage: parsed.stage, + status: parsed.status ?? "unknown", + durationMs: Number.isFinite(parsed.durationMs) ? parsed.durationMs : null, + source: parsed.source ?? "dockerfile-run" + }); + } + return timings; +} + +function emitBuildkitTimingEvents({ service, result, timingEnabled }) { + if (!timingEnabled) return; + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + const buildkitTimings = parseBuildkitPlainTimings(output); + for (const stage of ["buildkit-context-transfer", "cache-import", "cache-export"]) { + const timing = buildkitTimings.get(stage) ?? { stage, status: "not-observed", durationMs: null, source: "buildkit-plain-progress" }; + emitCiTiming({ + ...timing, + serviceId: service.serviceId, + buildkitDurationMs: result.durationMs + }); + } + for (const timing of parseDockerfileStepTimings(output)) { + emitCiTiming({ + ...timing, + serviceId: service.serviceId, + buildkitDurationMs: result.durationMs + }); + } +} + function blocker({ type, scope, summary, next }) { return { type, @@ -934,6 +1037,7 @@ if (runtimeKind === "node-command" && entrypoint) { } function dockerfile(baseImage, port, labels = []) { + const dependencyTimingScript = `started_at=$(node -e "console.log(Date.now())"); status=succeeded; if [ -d /opt/hwlab-node-runtime-base/node_modules ]; then cp -a /opt/hwlab-node-runtime-base/node_modules ./node_modules; fi && if [ -f package-lock.json ]; then npm install --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi && codex_version="$(node -p "require('./node_modules/@openai/codex/package.json').version")" && case "$(uname -m)" in x86_64|amd64) test -e node_modules/@openai/codex-linux-x64 || npm install --omit=dev --include=optional --ignore-scripts "@openai/codex-linux-x64@npm:@openai/codex@\${codex_version}-linux-x64" ;; aarch64|arm64) test -e node_modules/@openai/codex-linux-arm64 || npm install --omit=dev --include=optional --ignore-scripts "@openai/codex-linux-arm64@npm:@openai/codex@\${codex_version}-linux-arm64" ;; esac || status=failed; finished_at=$(node -e "console.log(Date.now())"); node -e "console.log(JSON.stringify({event:'g14-cicd-build-step-timing',stage:'dependency-install',status:process.argv[1],durationMs:Number(process.argv[3])-Number(process.argv[2]),source:'dockerfile-run'}))" "$status" "$started_at" "$finished_at"; test "$status" = succeeded`; return [ `FROM ${baseImage}`, "WORKDIR /app", @@ -950,7 +1054,7 @@ function dockerfile(baseImage, port, labels = []) { "ENV HWLAB_CODE_AGENT_SKILLS_DIRS=/app/skills", "COPY package.json ./package.json", "COPY package-lock.json* ./", - "RUN if [ -d /opt/hwlab-node-runtime-base/node_modules ]; then cp -a /opt/hwlab-node-runtime-base/node_modules ./node_modules; fi && if [ -f package-lock.json ]; then npm install --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi && codex_version=\"$(node -p \"require('./node_modules/@openai/codex/package.json').version\")\" && case \"$(uname -m)\" in x86_64|amd64) test -e node_modules/@openai/codex-linux-x64 || npm install --omit=dev --include=optional --ignore-scripts \"@openai/codex-linux-x64@npm:@openai/codex@${codex_version}-linux-x64\" ;; aarch64|arm64) test -e node_modules/@openai/codex-linux-arm64 || npm install --omit=dev --include=optional --ignore-scripts \"@openai/codex-linux-arm64@npm:@openai/codex@${codex_version}-linux-arm64\" ;; esac", + `RUN ${dependencyTimingScript}`, "COPY internal ./internal", "COPY cmd ./cmd", "COPY scripts ./scripts", @@ -1243,7 +1347,7 @@ async function buildServiceWithBuildkit({ args, service, ref, tag, buildCreatedA "--export-cache", `type=registry,ref=${cacheRef},mode=max${registryOption}`, "--output", `type=image,name=${ref},push=true${registryOption}` ]; - if (!args.quietBuild) argsList.push("--progress", "plain"); + if (!args.quietBuild || ciTimingEnabled()) argsList.push("--progress", "plain"); for (const [name, value] of envs) argsList.push("--opt", `build-arg:${name}=${value}`); const buildEnv = { ...process.env }; @@ -1253,6 +1357,7 @@ async function buildServiceWithBuildkit({ args, service, ref, tag, buildCreatedA await mkdir(buildEnv.XDG_RUNTIME_DIR, { recursive: true }); } const result = await run(args.buildkitCommand, argsList, { env: buildEnv }); + emitBuildkitTimingEvents({ service, result, timingEnabled: ciTimingEnabled(buildEnv) }); if (result.code !== 0) { return { ...service, diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index 617eaff6..86ab9886 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -590,6 +590,23 @@ function shellSingleQuote(value) { return `'${String(value).replace(/'/g, `'"'"'`)}'`; } +function ciTimingShellFunction() { + return `ci_now_ms() { + node -e 'console.log(Date.now())' +} + +ci_timing_emit() { + stage="$1" + status="$2" + started_ms="$3" + finished_ms="$(ci_now_ms)" + duration_ms="$((finished_ms - started_ms))" + service_id="\${HWLAB_TIMING_SERVICE_ID:-}" + node -e 'const [stage,status,durationMs,serviceId]=process.argv.slice(1); const payload={event:"g14-cicd-timing",schemaVersion:"v1",stage,status,durationMs:Number(durationMs),pipelineRun:process.env.HWLAB_TEKTON_PIPELINERUN||null,taskRun:process.env.HWLAB_TEKTON_TASKRUN||null,task:process.env.HWLAB_TEKTON_TASK||null,revision:process.env.HWLAB_SOURCE_REVISION||null,serviceId:serviceId||null,source:"scripts/g14-gitops-render.mjs",at:new Date().toISOString()}; console.log(JSON.stringify(payload));' "$stage" "$status" "$duration_ms" "$service_id" +} +`; +} + function dependencyProxyProbeScript(phase, targetUrl) { return `HWLAB_PROXY_PROBE_PHASE=${shellSingleQuote(phase)} HWLAB_PROXY_PROBE_URL=${shellSingleQuote(targetUrl)} node <<'NODE' || echo '{"event":"dependency-proxy-probe-nonblocking","phase":"${phase}","ok":false,"reason":"probe-failed-but-continuing"}' const net = require("node:net"); @@ -714,6 +731,11 @@ function prepareSourceScript() { const lines = [ "#!/bin/sh", "set -eu", + ciTimingShellFunction(), + "export HWLAB_TEKTON_PIPELINERUN=\"$(context.pipelineRun.name)\"", + "export HWLAB_TEKTON_TASKRUN=\"$(context.taskRun.name)\"", + "export HWLAB_TEKTON_TASK=\"prepare-source\"", + "export HWLAB_SOURCE_REVISION=\"$(params.revision)\"", dependencyProxyProbeScript("prepare-source-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease"), curlDownloadProbeFunction(), `echo '{"event":"ci-base-image","phase":"prepare-source","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}'`, @@ -725,11 +747,14 @@ function prepareSourceScript() { "chmod 600 /root/.ssh/id_rsa", "ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null", "rm -rf /workspace/source/repo", + "source_clone_started_ms=\"$(ci_now_ms)\"", "git clone --branch \"$(params.source-branch)\" \"$(params.git-url)\" /workspace/source/repo", "cd /workspace/source/repo", "git config --global --add safe.directory /workspace/source/repo", "git checkout \"$(params.revision)\"", "test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"", + "ci_timing_emit source-clone succeeded \"$source_clone_started_ms\"", + "catalog_fetch_started_ms=\"$(ci_now_ms)\"", "if git ls-remote --exit-code --heads origin \"$(params.gitops-branch)\" >/dev/null 2>&1; then", " git fetch --depth 1 origin \"$(params.gitops-branch)\" >/dev/null 2>&1 || true", " if git cat-file -e FETCH_HEAD:deploy/artifact-catalog.dev.json 2>/dev/null; then", @@ -746,7 +771,10 @@ function prepareSourceScript() { "else", " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"seed\",\"reason\":\"gitops-branch-missing\"}'", "fi", + "ci_timing_emit catalog-fetch succeeded \"$catalog_fetch_started_ms\"", + "npm_ci_started_ms=\"$(ci_now_ms)\"", "npm ci --ignore-scripts", + "ci_timing_emit npm-ci succeeded \"$npm_ci_started_ms\"", `echo ${shellSingleQuote(`G14 prepare-source complete; validation task count=${primitiveValidationTasks.length}`)}` ]; return `${lines.join("\n")}\n`; @@ -816,6 +844,12 @@ NODE function perServicePublishScript() { return `#!/bin/sh set -eu +${ciTimingShellFunction()} +export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" +export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" +export HWLAB_TEKTON_TASK="build-$(params.service-id)" +export HWLAB_SOURCE_REVISION="$(params.revision)" +export HWLAB_TIMING_SERVICE_ID="$(params.service-id)" ${dependencyProxyProbeScript("service-image-publish-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} echo '{"event":"ci-base-image","phase":"service-image-publish","serviceId":"'"$(params.service-id)"'","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' for tool in node npm git python3 ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"service-image-publish","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done @@ -858,6 +892,11 @@ export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-$(params.service-id export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton" export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)" export HWLAB_DEV_REGISTRY_K8S_NATIVE=1 +export HWLAB_G14_CICD_TIMING=1 +export HWLAB_TEKTON_PIPELINERUN +export HWLAB_TEKTON_TASKRUN +export HWLAB_TEKTON_TASK +export HWLAB_SOURCE_REVISION export PATH="/workspace/buildkit-bin:$PATH" export HWLAB_BUILDKIT_ADDR="unix:///workspace/buildkit-run/buildkitd.sock" if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi @@ -905,6 +944,11 @@ node scripts/refresh-artifact-catalog.mjs --target-ref HEAD --publish-report /wo function gitopsPromoteScript() { return `#!/bin/sh set -eu +${ciTimingShellFunction()} +export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" +export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" +export HWLAB_TEKTON_TASK="gitops-promote" +export HWLAB_SOURCE_REVISION="$(params.revision)" ${dependencyProxyProbeScript("gitops-promote-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} echo '{"event":"ci-base-image","phase":"gitops-promote","image":"${ciToolsRunnerImage}","policy":"no-runtime-apt"}' for tool in node npm git ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done @@ -936,10 +980,13 @@ git config --global user.email "hwlab-g14-gitops-bot@users.noreply.github.com" if [ -s /workspace/source/dev-artifacts.json ]; then node scripts/refresh-artifact-catalog.mjs --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write fi +gitops_render_started_ms="$(ci_now_ms)" node scripts/g14-gitops-render.mjs --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --registry-prefix "$(params.registry-prefix)" --use-deploy-images node scripts/g14-gitops-render.mjs --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --registry-prefix "$(params.registry-prefix)" --use-deploy-images --check +ci_timing_emit gitops-render succeeded "$gitops_render_started_ms" check_source_head before-push workdir="$(mktemp -d)" +gitops_clone_started_ms="$(ci_now_ms)" git clone --no-checkout "$(params.git-url)" "$workdir/gitops" cd "$workdir/gitops" if git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null 2>&1; then @@ -950,17 +997,24 @@ else git checkout --orphan "$(params.gitops-branch)" git rm -rf . >/dev/null 2>&1 || true fi +ci_timing_emit gitops-clone succeeded "$gitops_clone_started_ms" mkdir -p deploy/gitops cp -a /workspace/source/repo/deploy/gitops/g14 deploy/gitops/g14 cp /workspace/source/repo/deploy/artifact-catalog.dev.json deploy/artifact-catalog.dev.json git add deploy/artifact-catalog.dev.json deploy/gitops/g14 if git diff --cached --quiet; then echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' + ci_timing_emit gitops-commit unchanged "$(ci_now_ms)" + ci_timing_emit gitops-push unchanged "$(ci_now_ms)" exit 0 fi short="$(printf '%.7s' "$(params.revision)")" +gitops_commit_started_ms="$(ci_now_ms)" git commit -m "chore: promote G14 GitOps source $short" +ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms" +gitops_push_started_ms="$(ci_now_ms)" git push origin "HEAD:$(params.gitops-branch)" +ci_timing_emit gitops-push succeeded "$gitops_push_started_ms" echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' `; } @@ -1453,6 +1507,180 @@ function collectArtifactsTask() { }; } +function runtimeReadyScript() { + return `#!/bin/sh +set -eu +${ciTimingShellFunction()} +export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" +export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" +export HWLAB_TEKTON_TASK="runtime-ready" +export HWLAB_SOURCE_REVISION="$(params.revision)" +node <<'NODE' +const fs = require("node:fs"); +const https = require("node:https"); + +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = process.env.KUBERNETES_SERVICE_PORT || "443"; +const revision = process.env.HWLAB_SOURCE_REVISION || ""; +const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null; +const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null; +const task = process.env.HWLAB_TEKTON_TASK || null; +const timeoutMs = 240000; +const pollMs = 5000; + +function emit(payload) { + console.log(JSON.stringify({ + event: "g14-cicd-timing", + schemaVersion: "v1", + pipelineRun, + taskRun, + task, + revision, + source: "scripts/g14-gitops-render.mjs", + at: new Date().toISOString(), + ...payload + })); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function safeJson(text) { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +if (!host) { + emit({ stage: "argo-refresh", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 }); + emit({ stage: "workload-ready", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 }); + process.exit(0); +} + +const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); +const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +function request(method, path, body, contentType = "application/json") { + const payload = body ? JSON.stringify(body) : ""; + const headers = { Authorization: "Bearer " + token }; + if (payload) { + headers["Content-Type"] = contentType; + headers["Content-Length"] = Buffer.byteLength(payload); + } + return new Promise((resolve, reject) => { + const req = https.request({ host, port, method, path, ca, headers }, (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { data += chunk; }); + res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) })); + }); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +function listPath(group, version, namespace, resource) { + if (group) return "/apis/" + group + "/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource; + return "/api/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource; +} + +async function runtimeItems() { + const [deployments, statefulsets] = await Promise.all([ + request("GET", listPath("apps", "v1", "hwlab-dev", "deployments")), + request("GET", listPath("apps", "v1", "hwlab-dev", "statefulsets")) + ]); + if (deployments.statusCode === 403 || statefulsets.statusCode === 403) return { status: "rbac-denied", items: [] }; + const items = [...(deployments.json?.items || []), ...(statefulsets.json?.items || [])] + .filter((item) => item.metadata?.labels?.["hwlab.pikastech.local/gitops-target"] === "g14"); + return { status: "ok", items }; +} + +function runtimeSourceCommit(item) { + return item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || null; +} + +function readyWorkload(item) { + const specReplicas = item.spec?.replicas ?? 1; + if (specReplicas === 0) return { ready: true, skipped: true }; + const status = item.status || {}; + const desired = specReplicas; + const ready = status.readyReplicas ?? 0; + const updated = status.updatedReplicas ?? 0; + const observedGeneration = status.observedGeneration ?? 0; + const generation = item.metadata?.generation ?? 0; + return { ready: observedGeneration >= generation && ready >= desired && updated >= desired, desired, readyReplicas: ready, updatedReplicas: updated }; +} + +async function waitForArgoRefresh() { + const startedAt = Date.now(); + for (;;) { + const runtime = await runtimeItems(); + if (runtime.status === "rbac-denied") { + emit({ stage: "argo-refresh", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); + return { observed: false, skipped: true }; + } + const pending = runtime.items + .map((item) => ({ kind: item.kind, name: item.metadata?.name, sourceCommit: runtimeSourceCommit(item) })) + .filter((item) => item.sourceCommit !== revision); + if (runtime.items.length > 0 && pending.length === 0) { + emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length }); + return { observed: true, skipped: false }; + } + if (Date.now() - startedAt >= timeoutMs) { + emit({ stage: "argo-refresh", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) }); + return { observed: false, skipped: false }; + } + await sleep(pollMs); + } +} + +async function waitForWorkloads() { + const startedAt = Date.now(); + for (;;) { + const runtime = await runtimeItems(); + if (runtime.status === "rbac-denied") { + emit({ stage: "workload-ready", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); + return; + } + const blocked = runtime.items.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready); + if (runtime.items.length > 0 && blocked.length === 0) { + emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length }); + return; + } + if (Date.now() - startedAt >= timeoutMs) { + emit({ stage: "workload-ready", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, blocked: blocked.slice(0, 8) }); + return; + } + await sleep(pollMs); + } +} + +(async () => { + await waitForArgoRefresh(); + await waitForWorkloads(); +})().catch((error) => { + emit({ stage: "runtime-ready", status: "failed", reason: error.message, durationMs: 0 }); +}); +NODE +`; +} + +function runtimeReadyTask() { + return { + name: "runtime-ready", + runAfter: ["gitops-promote"], + taskSpec: { + params: [{ name: "revision" }], + steps: [{ name: "runtime-ready", image: ciToolsRunnerImage, env: proxyEnv(), script: runtimeReadyScript() }] + }, + params: [{ name: "revision", value: "$(params.revision)" }] + }; +} + function imagePublishTaskSet() { return [ planArtifactsTask(), @@ -1543,7 +1771,8 @@ function tektonPipeline() { { name: "revision", value: "$(params.revision)" }, { name: "registry-prefix", value: "$(params.registry-prefix)" } ] - } + }, + runtimeReadyTask() ] } };