From 7e4ec0a22562e5c2bd542e0760fc1bcfad723955 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 3 Jul 2026 12:45:32 +0000 Subject: [PATCH] fix: restore cicd reuse catalog before planning --- scripts/ci/restore-artifact-catalog.mjs | 77 +++++++ scripts/ci/runtime-ready.mjs | 247 ++++++++++++++++++++ scripts/gitops-render.mjs | 286 ++---------------------- scripts/gitops-render.test.ts | 32 +-- 4 files changed, 362 insertions(+), 280 deletions(-) create mode 100644 scripts/ci/restore-artifact-catalog.mjs create mode 100644 scripts/ci/runtime-ready.mjs diff --git a/scripts/ci/restore-artifact-catalog.mjs b/scripts/ci/restore-artifact-catalog.mjs new file mode 100644 index 00000000..77d2791d --- /dev/null +++ b/scripts/ci/restore-artifact-catalog.mjs @@ -0,0 +1,77 @@ +#!/usr/bin/env node +// Restore the node/lane artifact catalog from the GitOps branch before CI planning. + +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; + +const startedAt = Date.now(); + +function env(name) { + const value = process.env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function emit(payload) { + process.stderr.write(JSON.stringify({ + event: "artifact-catalog-restore", + source: "scripts/ci/restore-artifact-catalog.mjs", + at: new Date().toISOString(), + durationMs: Date.now() - startedAt, + catalogPath, + gitopsBranch, + ...payload + }) + "\n"); +} + +function catalogHasServices(filePath) { + if (!existsSync(filePath)) return false; + try { + const parsed = JSON.parse(readFileSync(filePath, "utf8")); + return Array.isArray(parsed?.services) && parsed.services.length > 0; + } catch { + return false; + } +} + +function git(args, options = {}) { + return execFileSync("git", args, { + cwd: process.cwd(), + encoding: "utf8", + stdio: options.capture === false ? ["ignore", "ignore", "pipe"] : ["ignore", "pipe", "pipe"], + timeout: options.timeoutMs ?? 45_000 + }); +} + +const catalogPath = env("HWLAB_CATALOG_PATH"); +const gitopsBranch = env("HWLAB_GITOPS_BRANCH"); +const remote = env("HWLAB_GIT_READ_URL") ?? env("HWLAB_GIT_URL"); + +if (!catalogPath || !gitopsBranch || !remote) { + emit({ status: "skipped", reason: "required-input-missing", hasCatalogPath: Boolean(catalogPath), hasGitopsBranch: Boolean(gitopsBranch), hasRemote: Boolean(remote) }); + process.exit(0); +} + +if (catalogHasServices(catalogPath)) { + emit({ status: "source-present", reason: "catalog-already-present" }); + process.exit(0); +} + +try { + git(["fetch", "--depth=1", remote, `refs/heads/${gitopsBranch}`], { capture: false, timeoutMs: 60_000 }); + const content = git(["show", `FETCH_HEAD:${catalogPath}`], { timeoutMs: 30_000 }); + const parsed = JSON.parse(content); + if (!Array.isArray(parsed?.services) || parsed.services.length === 0) { + emit({ status: "skipped", reason: "gitops-catalog-has-no-services" }); + process.exit(0); + } + mkdirSync(dirname(catalogPath), { recursive: true }); + writeFileSync(catalogPath, JSON.stringify(parsed, null, 2) + "\n"); + emit({ status: "restored", reason: "gitops-catalog", serviceCount: parsed.services.length, bytes: Buffer.byteLength(content) }); +} catch (error) { + emit({ + status: "skipped", + reason: "gitops-catalog-unavailable", + error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500) + }); +} diff --git a/scripts/ci/runtime-ready.mjs b/scripts/ci/runtime-ready.mjs new file mode 100644 index 00000000..486c0e1a --- /dev/null +++ b/scripts/ci/runtime-ready.mjs @@ -0,0 +1,247 @@ +#!/usr/bin/env node +// Wait for Argo/runtime convergence with a single wall-clock budget. + +import { readFileSync } from "node:fs"; +import https from "node:https"; + +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = process.env.KUBERNETES_SERVICE_PORT || "443"; +const startedAt = Date.now(); +const timeoutMs = positiveInteger(process.env.HWLAB_RUNTIME_READY_TIMEOUT_MS, 60_000); +const pollMs = positiveInteger(process.env.HWLAB_RUNTIME_READY_POLL_MS, 1_000); +const progressEveryMs = positiveInteger(process.env.HWLAB_RUNTIME_READY_PROGRESS_MS, 10_000); +const deadline = startedAt + timeoutMs; + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(name + " is required"); + return value; +} + +const revision = requiredEnv("HWLAB_SOURCE_REVISION"); +const runtimeNamespace = requiredEnv("HWLAB_RUNTIME_NAMESPACE"); +const gitopsTarget = requiredEnv("HWLAB_GITOPS_TARGET"); +const argoApplication = requiredEnv("HWLAB_ARGO_APPLICATION"); +const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null; +const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null; +const task = process.env.HWLAB_TEKTON_TASK || null; + +function positiveInteger(value, fallback) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function emit(payload) { + console.log(JSON.stringify({ + event: "node-cicd-timing", + schemaVersion: "v1", + pipelineRun, + taskRun, + task, + revision, + source: "scripts/ci/runtime-ready.mjs", + at: new Date().toISOString(), + elapsedMs: Date.now() - startedAt, + remainingMs: Math.max(0, deadline - Date.now()), + ...payload + })); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function safeJson(text) { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +if (!host) { + emit({ stage: "argo-refresh", status: "failed", reason: "kubernetes-service-host-missing", durationMs: 0 }); + process.exit(1); +} + +const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); +const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +function request(method, path, body, contentType = "application/json") { + const payload = body ? JSON.stringify(body) : ""; + const headers = { Authorization: "Bearer " + token }; + if (payload) { + headers["Content-Type"] = contentType; + headers["Content-Length"] = Buffer.byteLength(payload); + } + return new Promise((resolve, reject) => { + const req = https.request({ host, port, method, path, ca, headers }, (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { data += chunk; }); + res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) })); + }); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +function listPath(group, version, namespace, resource) { + if (group) return "/apis/" + group + "/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource; + return "/api/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource; +} + +async function runtimeItems() { + const [deployments, statefulsets] = await Promise.all([ + request("GET", listPath("apps", "v1", runtimeNamespace, "deployments")), + request("GET", listPath("apps", "v1", runtimeNamespace, "statefulsets")) + ]); + if (deployments.statusCode === 403 || statefulsets.statusCode === 403) return { status: "rbac-denied", items: [] }; + const items = [...(deployments.json?.items || []), ...(statefulsets.json?.items || [])] + .filter((item) => item.metadata?.labels?.["hwlab.pikastech.local/gitops-target"] === gitopsTarget); + return { status: "ok", items }; +} + +function runtimeSourceCommit(item) { + return item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] || + item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] || + item.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] || + item.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] || + item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || + item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || + item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || + item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || + null; +} + +function serviceIdForRuntimeItem(item) { + return item.metadata?.labels?.["hwlab.pikastech.local/service-id"] || + item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || + item.metadata?.labels?.["app.kubernetes.io/name"] || + item.metadata?.name || + null; +} + +function readObservedServiceIds() { + try { + const plan = JSON.parse(readFileSync("/workspace/source/affected-services.json", "utf8")); + const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices.filter(Boolean) : []; + if (rolloutServices.length === 0) return null; + return new Set(rolloutServices); + } catch { + return null; + } +} + +const observedServiceIds = readObservedServiceIds(); +const readinessMode = observedServiceIds ? "service-rollout" : "infra-runtime-change"; + +function observedRuntimeItems(items) { + if (!observedServiceIds) return items; + return items.filter((item) => observedServiceIds.has(serviceIdForRuntimeItem(item))); +} + +function readyWorkload(item) { + const specReplicas = item.spec?.replicas ?? 1; + if (specReplicas === 0) return { ready: true, skipped: true }; + const status = item.status || {}; + const desired = specReplicas; + const ready = status.readyReplicas ?? 0; + const updated = status.updatedReplicas ?? 0; + const observedGeneration = status.observedGeneration ?? 0; + const generation = item.metadata?.generation ?? 0; + return { ready: observedGeneration >= generation && ready >= desired && updated >= desired, desired, readyReplicas: ready, updatedReplicas: updated }; +} + +function maybeEmitProgress(state, stage, stageStartedAt, payload) { + const now = Date.now(); + if (now - state.lastProgressAt < progressEveryMs) return; + state.lastProgressAt = now; + emit({ stage, status: "progress", durationMs: now - stageStartedAt, readinessMode, ...payload }); +} + +async function argoApplicationState() { + const path = "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(argoApplication); + const response = await request("GET", path); + if (response.statusCode === 403) return { status: "rbac-denied", item: null, statusCode: response.statusCode }; + if (response.statusCode < 200 || response.statusCode > 299) return { status: "error", item: null, statusCode: response.statusCode, body: response.body.slice(0, 500) }; + return { status: "ok", item: response.json, statusCode: response.statusCode }; +} + +async function waitForArgoSyncedHealthy() { + const stageStartedAt = Date.now(); + const progressState = { lastProgressAt: 0 }; + while (Date.now() < deadline) { + const application = await argoApplicationState(); + if (application.status === "rbac-denied") return fail("argo-sync-health", "argocd-application-rbac-denied", stageStartedAt, { readinessMode }); + if (application.status !== "ok") return fail("argo-sync-health", "argocd-application-read-failed", stageStartedAt, { readinessMode, statusCode: application.statusCode, body: application.body || null }); + const syncStatus = application.item?.status?.sync?.status || null; + const healthStatus = application.item?.status?.health?.status || null; + const operationPhase = application.item?.status?.operationState?.phase || null; + const operationMessage = application.item?.status?.operationState?.message || null; + if (syncStatus === "Synced" && healthStatus === "Healthy") { + emit({ stage: "argo-sync-health", status: "succeeded", durationMs: Date.now() - stageStartedAt, readinessMode, syncStatus, healthStatus, operationPhase }); + return true; + } + maybeEmitProgress(progressState, "argo-sync-health", stageStartedAt, { syncStatus, healthStatus, operationPhase, operationMessage }); + await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now()))); + } + return fail("argo-sync-health", "runtime-ready-wall-clock-timeout", stageStartedAt, { readinessMode }, "timeout"); +} + +async function waitForArgoRefresh() { + const stageStartedAt = Date.now(); + const progressState = { lastProgressAt: 0 }; + while (Date.now() < deadline) { + const runtime = await runtimeItems(); + if (runtime.status === "rbac-denied") return fail("argo-refresh", "runtime-observer-rbac-denied", stageStartedAt, { readinessMode }); + const observed = observedRuntimeItems(runtime.items); + const pending = observed + .map((item) => ({ kind: item.kind, name: item.metadata?.name, sourceCommit: runtimeSourceCommit(item) })) + .filter((item) => item.sourceCommit !== revision); + if (observed.length > 0 && pending.length === 0) { + emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - stageStartedAt, readinessMode, workloadCount: runtime.items.length, observedCount: observed.length }); + return true; + } + maybeEmitProgress(progressState, "argo-refresh", stageStartedAt, { workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) }); + await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now()))); + } + return fail("argo-refresh", "source-commit-refresh-timeout", stageStartedAt, { readinessMode }, "timeout"); +} + +async function waitForWorkloads() { + const stageStartedAt = Date.now(); + const progressState = { lastProgressAt: 0 }; + while (Date.now() < deadline) { + const runtime = await runtimeItems(); + if (runtime.status === "rbac-denied") return fail("workload-ready", "runtime-observer-rbac-denied", stageStartedAt, { readinessMode }); + const observed = observedRuntimeItems(runtime.items); + const blocked = observed.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready); + if (observed.length > 0 && blocked.length === 0) { + emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - stageStartedAt, readinessMode, workloadCount: runtime.items.length, observedCount: observed.length }); + return true; + } + maybeEmitProgress(progressState, "workload-ready", stageStartedAt, { workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) }); + await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now()))); + } + return fail("workload-ready", "runtime-ready-wall-clock-timeout", stageStartedAt, { readinessMode }, "timeout"); +} + +function fail(stage, reason, stageStartedAt, extra = {}, status = "failed") { + emit({ stage, status, reason, durationMs: Date.now() - stageStartedAt, ...extra }); + process.exitCode = 1; + return false; +} + +emit({ stage: "runtime-ready", status: "started", readinessMode, observedServiceCount: observedServiceIds ? observedServiceIds.size : 0, timeoutMs }); +try { + if (!observedServiceIds) { + if (await waitForArgoSyncedHealthy()) await waitForWorkloads(); + } else if (await waitForArgoRefresh()) { + await waitForWorkloads(); + } +} catch (error) { + emit({ stage: "runtime-ready", status: "failed", reason: error instanceof Error ? error.message : String(error), durationMs: Date.now() - startedAt }); + process.exitCode = 1; +} diff --git a/scripts/gitops-render.mjs b/scripts/gitops-render.mjs index e4f7ffeb..6ebddd05 100644 --- a/scripts/gitops-render.mjs +++ b/scripts/gitops-render.mjs @@ -2061,6 +2061,11 @@ echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"${ciToolsRunner for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done cd /workspace/source/repo test "$(git rev-parse HEAD)" = "$(params.revision)" +HWLAB_CATALOG_PATH="$(params.catalog-path)" \ +HWLAB_GIT_URL="$(params.git-url)" \ +HWLAB_GIT_READ_URL="$(params.git-read-url)" \ +HWLAB_GITOPS_BRANCH="$(params.gitops-branch)" \ +node scripts/ci/restore-artifact-catalog.mjs node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" --verify-reuse-registry > /workspace/source/ci-plan.json node - <<'NODE' const fs = require("node:fs"); @@ -3096,6 +3101,10 @@ function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) { workspaces: [{ name: "source", workspace: "source" }], taskSpec: { params: [ + { name: "git-url" }, + { name: "git-read-url" }, + { name: "gitops-branch" }, + { name: "lane" }, { name: "revision" }, { name: "catalog-path" }, { name: "registry-prefix" }, @@ -3116,6 +3125,10 @@ function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) { }] }, params: [ + { name: "git-url", value: "$(params.git-url)" }, + { name: "git-read-url", value: "$(params.git-read-url)" }, + { name: "gitops-branch", value: "$(params.gitops-branch)" }, + { name: "lane", value: "$(params.lane)" }, { name: "revision", value: "$(params.revision)" }, { name: "catalog-path", value: "$(params.catalog-path)" }, { name: "registry-prefix", value: "$(params.registry-prefix)" }, @@ -3208,268 +3221,9 @@ export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" export HWLAB_TEKTON_TASK="runtime-ready" export HWLAB_SOURCE_REVISION="$(params.revision)" -node <<'NODE' -const fs = require("node:fs"); -const https = require("node:https"); - -const host = process.env.KUBERNETES_SERVICE_HOST; -const port = process.env.KUBERNETES_SERVICE_PORT || "443"; -function requiredEnv(name) { - const value = process.env[name]; - if (!value) throw new Error(name + " is required"); - return value; -} - -const revision = requiredEnv("HWLAB_SOURCE_REVISION"); -const runtimeNamespace = requiredEnv("HWLAB_RUNTIME_NAMESPACE"); -const gitopsTarget = requiredEnv("HWLAB_GITOPS_TARGET"); -const argoApplication = requiredEnv("HWLAB_ARGO_APPLICATION"); -const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null; -const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null; -const task = process.env.HWLAB_TEKTON_TASK || null; -const timeoutMs = 240000; -const pollMs = 1000; -const progressEveryMs = 10000; - -function emit(payload) { - console.log(JSON.stringify({ - event: "node-cicd-timing", - schemaVersion: "v1", - pipelineRun, - taskRun, - task, - revision, - source: "scripts/gitops-render.mjs", - at: new Date().toISOString(), - ...payload - })); -} - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function safeJson(text) { - try { - return JSON.parse(text); - } catch { - return null; - } -} - -if (!host) { - emit({ stage: "argo-refresh", status: "failed", reason: "kubernetes-service-host-missing", durationMs: 0 }); - process.exit(1); -} - -const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); -const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); - -function request(method, path, body, contentType = "application/json") { - const payload = body ? JSON.stringify(body) : ""; - const headers = { Authorization: "Bearer " + token }; - if (payload) { - headers["Content-Type"] = contentType; - headers["Content-Length"] = Buffer.byteLength(payload); - } - return new Promise((resolve, reject) => { - const req = https.request({ host, port, method, path, ca, headers }, (res) => { - let data = ""; - res.setEncoding("utf8"); - res.on("data", (chunk) => { data += chunk; }); - res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) })); - }); - req.on("error", reject); - if (payload) req.write(payload); - req.end(); - }); -} - -function listPath(group, version, namespace, resource) { - if (group) return "/apis/" + group + "/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource; - return "/api/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource; -} - -async function runtimeItems() { - const [deployments, statefulsets] = await Promise.all([ - request("GET", listPath("apps", "v1", runtimeNamespace, "deployments")), - request("GET", listPath("apps", "v1", runtimeNamespace, "statefulsets")) - ]); - if (deployments.statusCode === 403 || statefulsets.statusCode === 403) return { status: "rbac-denied", items: [] }; - const items = [...(deployments.json?.items || []), ...(statefulsets.json?.items || [])] - .filter((item) => item.metadata?.labels?.["hwlab.pikastech.local/gitops-target"] === gitopsTarget); - return { status: "ok", items }; -} - -function runtimeSourceCommit(item) { - return item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] || - item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] || - item.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] || - item.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] || - item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || - item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || - item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || - item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || - null; -} - -function serviceIdForRuntimeItem(item) { - return item.metadata?.labels?.["hwlab.pikastech.local/service-id"] || - item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || - item.metadata?.labels?.["app.kubernetes.io/name"] || - item.metadata?.name || - null; -} - -function readObservedServiceIds() { - try { - const plan = JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")); - const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices.filter(Boolean) : []; - if (rolloutServices.length === 0) return null; - return new Set(rolloutServices); - } catch { - return null; - } -} - -const observedServiceIds = readObservedServiceIds(); -const readinessMode = observedServiceIds ? "service-rollout" : "infra-runtime-change"; - -function observedRuntimeItems(items) { - if (!observedServiceIds) return items; - return items.filter((item) => observedServiceIds.has(serviceIdForRuntimeItem(item))); -} - -function readyWorkload(item) { - const specReplicas = item.spec?.replicas ?? 1; - if (specReplicas === 0) return { ready: true, skipped: true }; - const status = item.status || {}; - const desired = specReplicas; - const ready = status.readyReplicas ?? 0; - const updated = status.updatedReplicas ?? 0; - const observedGeneration = status.observedGeneration ?? 0; - const generation = item.metadata?.generation ?? 0; - return { ready: observedGeneration >= generation && ready >= desired && updated >= desired, desired, readyReplicas: ready, updatedReplicas: updated }; -} - -function maybeEmitProgress(state, stage, startedAt, payload) { - const now = Date.now(); - if (now - state.lastProgressAt < progressEveryMs) return; - state.lastProgressAt = now; - emit({ stage, status: "progress", durationMs: now - startedAt, readinessMode, ...payload }); -} - -async function argoApplicationState() { - const path = "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(argoApplication); - const response = await request("GET", path); - if (response.statusCode === 403) return { status: "rbac-denied", item: null, statusCode: response.statusCode }; - if (response.statusCode < 200 || response.statusCode > 299) return { status: "error", item: null, statusCode: response.statusCode, body: response.body.slice(0, 500) }; - return { status: "ok", item: response.json, statusCode: response.statusCode }; -} - -async function waitForArgoSyncedHealthy() { - const startedAt = Date.now(); - const progressState = { lastProgressAt: 0 }; - for (;;) { - const application = await argoApplicationState(); - if (application.status === "rbac-denied") { - emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-rbac-denied", durationMs: Date.now() - startedAt, readinessMode }); - process.exitCode = 1; - return { observed: false }; - } - if (application.status !== "ok") { - emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-read-failed", statusCode: application.statusCode, body: application.body || null, durationMs: Date.now() - startedAt, readinessMode }); - process.exitCode = 1; - return { observed: false }; - } - const syncStatus = application.item?.status?.sync?.status || null; - const healthStatus = application.item?.status?.health?.status || null; - const operationPhase = application.item?.status?.operationState?.phase || null; - const operationMessage = application.item?.status?.operationState?.message || null; - if (syncStatus === "Synced" && healthStatus === "Healthy") { - emit({ stage: "argo-sync-health", status: "succeeded", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase }); - return { observed: true }; - } - if (Date.now() - startedAt >= timeoutMs) { - emit({ stage: "argo-sync-health", status: "timeout", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase, operationMessage }); - process.exitCode = 1; - return { observed: false }; - } - maybeEmitProgress(progressState, "argo-sync-health", startedAt, { syncStatus, healthStatus, operationPhase, operationMessage }); - await sleep(pollMs); - } -} - -async function waitForArgoRefresh() { - const startedAt = Date.now(); - const progressState = { lastProgressAt: 0 }; - for (;;) { - const runtime = await runtimeItems(); - if (runtime.status === "rbac-denied") { - emit({ stage: "argo-refresh", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); - process.exitCode = 1; - return { observed: false, skipped: true }; - } - const observed = observedRuntimeItems(runtime.items); - const pending = observed - .map((item) => ({ kind: item.kind, name: item.metadata?.name, sourceCommit: runtimeSourceCommit(item) })) - .filter((item) => item.sourceCommit !== revision); - if (observed.length > 0 && pending.length === 0) { - emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length }); - return { observed: true, skipped: false }; - } - if (Date.now() - startedAt >= timeoutMs) { - emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) }); - process.exitCode = 1; - return { observed: false, skipped: false }; - } - maybeEmitProgress(progressState, "argo-refresh", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) }); - await sleep(pollMs); - } -} - -async function waitForWorkloads() { - const startedAt = Date.now(); - const progressState = { lastProgressAt: 0 }; - for (;;) { - const runtime = await runtimeItems(); - if (runtime.status === "rbac-denied") { - emit({ stage: "workload-ready", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); - process.exitCode = 1; - return; - } - const observed = observedRuntimeItems(runtime.items); - const blocked = observed.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready); - if (observed.length > 0 && blocked.length === 0) { - emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length }); - return; - } - if (Date.now() - startedAt >= timeoutMs) { - emit({ stage: "workload-ready", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) }); - process.exitCode = 1; - return; - } - maybeEmitProgress(progressState, "workload-ready", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) }); - await sleep(pollMs); - } -} - -(async () => { - emit({ stage: "runtime-ready", status: "started", readinessMode, observedServiceCount: observedServiceIds ? observedServiceIds.size : 0 }); - if (!observedServiceIds) { - const argo = await waitForArgoSyncedHealthy(); - if (!argo.observed) return; - await waitForWorkloads(); - return; - } - const refresh = await waitForArgoRefresh(); - if (!refresh.observed) return; - await waitForWorkloads(); -})().catch((error) => { - emit({ stage: "runtime-ready", status: "failed", reason: error.message, durationMs: 0 }); - process.exitCode = 1; -}); -NODE +export HWLAB_RUNTIME_READY_TIMEOUT_MS="$(params.timeout-ms)" +cd /workspace/source/repo +node scripts/ci/runtime-ready.mjs `; } @@ -3479,7 +3233,7 @@ function runtimeReadyTask({ profile = "dev" } = {}) { runAfter: ["gitops-promote"], workspaces: [{ name: "source", workspace: "source" }], taskSpec: { - params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }], + params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }, { name: "timeout-ms" }], workspaces: [{ name: "source" }], steps: [{ name: "runtime-ready", @@ -3497,7 +3251,8 @@ function runtimeReadyTask({ profile = "dev" } = {}) { { name: "revision", value: "$(params.revision)" }, { name: "runtime-namespace", value: namespaceNameForProfile(profile) }, { name: "gitops-target", value: gitopsTargetForProfile(profile) }, - { name: "argo-application", value: argoApplicationName(profile) } + { name: "argo-application", value: argoApplicationName(profile) }, + { name: "timeout-ms", value: "$(params.runtime-ready-timeout-ms)" } ] }; } @@ -3548,7 +3303,8 @@ function tektonPipeline(args = { lane: "node" }, deploy = null) { { name: "registry-prefix", type: "string", default: defaultRegistryPrefix }, { name: "services", type: "string", default: servicesParamForLane(args.lane, deploy) }, { name: "base-image", type: "string", default: defaultDevBaseImage }, - { name: "build-cache-mode", type: "string", default: "registry" } + { name: "build-cache-mode", type: "string", default: "registry" }, + { name: "runtime-ready-timeout-ms", type: "string", default: "60000" } ], workspaces: [ { name: "source" }, diff --git a/scripts/gitops-render.test.ts b/scripts/gitops-render.test.ts index 36837031..9cfdb1a5 100644 --- a/scripts/gitops-render.test.ts +++ b/scripts/gitops-render.test.ts @@ -165,29 +165,32 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.match(pipeline, /tar -C \/workspace\/source\/repo -cf - \. \| tar -C \/workspace\/service-work\/repo -xo -f -/u); assert.match(pipeline, /node scripts\/run-bun\.mjs test cmd\/hwlab-codex-api-responses-forwarder\/main\.test\.ts/u); assert.doesNotMatch(pipeline, /cmd\/hwlab-codex-api-responses-forwarder\/main\.mjs/u); - assert.match(pipeline, /process\.exitCode = 1/u); - assert.match(pipeline, /item\.metadata\?\.labels\?\.\[\\"hwlab\.pikastech\.local\/source-commit\\"\]/u); + const runtimeReadySource = await readFile(path.join(process.cwd(), "scripts/ci/runtime-ready.mjs"), "utf8"); + assert.match(runtimeReadySource, /process\.exitCode = 1/u); + assert.match(runtimeReadySource, /item\.metadata\?\.labels\?\.\["hwlab\.pikastech\.local\/source-commit"\]/u); const runtimeReady = taskByName(pipelineJson, "runtime-ready"); assert.deepEqual(runtimeReady.when, [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }]); const runtimeReadyScript = runtimeReady.taskSpec.steps[0].script; - assert.match(runtimeReadyScript, /const pollMs = 1000;/u); - assert.match(runtimeReadyScript, /readObservedServiceIds/u); - assert.match(runtimeReadyScript, /observedRuntimeItems/u); - assert.match(runtimeReadyScript, /observedCount/u); - assert.match(runtimeReadyScript, /readinessMode = observedServiceIds \? "service-rollout" : "infra-runtime-change"/u); - assert.match(runtimeReadyScript, /waitForArgoSyncedHealthy/u); - assert.match(runtimeReadyScript, /stage: "runtime-ready", status: "started"/u); - assert.match(runtimeReadyScript, /status: "progress"/u); - assert.match(runtimeReadyScript, /stage: "argo-sync-health"/u); - assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u); - assert.match(runtimeReadyScript, /if \(!observedServiceIds\) \{/u); - assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u); + assert.match(runtimeReadyScript, /HWLAB_RUNTIME_READY_TIMEOUT_MS="\$\(params\.timeout-ms\)"/u); + assert.match(runtimeReadyScript, /cd \/workspace\/source\/repo/u); + assert.match(runtimeReadyScript, /node scripts\/ci\/runtime-ready\.mjs/u); assert.ok(runtimeReady.taskSpec.params.some((param) => param.name === "argo-application")); + assert.ok(runtimeReady.taskSpec.params.some((param) => param.name === "timeout-ms")); assert.ok(runtimeReady.params.some((param) => param.name === "argo-application" && param.value === "hwlab-node-v02")); + assert.ok(runtimeReady.params.some((param) => param.name === "timeout-ms" && param.value === "$(params.runtime-ready-timeout-ms)")); assert.ok(runtimeReady.workspaces?.some((workspace) => workspace.name === "source" && workspace.workspace === "source")); assert.ok(runtimeReady.taskSpec.workspaces?.some((workspace) => workspace.name === "source")); + const planArtifacts = taskByName(pipelineJson, "plan-artifacts"); + const planArtifactsScript = planArtifacts.taskSpec.steps[0].script; + assert.match(planArtifactsScript, /node scripts\/ci\/restore-artifact-catalog\.mjs/u); + assert.match(planArtifactsScript, /HWLAB_GIT_READ_URL="\$\(params\.git-read-url\)"/u); + assert.ok(planArtifacts.taskSpec.params.some((param) => param.name === "git-read-url")); + assert.ok(planArtifacts.taskSpec.params.some((param) => param.name === "gitops-branch")); + assert.ok(planArtifacts.params.some((param) => param.name === "lane" && param.value === "$(params.lane)")); + assert.ok(pipelineJson.spec.params.some((param) => param.name === "runtime-ready-timeout-ms" && param.default === "60000")); + const gitopsPromote = taskByName(pipelineJson, "gitops-promote"); assert.ok(gitopsPromote.taskSpec.results.some((result) => result.name === "runtime-ready-required")); const gitopsPromoteScript = gitopsPromote.taskSpec.steps[0].script; @@ -238,7 +241,6 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.doesNotMatch(prepareSourceScript, /prepare-source-proxy-preflight/u); assert.doesNotMatch(prepareSourceScript, /dependency-curl-probe-start/u); - const planArtifacts = taskByName(pipelineJson, "plan-artifacts"); assert.doesNotMatch(planArtifacts.taskSpec.steps[0].script, /plan-artifacts-proxy-preflight/u); const collectArtifacts = taskByName(pipelineJson, "collect-artifacts"); assert.doesNotMatch(collectArtifacts.taskSpec.steps[0].script, /collect-artifacts-proxy-preflight/u);