fix: restore cicd reuse catalog before planning
This commit is contained in:
root
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env node
|
||||
// Restore the node/lane artifact catalog from the GitOps branch before CI planning.
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
const startedAt = Date.now();
|
||||
|
||||
function env(name) {
|
||||
const value = process.env[name];
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function emit(payload) {
|
||||
process.stderr.write(JSON.stringify({
|
||||
event: "artifact-catalog-restore",
|
||||
source: "scripts/ci/restore-artifact-catalog.mjs",
|
||||
at: new Date().toISOString(),
|
||||
durationMs: Date.now() - startedAt,
|
||||
catalogPath,
|
||||
gitopsBranch,
|
||||
...payload
|
||||
}) + "\n");
|
||||
}
|
||||
|
||||
function catalogHasServices(filePath) {
|
||||
if (!existsSync(filePath)) return false;
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(filePath, "utf8"));
|
||||
return Array.isArray(parsed?.services) && parsed.services.length > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function git(args, options = {}) {
|
||||
return execFileSync("git", args, {
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
stdio: options.capture === false ? ["ignore", "ignore", "pipe"] : ["ignore", "pipe", "pipe"],
|
||||
timeout: options.timeoutMs ?? 45_000
|
||||
});
|
||||
}
|
||||
|
||||
const catalogPath = env("HWLAB_CATALOG_PATH");
|
||||
const gitopsBranch = env("HWLAB_GITOPS_BRANCH");
|
||||
const remote = env("HWLAB_GIT_READ_URL") ?? env("HWLAB_GIT_URL");
|
||||
|
||||
if (!catalogPath || !gitopsBranch || !remote) {
|
||||
emit({ status: "skipped", reason: "required-input-missing", hasCatalogPath: Boolean(catalogPath), hasGitopsBranch: Boolean(gitopsBranch), hasRemote: Boolean(remote) });
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (catalogHasServices(catalogPath)) {
|
||||
emit({ status: "source-present", reason: "catalog-already-present" });
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
try {
|
||||
git(["fetch", "--depth=1", remote, `refs/heads/${gitopsBranch}`], { capture: false, timeoutMs: 60_000 });
|
||||
const content = git(["show", `FETCH_HEAD:${catalogPath}`], { timeoutMs: 30_000 });
|
||||
const parsed = JSON.parse(content);
|
||||
if (!Array.isArray(parsed?.services) || parsed.services.length === 0) {
|
||||
emit({ status: "skipped", reason: "gitops-catalog-has-no-services" });
|
||||
process.exit(0);
|
||||
}
|
||||
mkdirSync(dirname(catalogPath), { recursive: true });
|
||||
writeFileSync(catalogPath, JSON.stringify(parsed, null, 2) + "\n");
|
||||
emit({ status: "restored", reason: "gitops-catalog", serviceCount: parsed.services.length, bytes: Buffer.byteLength(content) });
|
||||
} catch (error) {
|
||||
emit({
|
||||
status: "skipped",
|
||||
reason: "gitops-catalog-unavailable",
|
||||
error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500)
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env node
|
||||
// Wait for Argo/runtime convergence with a single wall-clock budget.
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import https from "node:https";
|
||||
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = positiveInteger(process.env.HWLAB_RUNTIME_READY_TIMEOUT_MS, 60_000);
|
||||
const pollMs = positiveInteger(process.env.HWLAB_RUNTIME_READY_POLL_MS, 1_000);
|
||||
const progressEveryMs = positiveInteger(process.env.HWLAB_RUNTIME_READY_PROGRESS_MS, 10_000);
|
||||
const deadline = startedAt + timeoutMs;
|
||||
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(name + " is required");
|
||||
return value;
|
||||
}
|
||||
|
||||
const revision = requiredEnv("HWLAB_SOURCE_REVISION");
|
||||
const runtimeNamespace = requiredEnv("HWLAB_RUNTIME_NAMESPACE");
|
||||
const gitopsTarget = requiredEnv("HWLAB_GITOPS_TARGET");
|
||||
const argoApplication = requiredEnv("HWLAB_ARGO_APPLICATION");
|
||||
const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null;
|
||||
const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null;
|
||||
const task = process.env.HWLAB_TEKTON_TASK || null;
|
||||
|
||||
function positiveInteger(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function emit(payload) {
|
||||
console.log(JSON.stringify({
|
||||
event: "node-cicd-timing",
|
||||
schemaVersion: "v1",
|
||||
pipelineRun,
|
||||
taskRun,
|
||||
task,
|
||||
revision,
|
||||
source: "scripts/ci/runtime-ready.mjs",
|
||||
at: new Date().toISOString(),
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
remainingMs: Math.max(0, deadline - Date.now()),
|
||||
...payload
|
||||
}));
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function safeJson(text) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!host) {
|
||||
emit({ stage: "argo-refresh", status: "failed", reason: "kubernetes-service-host-missing", durationMs: 0 });
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
|
||||
function request(method, path, body, contentType = "application/json") {
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = contentType;
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function listPath(group, version, namespace, resource) {
|
||||
if (group) return "/apis/" + group + "/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource;
|
||||
return "/api/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource;
|
||||
}
|
||||
|
||||
async function runtimeItems() {
|
||||
const [deployments, statefulsets] = await Promise.all([
|
||||
request("GET", listPath("apps", "v1", runtimeNamespace, "deployments")),
|
||||
request("GET", listPath("apps", "v1", runtimeNamespace, "statefulsets"))
|
||||
]);
|
||||
if (deployments.statusCode === 403 || statefulsets.statusCode === 403) return { status: "rbac-denied", items: [] };
|
||||
const items = [...(deployments.json?.items || []), ...(statefulsets.json?.items || [])]
|
||||
.filter((item) => item.metadata?.labels?.["hwlab.pikastech.local/gitops-target"] === gitopsTarget);
|
||||
return { status: "ok", items };
|
||||
}
|
||||
|
||||
function runtimeSourceCommit(item) {
|
||||
return item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
|
||||
item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] ||
|
||||
item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
|
||||
item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] ||
|
||||
null;
|
||||
}
|
||||
|
||||
function serviceIdForRuntimeItem(item) {
|
||||
return item.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
||||
item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
||||
item.metadata?.labels?.["app.kubernetes.io/name"] ||
|
||||
item.metadata?.name ||
|
||||
null;
|
||||
}
|
||||
|
||||
function readObservedServiceIds() {
|
||||
try {
|
||||
const plan = JSON.parse(readFileSync("/workspace/source/affected-services.json", "utf8"));
|
||||
const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices.filter(Boolean) : [];
|
||||
if (rolloutServices.length === 0) return null;
|
||||
return new Set(rolloutServices);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const observedServiceIds = readObservedServiceIds();
|
||||
const readinessMode = observedServiceIds ? "service-rollout" : "infra-runtime-change";
|
||||
|
||||
function observedRuntimeItems(items) {
|
||||
if (!observedServiceIds) return items;
|
||||
return items.filter((item) => observedServiceIds.has(serviceIdForRuntimeItem(item)));
|
||||
}
|
||||
|
||||
function readyWorkload(item) {
|
||||
const specReplicas = item.spec?.replicas ?? 1;
|
||||
if (specReplicas === 0) return { ready: true, skipped: true };
|
||||
const status = item.status || {};
|
||||
const desired = specReplicas;
|
||||
const ready = status.readyReplicas ?? 0;
|
||||
const updated = status.updatedReplicas ?? 0;
|
||||
const observedGeneration = status.observedGeneration ?? 0;
|
||||
const generation = item.metadata?.generation ?? 0;
|
||||
return { ready: observedGeneration >= generation && ready >= desired && updated >= desired, desired, readyReplicas: ready, updatedReplicas: updated };
|
||||
}
|
||||
|
||||
function maybeEmitProgress(state, stage, stageStartedAt, payload) {
|
||||
const now = Date.now();
|
||||
if (now - state.lastProgressAt < progressEveryMs) return;
|
||||
state.lastProgressAt = now;
|
||||
emit({ stage, status: "progress", durationMs: now - stageStartedAt, readinessMode, ...payload });
|
||||
}
|
||||
|
||||
async function argoApplicationState() {
|
||||
const path = "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(argoApplication);
|
||||
const response = await request("GET", path);
|
||||
if (response.statusCode === 403) return { status: "rbac-denied", item: null, statusCode: response.statusCode };
|
||||
if (response.statusCode < 200 || response.statusCode > 299) return { status: "error", item: null, statusCode: response.statusCode, body: response.body.slice(0, 500) };
|
||||
return { status: "ok", item: response.json, statusCode: response.statusCode };
|
||||
}
|
||||
|
||||
async function waitForArgoSyncedHealthy() {
|
||||
const stageStartedAt = Date.now();
|
||||
const progressState = { lastProgressAt: 0 };
|
||||
while (Date.now() < deadline) {
|
||||
const application = await argoApplicationState();
|
||||
if (application.status === "rbac-denied") return fail("argo-sync-health", "argocd-application-rbac-denied", stageStartedAt, { readinessMode });
|
||||
if (application.status !== "ok") return fail("argo-sync-health", "argocd-application-read-failed", stageStartedAt, { readinessMode, statusCode: application.statusCode, body: application.body || null });
|
||||
const syncStatus = application.item?.status?.sync?.status || null;
|
||||
const healthStatus = application.item?.status?.health?.status || null;
|
||||
const operationPhase = application.item?.status?.operationState?.phase || null;
|
||||
const operationMessage = application.item?.status?.operationState?.message || null;
|
||||
if (syncStatus === "Synced" && healthStatus === "Healthy") {
|
||||
emit({ stage: "argo-sync-health", status: "succeeded", durationMs: Date.now() - stageStartedAt, readinessMode, syncStatus, healthStatus, operationPhase });
|
||||
return true;
|
||||
}
|
||||
maybeEmitProgress(progressState, "argo-sync-health", stageStartedAt, { syncStatus, healthStatus, operationPhase, operationMessage });
|
||||
await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now())));
|
||||
}
|
||||
return fail("argo-sync-health", "runtime-ready-wall-clock-timeout", stageStartedAt, { readinessMode }, "timeout");
|
||||
}
|
||||
|
||||
async function waitForArgoRefresh() {
|
||||
const stageStartedAt = Date.now();
|
||||
const progressState = { lastProgressAt: 0 };
|
||||
while (Date.now() < deadline) {
|
||||
const runtime = await runtimeItems();
|
||||
if (runtime.status === "rbac-denied") return fail("argo-refresh", "runtime-observer-rbac-denied", stageStartedAt, { readinessMode });
|
||||
const observed = observedRuntimeItems(runtime.items);
|
||||
const pending = observed
|
||||
.map((item) => ({ kind: item.kind, name: item.metadata?.name, sourceCommit: runtimeSourceCommit(item) }))
|
||||
.filter((item) => item.sourceCommit !== revision);
|
||||
if (observed.length > 0 && pending.length === 0) {
|
||||
emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - stageStartedAt, readinessMode, workloadCount: runtime.items.length, observedCount: observed.length });
|
||||
return true;
|
||||
}
|
||||
maybeEmitProgress(progressState, "argo-refresh", stageStartedAt, { workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) });
|
||||
await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now())));
|
||||
}
|
||||
return fail("argo-refresh", "source-commit-refresh-timeout", stageStartedAt, { readinessMode }, "timeout");
|
||||
}
|
||||
|
||||
async function waitForWorkloads() {
|
||||
const stageStartedAt = Date.now();
|
||||
const progressState = { lastProgressAt: 0 };
|
||||
while (Date.now() < deadline) {
|
||||
const runtime = await runtimeItems();
|
||||
if (runtime.status === "rbac-denied") return fail("workload-ready", "runtime-observer-rbac-denied", stageStartedAt, { readinessMode });
|
||||
const observed = observedRuntimeItems(runtime.items);
|
||||
const blocked = observed.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready);
|
||||
if (observed.length > 0 && blocked.length === 0) {
|
||||
emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - stageStartedAt, readinessMode, workloadCount: runtime.items.length, observedCount: observed.length });
|
||||
return true;
|
||||
}
|
||||
maybeEmitProgress(progressState, "workload-ready", stageStartedAt, { workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) });
|
||||
await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now())));
|
||||
}
|
||||
return fail("workload-ready", "runtime-ready-wall-clock-timeout", stageStartedAt, { readinessMode }, "timeout");
|
||||
}
|
||||
|
||||
function fail(stage, reason, stageStartedAt, extra = {}, status = "failed") {
|
||||
emit({ stage, status, reason, durationMs: Date.now() - stageStartedAt, ...extra });
|
||||
process.exitCode = 1;
|
||||
return false;
|
||||
}
|
||||
|
||||
emit({ stage: "runtime-ready", status: "started", readinessMode, observedServiceCount: observedServiceIds ? observedServiceIds.size : 0, timeoutMs });
|
||||
try {
|
||||
if (!observedServiceIds) {
|
||||
if (await waitForArgoSyncedHealthy()) await waitForWorkloads();
|
||||
} else if (await waitForArgoRefresh()) {
|
||||
await waitForWorkloads();
|
||||
}
|
||||
} catch (error) {
|
||||
emit({ stage: "runtime-ready", status: "failed", reason: error instanceof Error ? error.message : String(error), durationMs: Date.now() - startedAt });
|
||||
process.exitCode = 1;
|
||||
}
|
||||
+21
-265
@@ -2061,6 +2061,11 @@ echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"${ciToolsRunner
|
||||
for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
HWLAB_CATALOG_PATH="$(params.catalog-path)" \
|
||||
HWLAB_GIT_URL="$(params.git-url)" \
|
||||
HWLAB_GIT_READ_URL="$(params.git-read-url)" \
|
||||
HWLAB_GITOPS_BRANCH="$(params.gitops-branch)" \
|
||||
node scripts/ci/restore-artifact-catalog.mjs
|
||||
node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" --verify-reuse-registry > /workspace/source/ci-plan.json
|
||||
node - <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
@@ -3096,6 +3101,10 @@ function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) {
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "git-url" },
|
||||
{ name: "git-read-url" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "lane" },
|
||||
{ name: "revision" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "registry-prefix" },
|
||||
@@ -3116,6 +3125,10 @@ function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) {
|
||||
}]
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: "$(params.git-url)" },
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
@@ -3208,268 +3221,9 @@ export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="runtime-ready"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(name + " is required");
|
||||
return value;
|
||||
}
|
||||
|
||||
const revision = requiredEnv("HWLAB_SOURCE_REVISION");
|
||||
const runtimeNamespace = requiredEnv("HWLAB_RUNTIME_NAMESPACE");
|
||||
const gitopsTarget = requiredEnv("HWLAB_GITOPS_TARGET");
|
||||
const argoApplication = requiredEnv("HWLAB_ARGO_APPLICATION");
|
||||
const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null;
|
||||
const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null;
|
||||
const task = process.env.HWLAB_TEKTON_TASK || null;
|
||||
const timeoutMs = 240000;
|
||||
const pollMs = 1000;
|
||||
const progressEveryMs = 10000;
|
||||
|
||||
function emit(payload) {
|
||||
console.log(JSON.stringify({
|
||||
event: "node-cicd-timing",
|
||||
schemaVersion: "v1",
|
||||
pipelineRun,
|
||||
taskRun,
|
||||
task,
|
||||
revision,
|
||||
source: "scripts/gitops-render.mjs",
|
||||
at: new Date().toISOString(),
|
||||
...payload
|
||||
}));
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function safeJson(text) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!host) {
|
||||
emit({ stage: "argo-refresh", status: "failed", reason: "kubernetes-service-host-missing", durationMs: 0 });
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
|
||||
function request(method, path, body, contentType = "application/json") {
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = contentType;
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function listPath(group, version, namespace, resource) {
|
||||
if (group) return "/apis/" + group + "/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource;
|
||||
return "/api/" + version + "/namespaces/" + encodeURIComponent(namespace) + "/" + resource;
|
||||
}
|
||||
|
||||
async function runtimeItems() {
|
||||
const [deployments, statefulsets] = await Promise.all([
|
||||
request("GET", listPath("apps", "v1", runtimeNamespace, "deployments")),
|
||||
request("GET", listPath("apps", "v1", runtimeNamespace, "statefulsets"))
|
||||
]);
|
||||
if (deployments.statusCode === 403 || statefulsets.statusCode === 403) return { status: "rbac-denied", items: [] };
|
||||
const items = [...(deployments.json?.items || []), ...(statefulsets.json?.items || [])]
|
||||
.filter((item) => item.metadata?.labels?.["hwlab.pikastech.local/gitops-target"] === gitopsTarget);
|
||||
return { status: "ok", items };
|
||||
}
|
||||
|
||||
function runtimeSourceCommit(item) {
|
||||
return item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.metadata?.labels?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.metadata?.annotations?.["hwlab.pikastech.local/gitops-render-source-commit"] ||
|
||||
item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
|
||||
item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] ||
|
||||
item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
|
||||
item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] ||
|
||||
null;
|
||||
}
|
||||
|
||||
function serviceIdForRuntimeItem(item) {
|
||||
return item.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
||||
item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
||||
item.metadata?.labels?.["app.kubernetes.io/name"] ||
|
||||
item.metadata?.name ||
|
||||
null;
|
||||
}
|
||||
|
||||
function readObservedServiceIds() {
|
||||
try {
|
||||
const plan = JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8"));
|
||||
const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices.filter(Boolean) : [];
|
||||
if (rolloutServices.length === 0) return null;
|
||||
return new Set(rolloutServices);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const observedServiceIds = readObservedServiceIds();
|
||||
const readinessMode = observedServiceIds ? "service-rollout" : "infra-runtime-change";
|
||||
|
||||
function observedRuntimeItems(items) {
|
||||
if (!observedServiceIds) return items;
|
||||
return items.filter((item) => observedServiceIds.has(serviceIdForRuntimeItem(item)));
|
||||
}
|
||||
|
||||
function readyWorkload(item) {
|
||||
const specReplicas = item.spec?.replicas ?? 1;
|
||||
if (specReplicas === 0) return { ready: true, skipped: true };
|
||||
const status = item.status || {};
|
||||
const desired = specReplicas;
|
||||
const ready = status.readyReplicas ?? 0;
|
||||
const updated = status.updatedReplicas ?? 0;
|
||||
const observedGeneration = status.observedGeneration ?? 0;
|
||||
const generation = item.metadata?.generation ?? 0;
|
||||
return { ready: observedGeneration >= generation && ready >= desired && updated >= desired, desired, readyReplicas: ready, updatedReplicas: updated };
|
||||
}
|
||||
|
||||
function maybeEmitProgress(state, stage, startedAt, payload) {
|
||||
const now = Date.now();
|
||||
if (now - state.lastProgressAt < progressEveryMs) return;
|
||||
state.lastProgressAt = now;
|
||||
emit({ stage, status: "progress", durationMs: now - startedAt, readinessMode, ...payload });
|
||||
}
|
||||
|
||||
async function argoApplicationState() {
|
||||
const path = "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(argoApplication);
|
||||
const response = await request("GET", path);
|
||||
if (response.statusCode === 403) return { status: "rbac-denied", item: null, statusCode: response.statusCode };
|
||||
if (response.statusCode < 200 || response.statusCode > 299) return { status: "error", item: null, statusCode: response.statusCode, body: response.body.slice(0, 500) };
|
||||
return { status: "ok", item: response.json, statusCode: response.statusCode };
|
||||
}
|
||||
|
||||
async function waitForArgoSyncedHealthy() {
|
||||
const startedAt = Date.now();
|
||||
const progressState = { lastProgressAt: 0 };
|
||||
for (;;) {
|
||||
const application = await argoApplicationState();
|
||||
if (application.status === "rbac-denied") {
|
||||
emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-rbac-denied", durationMs: Date.now() - startedAt, readinessMode });
|
||||
process.exitCode = 1;
|
||||
return { observed: false };
|
||||
}
|
||||
if (application.status !== "ok") {
|
||||
emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-read-failed", statusCode: application.statusCode, body: application.body || null, durationMs: Date.now() - startedAt, readinessMode });
|
||||
process.exitCode = 1;
|
||||
return { observed: false };
|
||||
}
|
||||
const syncStatus = application.item?.status?.sync?.status || null;
|
||||
const healthStatus = application.item?.status?.health?.status || null;
|
||||
const operationPhase = application.item?.status?.operationState?.phase || null;
|
||||
const operationMessage = application.item?.status?.operationState?.message || null;
|
||||
if (syncStatus === "Synced" && healthStatus === "Healthy") {
|
||||
emit({ stage: "argo-sync-health", status: "succeeded", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase });
|
||||
return { observed: true };
|
||||
}
|
||||
if (Date.now() - startedAt >= timeoutMs) {
|
||||
emit({ stage: "argo-sync-health", status: "timeout", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase, operationMessage });
|
||||
process.exitCode = 1;
|
||||
return { observed: false };
|
||||
}
|
||||
maybeEmitProgress(progressState, "argo-sync-health", startedAt, { syncStatus, healthStatus, operationPhase, operationMessage });
|
||||
await sleep(pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForArgoRefresh() {
|
||||
const startedAt = Date.now();
|
||||
const progressState = { lastProgressAt: 0 };
|
||||
for (;;) {
|
||||
const runtime = await runtimeItems();
|
||||
if (runtime.status === "rbac-denied") {
|
||||
emit({ stage: "argo-refresh", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
|
||||
process.exitCode = 1;
|
||||
return { observed: false, skipped: true };
|
||||
}
|
||||
const observed = observedRuntimeItems(runtime.items);
|
||||
const pending = observed
|
||||
.map((item) => ({ kind: item.kind, name: item.metadata?.name, sourceCommit: runtimeSourceCommit(item) }))
|
||||
.filter((item) => item.sourceCommit !== revision);
|
||||
if (observed.length > 0 && pending.length === 0) {
|
||||
emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length });
|
||||
return { observed: true, skipped: false };
|
||||
}
|
||||
if (Date.now() - startedAt >= timeoutMs) {
|
||||
emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) });
|
||||
process.exitCode = 1;
|
||||
return { observed: false, skipped: false };
|
||||
}
|
||||
maybeEmitProgress(progressState, "argo-refresh", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) });
|
||||
await sleep(pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForWorkloads() {
|
||||
const startedAt = Date.now();
|
||||
const progressState = { lastProgressAt: 0 };
|
||||
for (;;) {
|
||||
const runtime = await runtimeItems();
|
||||
if (runtime.status === "rbac-denied") {
|
||||
emit({ stage: "workload-ready", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const observed = observedRuntimeItems(runtime.items);
|
||||
const blocked = observed.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready);
|
||||
if (observed.length > 0 && blocked.length === 0) {
|
||||
emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length });
|
||||
return;
|
||||
}
|
||||
if (Date.now() - startedAt >= timeoutMs) {
|
||||
emit({ stage: "workload-ready", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) });
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
maybeEmitProgress(progressState, "workload-ready", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) });
|
||||
await sleep(pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
emit({ stage: "runtime-ready", status: "started", readinessMode, observedServiceCount: observedServiceIds ? observedServiceIds.size : 0 });
|
||||
if (!observedServiceIds) {
|
||||
const argo = await waitForArgoSyncedHealthy();
|
||||
if (!argo.observed) return;
|
||||
await waitForWorkloads();
|
||||
return;
|
||||
}
|
||||
const refresh = await waitForArgoRefresh();
|
||||
if (!refresh.observed) return;
|
||||
await waitForWorkloads();
|
||||
})().catch((error) => {
|
||||
emit({ stage: "runtime-ready", status: "failed", reason: error.message, durationMs: 0 });
|
||||
process.exitCode = 1;
|
||||
});
|
||||
NODE
|
||||
export HWLAB_RUNTIME_READY_TIMEOUT_MS="$(params.timeout-ms)"
|
||||
cd /workspace/source/repo
|
||||
node scripts/ci/runtime-ready.mjs
|
||||
`;
|
||||
}
|
||||
|
||||
@@ -3479,7 +3233,7 @@ function runtimeReadyTask({ profile = "dev" } = {}) {
|
||||
runAfter: ["gitops-promote"],
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }],
|
||||
params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }, { name: "timeout-ms" }],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{
|
||||
name: "runtime-ready",
|
||||
@@ -3497,7 +3251,8 @@ function runtimeReadyTask({ profile = "dev" } = {}) {
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "runtime-namespace", value: namespaceNameForProfile(profile) },
|
||||
{ name: "gitops-target", value: gitopsTargetForProfile(profile) },
|
||||
{ name: "argo-application", value: argoApplicationName(profile) }
|
||||
{ name: "argo-application", value: argoApplicationName(profile) },
|
||||
{ name: "timeout-ms", value: "$(params.runtime-ready-timeout-ms)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
@@ -3548,7 +3303,8 @@ function tektonPipeline(args = { lane: "node" }, deploy = null) {
|
||||
{ name: "registry-prefix", type: "string", default: defaultRegistryPrefix },
|
||||
{ name: "services", type: "string", default: servicesParamForLane(args.lane, deploy) },
|
||||
{ name: "base-image", type: "string", default: defaultDevBaseImage },
|
||||
{ name: "build-cache-mode", type: "string", default: "registry" }
|
||||
{ name: "build-cache-mode", type: "string", default: "registry" },
|
||||
{ name: "runtime-ready-timeout-ms", type: "string", default: "60000" }
|
||||
],
|
||||
workspaces: [
|
||||
{ name: "source" },
|
||||
|
||||
@@ -165,29 +165,32 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
|
||||
assert.match(pipeline, /tar -C \/workspace\/source\/repo -cf - \. \| tar -C \/workspace\/service-work\/repo -xo -f -/u);
|
||||
assert.match(pipeline, /node scripts\/run-bun\.mjs test cmd\/hwlab-codex-api-responses-forwarder\/main\.test\.ts/u);
|
||||
assert.doesNotMatch(pipeline, /cmd\/hwlab-codex-api-responses-forwarder\/main\.mjs/u);
|
||||
assert.match(pipeline, /process\.exitCode = 1/u);
|
||||
assert.match(pipeline, /item\.metadata\?\.labels\?\.\[\\"hwlab\.pikastech\.local\/source-commit\\"\]/u);
|
||||
const runtimeReadySource = await readFile(path.join(process.cwd(), "scripts/ci/runtime-ready.mjs"), "utf8");
|
||||
assert.match(runtimeReadySource, /process\.exitCode = 1/u);
|
||||
assert.match(runtimeReadySource, /item\.metadata\?\.labels\?\.\["hwlab\.pikastech\.local\/source-commit"\]/u);
|
||||
|
||||
const runtimeReady = taskByName(pipelineJson, "runtime-ready");
|
||||
assert.deepEqual(runtimeReady.when, [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }]);
|
||||
const runtimeReadyScript = runtimeReady.taskSpec.steps[0].script;
|
||||
assert.match(runtimeReadyScript, /const pollMs = 1000;/u);
|
||||
assert.match(runtimeReadyScript, /readObservedServiceIds/u);
|
||||
assert.match(runtimeReadyScript, /observedRuntimeItems/u);
|
||||
assert.match(runtimeReadyScript, /observedCount/u);
|
||||
assert.match(runtimeReadyScript, /readinessMode = observedServiceIds \? "service-rollout" : "infra-runtime-change"/u);
|
||||
assert.match(runtimeReadyScript, /waitForArgoSyncedHealthy/u);
|
||||
assert.match(runtimeReadyScript, /stage: "runtime-ready", status: "started"/u);
|
||||
assert.match(runtimeReadyScript, /status: "progress"/u);
|
||||
assert.match(runtimeReadyScript, /stage: "argo-sync-health"/u);
|
||||
assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u);
|
||||
assert.match(runtimeReadyScript, /if \(!observedServiceIds\) \{/u);
|
||||
assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u);
|
||||
assert.match(runtimeReadyScript, /HWLAB_RUNTIME_READY_TIMEOUT_MS="\$\(params\.timeout-ms\)"/u);
|
||||
assert.match(runtimeReadyScript, /cd \/workspace\/source\/repo/u);
|
||||
assert.match(runtimeReadyScript, /node scripts\/ci\/runtime-ready\.mjs/u);
|
||||
assert.ok(runtimeReady.taskSpec.params.some((param) => param.name === "argo-application"));
|
||||
assert.ok(runtimeReady.taskSpec.params.some((param) => param.name === "timeout-ms"));
|
||||
assert.ok(runtimeReady.params.some((param) => param.name === "argo-application" && param.value === "hwlab-node-v02"));
|
||||
assert.ok(runtimeReady.params.some((param) => param.name === "timeout-ms" && param.value === "$(params.runtime-ready-timeout-ms)"));
|
||||
assert.ok(runtimeReady.workspaces?.some((workspace) => workspace.name === "source" && workspace.workspace === "source"));
|
||||
assert.ok(runtimeReady.taskSpec.workspaces?.some((workspace) => workspace.name === "source"));
|
||||
|
||||
const planArtifacts = taskByName(pipelineJson, "plan-artifacts");
|
||||
const planArtifactsScript = planArtifacts.taskSpec.steps[0].script;
|
||||
assert.match(planArtifactsScript, /node scripts\/ci\/restore-artifact-catalog\.mjs/u);
|
||||
assert.match(planArtifactsScript, /HWLAB_GIT_READ_URL="\$\(params\.git-read-url\)"/u);
|
||||
assert.ok(planArtifacts.taskSpec.params.some((param) => param.name === "git-read-url"));
|
||||
assert.ok(planArtifacts.taskSpec.params.some((param) => param.name === "gitops-branch"));
|
||||
assert.ok(planArtifacts.params.some((param) => param.name === "lane" && param.value === "$(params.lane)"));
|
||||
assert.ok(pipelineJson.spec.params.some((param) => param.name === "runtime-ready-timeout-ms" && param.default === "60000"));
|
||||
|
||||
const gitopsPromote = taskByName(pipelineJson, "gitops-promote");
|
||||
assert.ok(gitopsPromote.taskSpec.results.some((result) => result.name === "runtime-ready-required"));
|
||||
const gitopsPromoteScript = gitopsPromote.taskSpec.steps[0].script;
|
||||
@@ -238,7 +241,6 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
|
||||
assert.doesNotMatch(prepareSourceScript, /prepare-source-proxy-preflight/u);
|
||||
assert.doesNotMatch(prepareSourceScript, /dependency-curl-probe-start/u);
|
||||
|
||||
const planArtifacts = taskByName(pipelineJson, "plan-artifacts");
|
||||
assert.doesNotMatch(planArtifacts.taskSpec.steps[0].script, /plan-artifacts-proxy-preflight/u);
|
||||
const collectArtifacts = taskByName(pipelineJson, "collect-artifacts");
|
||||
assert.doesNotMatch(collectArtifacts.taskSpec.steps[0].script, /collect-artifacts-proxy-preflight/u);
|
||||
|
||||
Reference in New Issue
Block a user