From c64c9a04553ef7cc3c2635093d572140efc8189e Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 31 May 2026 12:51:20 +0800 Subject: [PATCH] fix: expose v02 runtime readiness progress --- docs/reference/spec-v02-cicd.md | 2 + scripts/g14-gitops-render.mjs | 70 +++++++++++++++++++++++++++++-- scripts/g14-gitops-render.test.ts | 8 ++++ 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/docs/reference/spec-v02-cicd.md b/docs/reference/spec-v02-cicd.md index 9247adcd..9df3fed3 100644 --- a/docs/reference/spec-v02-cicd.md +++ b/docs/reference/spec-v02-cicd.md @@ -219,6 +219,8 @@ P1 no-op runtime skip 只跳过无实际 runtime 变化的等待。planner 输 `runtime-ready` 只能观察本轮 `rolloutServices` 对应的 workload readiness，不能把整个 `hwlab-v02` namespace 的全量 workload 都绑定到当前 source commit 后等待。复用 service 的 Deployment metadata 可以记录本次 source commit 作为 GitOps revision 线索，但 Pod template identity 必须来自该 service 的 artifact/runtime commit、boot commit 或配置内容 hash；否则 catalog 复用会被误转成不必要 rollout，`runtime-ready` 会被无关服务拖慢。`hwlab-cloud-api` 的 rollout 允许连带观察 `hwlab-deepseek-proxy`，因为 deepseek bridge 复用 cloud-api 镜像作为运行载体；其他连带关系必须有明确运行依赖后再加入观察集合。基础对象如 FRP、Postgres 的 Pod template 不得写入全局 source commit，应用配置 hash 或 migration hash 表达真实重启边界。 +`runtime-ready` 的可见性优先级高于耗时优化。Task 启动后必须立即输出 `runtime-ready started` 事件，明确 `readinessMode`、观察服务数和当前 revision；等待 Argo、source commit refresh 或 workload ready 时必须周期性输出 `progress` 事件，包含 `observedCount`、`workloadCount`、pending/blocked 摘要或 Argo sync/health 状态。service rollout 场景继续用 `rolloutServices` 对应的 Pod template source commit 判定；`rolloutServices=[]` 但 GitOps runtime 真实变化的 infra-only 场景不得黑盒等待 source commit，应改为等待 Argo Application `Synced/Healthy` 和 workload generation ready，并输出 `argo-sync-health` 事件。禁止再出现 runtime-ready 只在 240s timeout 后才输出一条日志的不可观察状态。 + Kubernetes label 里的配置 hash 必须使用短 hash，完整 64 位 sha256 只能放 annotation。FRP `config-sha256`、Postgres `migration-sha256` 这类字段如果进入 Pod template label，长度必须小于 Kubernetes label value 的 63 字节上限；annotation 保留完整 hash 用于排障和人工比对。Argo sync 报 `must be no more than 63 bytes` 时，优先检查是否把完整 sha256 写入 label，不要通过删除 hash、放宽 sync 或忽略 OutOfSync 绕过。 快速优化不能绕过 fail-closed 语义。mirror miss、commit ancestry 不合法、boot script 缺失、digest 缺失、Argo observer RBAC 不足、workload 未 ready 或公网 health 不一致，都不能为了追求耗时而降级为 warning。允许跳过的只有已经证明 runtime desired state 没有实际变化的等待。 diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index 88d4b786..2fe4f9e7 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -2368,11 +2368,13 @@ function requiredEnv(name) { const revision = requiredEnv("HWLAB_SOURCE_REVISION"); const runtimeNamespace = requiredEnv("HWLAB_RUNTIME_NAMESPACE"); const gitopsTarget = requiredEnv("HWLAB_GITOPS_TARGET"); +const argoApplication = requiredEnv("HWLAB_ARGO_APPLICATION"); const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null; const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null; const task = process.env.HWLAB_TEKTON_TASK || null; const timeoutMs = 240000; const pollMs = 1000; +const progressEveryMs = 10000; function emit(payload) { console.log(JSON.stringify({ @@ -2474,6 +2476,7 @@ function readObservedServiceIds() { } const observedServiceIds = readObservedServiceIds(); +const readinessMode = observedServiceIds ? "service-rollout" : "infra-runtime-change"; function observedRuntimeItems(items) { if (!observedServiceIds) return items; @@ -2492,8 +2495,57 @@ function readyWorkload(item) { return { ready: observedGeneration >= generation && ready >= desired && updated >= desired, desired, readyReplicas: ready, updatedReplicas: updated }; } +function maybeEmitProgress(state, stage, startedAt, payload) { + const now = Date.now(); + if (now - state.lastProgressAt < progressEveryMs) return; + state.lastProgressAt = now; + emit({ stage, status: "progress", durationMs: now - startedAt, readinessMode, ...payload }); +} + +async function argoApplicationState() { + const path = "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(argoApplication); + const response = await request("GET", path); + if (response.statusCode === 403) return { status: "rbac-denied", item: null, statusCode: response.statusCode }; + if (response.statusCode < 200 || response.statusCode > 299) return { status: "error", item: null, statusCode: response.statusCode, body: response.body.slice(0, 500) }; + return { status: "ok", item: response.json, statusCode: response.statusCode }; +} + +async function waitForArgoSyncedHealthy() { + const startedAt = Date.now(); + const progressState = { lastProgressAt: 0 }; + for (;;) { + const application = await argoApplicationState(); + if (application.status === "rbac-denied") { + emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-rbac-denied", durationMs: Date.now() - startedAt, readinessMode }); + process.exitCode = 1; + return { observed: false }; + } + if (application.status !== "ok") { + emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-read-failed", statusCode: application.statusCode, body: application.body || null, durationMs: Date.now() - startedAt, readinessMode }); + process.exitCode = 1; + return { observed: false }; + } + const syncStatus = application.item?.status?.sync?.status || null; + const healthStatus = application.item?.status?.health?.status || null; + const operationPhase = application.item?.status?.operationState?.phase || null; + const operationMessage = application.item?.status?.operationState?.message || null; + if (syncStatus === "Synced" && healthStatus === "Healthy") { + emit({ stage: "argo-sync-health", status: "succeeded", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase }); + return { observed: true }; + } + if (Date.now() - startedAt >= timeoutMs) { + emit({ stage: "argo-sync-health", status: "timeout", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase, operationMessage }); + process.exitCode = 1; + return { observed: false }; + } + maybeEmitProgress(progressState, "argo-sync-health", startedAt, { syncStatus, healthStatus, operationPhase, operationMessage }); + await sleep(pollMs); + } +} + async function waitForArgoRefresh() { const startedAt = Date.now(); + const progressState = { lastProgressAt: 0 }; for (;;) { const runtime = await runtimeItems(); if (runtime.status === "rbac-denied") { @@ -2514,12 +2566,14 @@ async function waitForArgoRefresh() { process.exitCode = 1; return { observed: false, skipped: false }; } + maybeEmitProgress(progressState, "argo-refresh", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) }); await sleep(pollMs); } } async function waitForWorkloads() { const startedAt = Date.now(); + const progressState = { lastProgressAt: 0 }; for (;;) { const runtime = await runtimeItems(); if (runtime.status === "rbac-denied") { @@ -2538,11 +2592,19 @@ async function waitForWorkloads() { process.exitCode = 1; return; } + maybeEmitProgress(progressState, "workload-ready", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) }); await sleep(pollMs); } } (async () => { + emit({ stage: "runtime-ready", status: "started", readinessMode, observedServiceCount: observedServiceIds ? observedServiceIds.size : 0 }); + if (!observedServiceIds) { + const argo = await waitForArgoSyncedHealthy(); + if (!argo.observed) return; + await waitForWorkloads(); + return; + } const refresh = await waitForArgoRefresh(); if (!refresh.observed) return; await waitForWorkloads(); @@ -2560,7 +2622,7 @@ function runtimeReadyTask({ profile = "dev" } = {}) { runAfter: ["gitops-promote"], workspaces: [{ name: "source", workspace: "source" }], taskSpec: { - params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }], + params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }], workspaces: [{ name: "source" }], steps: [{ name: "runtime-ready", @@ -2568,7 +2630,8 @@ function runtimeReadyTask({ profile = "dev" } = {}) { env: [ ...proxyEnv(), { name: "HWLAB_RUNTIME_NAMESPACE", value: "$(params.runtime-namespace)" }, - { name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" } + { name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" }, + { name: "HWLAB_ARGO_APPLICATION", value: "$(params.argo-application)" } ], script: runtimeReadyScript() }] @@ -2576,7 +2639,8 @@ function runtimeReadyTask({ profile = "dev" } = {}) { params: [ { name: "revision", value: "$(params.revision)" }, { name: "runtime-namespace", value: namespaceNameForProfile(profile) }, - { name: "gitops-target", value: gitopsTargetForProfile(profile) } + { name: "gitops-target", value: gitopsTargetForProfile(profile) }, + { name: "argo-application", value: argoApplicationName(profile) } ] }; } diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index 02c49a1a..7680ab3e 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -55,9 +55,17 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.match(runtimeReadyScript, /readObservedServiceIds/u); assert.match(runtimeReadyScript, /observedRuntimeItems/u); assert.match(runtimeReadyScript, /observedCount/u); + assert.match(runtimeReadyScript, /readinessMode = observedServiceIds \? "service-rollout" : "infra-runtime-change"/u); + assert.match(runtimeReadyScript, /waitForArgoSyncedHealthy/u); + assert.match(runtimeReadyScript, /stage: "runtime-ready", status: "started"/u); + assert.match(runtimeReadyScript, /status: "progress"/u); + assert.match(runtimeReadyScript, /stage: "argo-sync-health"/u); assert.match(runtimeReadyScript, /ids\.has\("hwlab-cloud-api"\)\) ids\.add\("hwlab-deepseek-proxy"\)/u); assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u); + assert.match(runtimeReadyScript, /if \(!observedServiceIds\) \{/u); assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u); + assert.ok(runtimeReady.taskSpec.params.some((param) => param.name === "argo-application")); + assert.ok(runtimeReady.params.some((param) => param.name === "argo-application" && param.value === "hwlab-g14-v02")); assert.ok(runtimeReady.workspaces?.some((workspace) => workspace.name === "source" && workspace.workspace === "source")); assert.ok(runtimeReady.taskSpec.workspaces?.some((workspace) => workspace.name === "source"));