From f68c88a76425f27906bf747778a3e6c25570ec93 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 29 May 2026 11:51:40 +0800 Subject: [PATCH] fix: fail closed v02 runtime readiness --- docs/reference/spec-v02-cicd.md | 5 +- scripts/g14-gitops-render.mjs | 102 ++++++++++++++++++++++++++++-- scripts/g14-gitops-render.test.ts | 9 ++- 3 files changed, 106 insertions(+), 10 deletions(-) diff --git a/docs/reference/spec-v02-cicd.md b/docs/reference/spec-v02-cicd.md index b5ad3edc..71a22c5a 100644 --- a/docs/reference/spec-v02-cicd.md +++ b/docs/reference/spec-v02-cicd.md @@ -159,7 +159,8 @@ CI/CD 内部由 branch poller、PipelineRun、component planner、BuildKit publi - `v0.2-gitops` 中存在 `deploy/artifact-catalog.v02.json` 与 `deploy/gitops/g14/runtime-v02/**`。 - `argocd/hwlab-g14-v02` 指向 `v0.2-gitops:deploy/gitops/g14/runtime-v02`，sync revision 与目标 GitOps revision 对齐。 - `hwlab-v02` 中长驻 workload ready，没有把 DEV/PROD namespace 当成 `v0.2` 通过证据。 -- `runtime-ready` 必须以 workload Pod template 的 source commit 判断 Argo 刷新，并在 Argo 刷新超时或 workload 未就绪时失败；不得把 CrashLoop、未刷新或公网不可用的 runtime 标成绿色。 +- `gitops-promote` 推送 `v0.2-gitops` 后应触发 `argocd/hwlab-g14-v02` hard refresh，减少 GitOps push 与 Argo 仓库发现之间的漂移窗口；refresh 触发失败只作为低噪声事件输出，最终通过仍由 `runtime-ready` 判定。 +- `runtime-ready` 必须以 workload Pod template 的 source commit 判断 Argo 刷新，并在 Argo 刷新超时、observer RBAC 不足或 workload 未就绪时失败；不得把 CrashLoop、未刷新或公网不可用的 runtime 标成绿色。 - `http://74.48.78.17:19666/` 返回 `v0.2` Cloud Web。 - `http://74.48.78.17:19667/health/live` 返回 `v0.2` runtime health，payload 中的 namespace、revision 或 runtime identity 能与 `hwlab-v02`/`v0.2` 对齐。 @@ -189,7 +190,7 @@ GitOps branch 已更新、source branch render 通过、PipelineRun 名称存在 | --- | --- | --- | | v02 独立 source/GitOps/runtime lane | 已实现 | `v0.2`、`v0.2-gitops`、`hwlab-v02` 和 `runtime-v02` 已固定。 | | Tekton poller/pipeline/promotion | 已实现 | 通过 `hwlab-v02-*` 对象和 GitOps promotion 管理。 | -| v02 runtime readiness fail-closed | 已实现 | `runtime-ready` 读取 workload Pod template source commit，timeout 或未就绪会让 PipelineRun 失败。 | +| v02 runtime readiness fail-closed | 已实现 | `gitops-promote` 推送后触发 Argo hard refresh；`runtime-ready` 读取 workload Pod template source commit，timeout、observer RBAC 不足或未就绪会让 PipelineRun 失败。 | | v02 裁撤服务不进入发布面 | 已实现 | v02 Tekton build service set、artifact catalog 和 runtime render 只包含保留服务；v02 Argo 开启 prune 清理旧 live 对象；裁撤服务仍可保留在 DEV legacy 源码中。 | | Argo v02 Application | 已实现 | `hwlab-g14-v02` 指向 v02 GitOps path 和 namespace。 | | FRP `19666/19667` 入口 | 已实现 | 由 `hwlab-v02-frpc` 与 master frps allowlist 共同提供。 | diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index 4dd8178f..3ed65a36 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -1266,6 +1266,89 @@ git config --global user.email "hwlab-g14-gitops-bot@users.noreply.github.com" catalog_path="$(params.catalog-path)" runtime_path="$(params.runtime-path)" lane="$(params.lane)" + +argo_hard_refresh_v02() { + if [ "$lane" != "v02" ]; then return 0; fi + node <<'NODE' +const fs = require("node:fs"); +const https = require("node:https"); + +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = process.env.KUBERNETES_SERVICE_PORT || "443"; +const startedAt = Date.now(); +const application = "hwlab-g14-v02"; +const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null; +const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null; +const task = process.env.HWLAB_TEKTON_TASK || null; +const revision = process.env.HWLAB_SOURCE_REVISION || null; + +function emit(payload) { + console.log(JSON.stringify({ + event: "g14-cicd-timing", + schemaVersion: "v1", + stage: "argo-hard-refresh", + pipelineRun, + taskRun, + task, + revision, + application, + source: "scripts/g14-gitops-render.mjs", + at: new Date().toISOString(), + ...payload + })); +} + +function safeJson(text) { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +function request(method, path, body, contentType = "application/merge-patch+json") { + const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); + const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + const payload = body ? JSON.stringify(body) : ""; + const headers = { Authorization: "Bearer " + token }; + if (payload) { + headers["Content-Type"] = contentType; + headers["Content-Length"] = Buffer.byteLength(payload); + } + return new Promise((resolve, reject) => { + const req = https.request({ host, port, method, path, ca, headers }, (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { data += chunk; }); + res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) })); + }); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +(async () => { + if (!host) { + emit({ status: "skipped", reason: "kubernetes-service-host-missing", durationMs: Date.now() - startedAt }); + return; + } + const response = await request( + "PATCH", + "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(application) + "?fieldManager=hwlab-v02-gitops-promote", + { metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } } + ); + if (response.statusCode >= 200 && response.statusCode < 300) { + emit({ status: "succeeded", durationMs: Date.now() - startedAt, statusCode: response.statusCode }); + return; + } + emit({ status: "degraded", reason: "argo-refresh-patch-failed", durationMs: Date.now() - startedAt, statusCode: response.statusCode, body: response.body.slice(0, 1000) }); +})().catch((error) => { + emit({ status: "degraded", reason: "argo-refresh-patch-error", durationMs: Date.now() - startedAt, error: error.message }); +}); +NODE +} + if [ -s /workspace/source/dev-artifacts.json ]; then node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-json deploy/deploy.json --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write fi @@ -1302,6 +1385,7 @@ if git diff --cached --quiet; then echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' ci_timing_emit gitops-commit unchanged "$(ci_now_ms)" ci_timing_emit gitops-push unchanged "$(ci_now_ms)" + argo_hard_refresh_v02 exit 0 fi short="$(printf '%.7s' "$(params.revision)")" @@ -1311,6 +1395,7 @@ ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms" gitops_push_started_ms="$(ci_now_ms)" git push origin "HEAD:$(params.gitops-branch)" ci_timing_emit gitops-push succeeded "$gitops_push_started_ms" +argo_hard_refresh_v02 echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' `; } @@ -2012,9 +2097,8 @@ function safeJson(text) { } if (!host) { - emit({ stage: "argo-refresh", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 }); - emit({ stage: "workload-ready", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 }); - process.exit(0); + emit({ stage: "argo-refresh", status: "failed", reason: "kubernetes-service-host-missing", durationMs: 0 }); + process.exit(1); } const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); @@ -2081,7 +2165,8 @@ async function waitForArgoRefresh() { for (;;) { const runtime = await runtimeItems(); if (runtime.status === "rbac-denied") { - emit({ stage: "argo-refresh", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); + emit({ stage: "argo-refresh", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); + process.exitCode = 1; return { observed: false, skipped: true }; } const pending = runtime.items @@ -2092,7 +2177,8 @@ async function waitForArgoRefresh() { return { observed: true, skipped: false }; } if (Date.now() - startedAt >= timeoutMs) { - emit({ stage: "argo-refresh", status: "degraded", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) }); + emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) }); + process.exitCode = 1; return { observed: false, skipped: false }; } await sleep(pollMs); @@ -2104,7 +2190,8 @@ async function waitForWorkloads() { for (;;) { const runtime = await runtimeItems(); if (runtime.status === "rbac-denied") { - emit({ stage: "workload-ready", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); + emit({ stage: "workload-ready", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt }); + process.exitCode = 1; return; } const blocked = runtime.items.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready); @@ -2122,7 +2209,8 @@ async function waitForWorkloads() { } (async () => { - await waitForArgoRefresh(); + const refresh = await waitForArgoRefresh(); + if (!refresh.observed) return; await waitForWorkloads(); })().catch((error) => { emit({ stage: "runtime-ready", status: "failed", reason: error.message, durationMs: 0 }); diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index c48b8cc6..d59688c2 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -41,7 +41,14 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots const runtimeReady = taskByName(pipelineJson, "runtime-ready"); const runtimeReadyScript = runtimeReady.taskSpec.steps[0].script; - assert.match(runtimeReadyScript, /status: "degraded", reason: "source-commit-refresh-timeout"/u); + assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u); + assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u); + + const gitopsPromote = taskByName(pipelineJson, "gitops-promote"); + const gitopsPromoteScript = gitopsPromote.taskSpec.steps[0].script; + assert.match(gitopsPromoteScript, /argo_hard_refresh_v02\(\) \{/u); + assert.match(gitopsPromoteScript, /argocd\.argoproj\.io\/refresh": "hard"/u); + assert.match(gitopsPromoteScript, /applications\/" \+ encodeURIComponent\(application\) \+ "\?fieldManager=hwlab-v02-gitops-promote/u); const removedServiceIds = [ "hwlab-router",