Merge pull request #555 from pikasTech/fix/v02-runtime-ready-fail-closed

fix: fail closed v02 runtime readiness

docs/reference/spec-v02-cicd.md

@@ -159,7 +159,8 @@ CI/CD 内部由 branch poller、PipelineRun、component planner、BuildKit publi
 - `v0.2-gitops` 中存在 `deploy/artifact-catalog.v02.json` 与 `deploy/gitops/g14/runtime-v02/**`。
 - `argocd/hwlab-g14-v02` 指向 `v0.2-gitops:deploy/gitops/g14/runtime-v02`，sync revision 与目标 GitOps revision 对齐。
 - `hwlab-v02` 中长驻 workload ready，没有把 DEV/PROD namespace 当成 `v0.2` 通过证据。
-- `runtime-ready` 必须以 workload Pod template 的 source commit 判断 Argo 刷新，并在 Argo 刷新超时或 workload 未就绪时失败；不得把 CrashLoop、未刷新或公网不可用的 runtime 标成绿色。
+- `gitops-promote` 推送 `v0.2-gitops` 后应触发 `argocd/hwlab-g14-v02` hard refresh，减少 GitOps push 与 Argo 仓库发现之间的漂移窗口；refresh 触发失败只作为低噪声事件输出，最终通过仍由 `runtime-ready` 判定。
+- `runtime-ready` 必须以 workload Pod template 的 source commit 判断 Argo 刷新，并在 Argo 刷新超时、observer RBAC 不足或 workload 未就绪时失败；不得把 CrashLoop、未刷新或公网不可用的 runtime 标成绿色。
 - `http://74.48.78.17:19666/` 返回 `v0.2` Cloud Web。
 - `http://74.48.78.17:19667/health/live` 返回 `v0.2` runtime health，payload 中的 namespace、revision 或 runtime identity 能与 `hwlab-v02`/`v0.2` 对齐。
 
@@ -189,7 +190,7 @@ GitOps branch 已更新、source branch render 通过、PipelineRun 名称存在
 | --- | --- | --- |
 | v02 独立 source/GitOps/runtime lane | 已实现 | `v0.2`、`v0.2-gitops`、`hwlab-v02` 和 `runtime-v02` 已固定。 |
 | Tekton poller/pipeline/promotion | 已实现 | 通过 `hwlab-v02-*` 对象和 GitOps promotion 管理。 |
-| v02 runtime readiness fail-closed | 已实现 | `runtime-ready` 读取 workload Pod template source commit，timeout 或未就绪会让 PipelineRun 失败。 |
+| v02 runtime readiness fail-closed | 已实现 | `gitops-promote` 推送后触发 Argo hard refresh；`runtime-ready` 读取 workload Pod template source commit，timeout、observer RBAC 不足或未就绪会让 PipelineRun 失败。 |
 | v02 裁撤服务不进入发布面 | 已实现 | v02 Tekton build service set、artifact catalog 和 runtime render 只包含保留服务；v02 Argo 开启 prune 清理旧 live 对象；裁撤服务仍可保留在 DEV legacy 源码中。 |
 | Argo v02 Application | 已实现 | `hwlab-g14-v02` 指向 v02 GitOps path 和 namespace。 |
 | FRP `19666/19667` 入口 | 已实现 | 由 `hwlab-v02-frpc` 与 master frps allowlist 共同提供。 |

scripts/g14-gitops-render.mjs

@@ -1266,6 +1266,89 @@ git config --global user.email "hwlab-g14-gitops-bot@users.noreply.github.com"
 catalog_path="$(params.catalog-path)"
 runtime_path="$(params.runtime-path)"
 lane="$(params.lane)"
+
+argo_hard_refresh_v02() {
+  if [ "$lane" != "v02" ]; then return 0; fi
+  node <<'NODE'
+const fs = require("node:fs");
+const https = require("node:https");
+
+const host = process.env.KUBERNETES_SERVICE_HOST;
+const port = process.env.KUBERNETES_SERVICE_PORT || "443";
+const startedAt = Date.now();
+const application = "hwlab-g14-v02";
+const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null;
+const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null;
+const task = process.env.HWLAB_TEKTON_TASK || null;
+const revision = process.env.HWLAB_SOURCE_REVISION || null;
+
+function emit(payload) {
+  console.log(JSON.stringify({
+    event: "g14-cicd-timing",
+    schemaVersion: "v1",
+    stage: "argo-hard-refresh",
+    pipelineRun,
+    taskRun,
+    task,
+    revision,
+    application,
+    source: "scripts/g14-gitops-render.mjs",
+    at: new Date().toISOString(),
+    ...payload
+  }));
+}
+
+function safeJson(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}
+
+function request(method, path, body, contentType = "application/merge-patch+json") {
+  const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
+  const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
+  const payload = body ? JSON.stringify(body) : "";
+  const headers = { Authorization: "Bearer " + token };
+  if (payload) {
+    headers["Content-Type"] = contentType;
+    headers["Content-Length"] = Buffer.byteLength(payload);
+  }
+  return new Promise((resolve, reject) => {
+    const req = https.request({ host, port, method, path, ca, headers }, (res) => {
+      let data = "";
+      res.setEncoding("utf8");
+      res.on("data", (chunk) => { data += chunk; });
+      res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) }));
+    });
+    req.on("error", reject);
+    if (payload) req.write(payload);
+    req.end();
+  });
+}
+
+(async () => {
+  if (!host) {
+    emit({ status: "skipped", reason: "kubernetes-service-host-missing", durationMs: Date.now() - startedAt });
+    return;
+  }
+  const response = await request(
+    "PATCH",
+    "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(application) + "?fieldManager=hwlab-v02-gitops-promote",
+    { metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } }
+  );
+  if (response.statusCode >= 200 && response.statusCode < 300) {
+    emit({ status: "succeeded", durationMs: Date.now() - startedAt, statusCode: response.statusCode });
+    return;
+  }
+  emit({ status: "degraded", reason: "argo-refresh-patch-failed", durationMs: Date.now() - startedAt, statusCode: response.statusCode, body: response.body.slice(0, 1000) });
+})().catch((error) => {
+  emit({ status: "degraded", reason: "argo-refresh-patch-error", durationMs: Date.now() - startedAt, error: error.message });
+});
+NODE
+}
+
 if [ -s /workspace/source/dev-artifacts.json ]; then
   node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-json deploy/deploy.json --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write
 fi
@@ -1302,6 +1385,7 @@ if git diff --cached --quiet; then
   echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
   ci_timing_emit gitops-commit unchanged "$(ci_now_ms)"
   ci_timing_emit gitops-push unchanged "$(ci_now_ms)"
+  argo_hard_refresh_v02
   exit 0
 fi
 short="$(printf '%.7s' "$(params.revision)")"
@@ -1311,6 +1395,7 @@ ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms"
 gitops_push_started_ms="$(ci_now_ms)"
 git push origin "HEAD:$(params.gitops-branch)"
 ci_timing_emit gitops-push succeeded "$gitops_push_started_ms"
+argo_hard_refresh_v02
 echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
 `;
 }
@@ -2012,9 +2097,8 @@ function safeJson(text) {
 }
 
 if (!host) {
-  emit({ stage: "argo-refresh", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 });
+  emit({ stage: "argo-refresh", status: "failed", reason: "kubernetes-service-host-missing", durationMs: 0 });
-  emit({ stage: "workload-ready", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 });
+  process.exit(1);
-  process.exit(0);
 }
 
 const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
@@ -2081,7 +2165,8 @@ async function waitForArgoRefresh() {
   for (;;) {
     const runtime = await runtimeItems();
     if (runtime.status === "rbac-denied") {
-      emit({ stage: "argo-refresh", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
+      emit({ stage: "argo-refresh", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
+      process.exitCode = 1;
       return { observed: false, skipped: true };
     }
     const pending = runtime.items
@@ -2092,7 +2177,8 @@ async function waitForArgoRefresh() {
       return { observed: true, skipped: false };
     }
     if (Date.now() - startedAt >= timeoutMs) {
-      emit({ stage: "argo-refresh", status: "degraded", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) });
+      emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) });
+      process.exitCode = 1;
       return { observed: false, skipped: false };
     }
     await sleep(pollMs);
@@ -2104,7 +2190,8 @@ async function waitForWorkloads() {
   for (;;) {
     const runtime = await runtimeItems();
     if (runtime.status === "rbac-denied") {
-      emit({ stage: "workload-ready", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
+      emit({ stage: "workload-ready", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
+      process.exitCode = 1;
       return;
     }
     const blocked = runtime.items.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready);
@@ -2122,7 +2209,8 @@ async function waitForWorkloads() {
 }
 
 (async () => {
-  await waitForArgoRefresh();
+  const refresh = await waitForArgoRefresh();
+  if (!refresh.observed) return;
   await waitForWorkloads();
 })().catch((error) => {
   emit({ stage: "runtime-ready", status: "failed", reason: error.message, durationMs: 0 });

scripts/g14-gitops-render.test.ts

@@ -41,7 +41,14 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
 
     const runtimeReady = taskByName(pipelineJson, "runtime-ready");
     const runtimeReadyScript = runtimeReady.taskSpec.steps[0].script;
-    assert.match(runtimeReadyScript, /status: "degraded", reason: "source-commit-refresh-timeout"/u);
+    assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u);
+    assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n  if \(!refresh\.observed\) return;/u);
+
+    const gitopsPromote = taskByName(pipelineJson, "gitops-promote");
+    const gitopsPromoteScript = gitopsPromote.taskSpec.steps[0].script;
+    assert.match(gitopsPromoteScript, /argo_hard_refresh_v02\(\) \{/u);
+    assert.match(gitopsPromoteScript, /argocd\.argoproj\.io\/refresh": "hard"/u);
+    assert.match(gitopsPromoteScript, /applications\/" \+ encodeURIComponent\(application\) \+ "\?fieldManager=hwlab-v02-gitops-promote/u);
 
     const removedServiceIds = [
       "hwlab-router",