fix: fail closed v02 runtime readiness
This commit is contained in:
Codex
@@ -159,7 +159,8 @@ CI/CD 内部由 branch poller、PipelineRun、component planner、BuildKit publi
|
||||
- `v0.2-gitops` 中存在 `deploy/artifact-catalog.v02.json` 与 `deploy/gitops/g14/runtime-v02/**`。
|
||||
- `argocd/hwlab-g14-v02` 指向 `v0.2-gitops:deploy/gitops/g14/runtime-v02`，sync revision 与目标 GitOps revision 对齐。
|
||||
- `hwlab-v02` 中长驻 workload ready，没有把 DEV/PROD namespace 当成 `v0.2` 通过证据。
|
||||
- `runtime-ready` 必须以 workload Pod template 的 source commit 判断 Argo 刷新，并在 Argo 刷新超时或 workload 未就绪时失败；不得把 CrashLoop、未刷新或公网不可用的 runtime 标成绿色。
|
||||
- `gitops-promote` 推送 `v0.2-gitops` 后应触发 `argocd/hwlab-g14-v02` hard refresh，减少 GitOps push 与 Argo 仓库发现之间的漂移窗口；refresh 触发失败只作为低噪声事件输出，最终通过仍由 `runtime-ready` 判定。
|
||||
- `runtime-ready` 必须以 workload Pod template 的 source commit 判断 Argo 刷新，并在 Argo 刷新超时、observer RBAC 不足或 workload 未就绪时失败；不得把 CrashLoop、未刷新或公网不可用的 runtime 标成绿色。
|
||||
- `http://74.48.78.17:19666/` 返回 `v0.2` Cloud Web。
|
||||
- `http://74.48.78.17:19667/health/live` 返回 `v0.2` runtime health，payload 中的 namespace、revision 或 runtime identity 能与 `hwlab-v02`/`v0.2` 对齐。
|
||||
|
||||
@@ -189,7 +190,7 @@ GitOps branch 已更新、source branch render 通过、PipelineRun 名称存在
|
||||
| --- | --- | --- |
|
||||
| v02 独立 source/GitOps/runtime lane | 已实现 | `v0.2`、`v0.2-gitops`、`hwlab-v02` 和 `runtime-v02` 已固定。 |
|
||||
| Tekton poller/pipeline/promotion | 已实现 | 通过 `hwlab-v02-*` 对象和 GitOps promotion 管理。 |
|
||||
| v02 runtime readiness fail-closed | 已实现 | `runtime-ready` 读取 workload Pod template source commit，timeout 或未就绪会让 PipelineRun 失败。 |
|
||||
| v02 runtime readiness fail-closed | 已实现 | `gitops-promote` 推送后触发 Argo hard refresh；`runtime-ready` 读取 workload Pod template source commit，timeout、observer RBAC 不足或未就绪会让 PipelineRun 失败。 |
|
||||
| v02 裁撤服务不进入发布面 | 已实现 | v02 Tekton build service set、artifact catalog 和 runtime render 只包含保留服务；v02 Argo 开启 prune 清理旧 live 对象；裁撤服务仍可保留在 DEV legacy 源码中。 |
|
||||
| Argo v02 Application | 已实现 | `hwlab-g14-v02` 指向 v02 GitOps path 和 namespace。 |
|
||||
| FRP `19666/19667` 入口 | 已实现 | 由 `hwlab-v02-frpc` 与 master frps allowlist 共同提供。 |
|
||||
|
||||
@@ -1266,6 +1266,89 @@ git config --global user.email "hwlab-g14-gitops-bot@users.noreply.github.com"
|
||||
catalog_path="$(params.catalog-path)"
|
||||
runtime_path="$(params.runtime-path)"
|
||||
lane="$(params.lane)"
|
||||
|
||||
argo_hard_refresh_v02() {
|
||||
if [ "$lane" != "v02" ]; then return 0; fi
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
const startedAt = Date.now();
|
||||
const application = "hwlab-g14-v02";
|
||||
const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null;
|
||||
const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null;
|
||||
const task = process.env.HWLAB_TEKTON_TASK || null;
|
||||
const revision = process.env.HWLAB_SOURCE_REVISION || null;
|
||||
|
||||
function emit(payload) {
|
||||
console.log(JSON.stringify({
|
||||
event: "g14-cicd-timing",
|
||||
schemaVersion: "v1",
|
||||
stage: "argo-hard-refresh",
|
||||
pipelineRun,
|
||||
taskRun,
|
||||
task,
|
||||
revision,
|
||||
application,
|
||||
source: "scripts/g14-gitops-render.mjs",
|
||||
at: new Date().toISOString(),
|
||||
...payload
|
||||
}));
|
||||
}
|
||||
|
||||
function safeJson(text) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function request(method, path, body, contentType = "application/merge-patch+json") {
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = contentType;
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
if (!host) {
|
||||
emit({ status: "skipped", reason: "kubernetes-service-host-missing", durationMs: Date.now() - startedAt });
|
||||
return;
|
||||
}
|
||||
const response = await request(
|
||||
"PATCH",
|
||||
"/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(application) + "?fieldManager=hwlab-v02-gitops-promote",
|
||||
{ metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } }
|
||||
);
|
||||
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||
emit({ status: "succeeded", durationMs: Date.now() - startedAt, statusCode: response.statusCode });
|
||||
return;
|
||||
}
|
||||
emit({ status: "degraded", reason: "argo-refresh-patch-failed", durationMs: Date.now() - startedAt, statusCode: response.statusCode, body: response.body.slice(0, 1000) });
|
||||
})().catch((error) => {
|
||||
emit({ status: "degraded", reason: "argo-refresh-patch-error", durationMs: Date.now() - startedAt, error: error.message });
|
||||
});
|
||||
NODE
|
||||
}
|
||||
|
||||
if [ -s /workspace/source/dev-artifacts.json ]; then
|
||||
node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-json deploy/deploy.json --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write
|
||||
fi
|
||||
@@ -1302,6 +1385,7 @@ if git diff --cached --quiet; then
|
||||
echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
ci_timing_emit gitops-commit unchanged "$(ci_now_ms)"
|
||||
ci_timing_emit gitops-push unchanged "$(ci_now_ms)"
|
||||
argo_hard_refresh_v02
|
||||
exit 0
|
||||
fi
|
||||
short="$(printf '%.7s' "$(params.revision)")"
|
||||
@@ -1311,6 +1395,7 @@ ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms"
|
||||
gitops_push_started_ms="$(ci_now_ms)"
|
||||
git push origin "HEAD:$(params.gitops-branch)"
|
||||
ci_timing_emit gitops-push succeeded "$gitops_push_started_ms"
|
||||
argo_hard_refresh_v02
|
||||
echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
`;
|
||||
}
|
||||
@@ -2012,9 +2097,8 @@ function safeJson(text) {
|
||||
}
|
||||
|
||||
if (!host) {
|
||||
emit({ stage: "argo-refresh", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 });
|
||||
emit({ stage: "workload-ready", status: "skipped", reason: "kubernetes-service-host-missing", durationMs: 0 });
|
||||
process.exit(0);
|
||||
emit({ stage: "argo-refresh", status: "failed", reason: "kubernetes-service-host-missing", durationMs: 0 });
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
@@ -2081,7 +2165,8 @@ async function waitForArgoRefresh() {
|
||||
for (;;) {
|
||||
const runtime = await runtimeItems();
|
||||
if (runtime.status === "rbac-denied") {
|
||||
emit({ stage: "argo-refresh", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
|
||||
emit({ stage: "argo-refresh", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
|
||||
process.exitCode = 1;
|
||||
return { observed: false, skipped: true };
|
||||
}
|
||||
const pending = runtime.items
|
||||
@@ -2092,7 +2177,8 @@ async function waitForArgoRefresh() {
|
||||
return { observed: true, skipped: false };
|
||||
}
|
||||
if (Date.now() - startedAt >= timeoutMs) {
|
||||
emit({ stage: "argo-refresh", status: "degraded", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) });
|
||||
emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) });
|
||||
process.exitCode = 1;
|
||||
return { observed: false, skipped: false };
|
||||
}
|
||||
await sleep(pollMs);
|
||||
@@ -2104,7 +2190,8 @@ async function waitForWorkloads() {
|
||||
for (;;) {
|
||||
const runtime = await runtimeItems();
|
||||
if (runtime.status === "rbac-denied") {
|
||||
emit({ stage: "workload-ready", status: "skipped", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
|
||||
emit({ stage: "workload-ready", status: "failed", reason: "runtime-observer-rbac-denied", durationMs: Date.now() - startedAt });
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const blocked = runtime.items.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready);
|
||||
@@ -2122,7 +2209,8 @@ async function waitForWorkloads() {
|
||||
}
|
||||
|
||||
(async () => {
|
||||
await waitForArgoRefresh();
|
||||
const refresh = await waitForArgoRefresh();
|
||||
if (!refresh.observed) return;
|
||||
await waitForWorkloads();
|
||||
})().catch((error) => {
|
||||
emit({ stage: "runtime-ready", status: "failed", reason: error.message, durationMs: 0 });
|
||||
|
||||
@@ -41,7 +41,14 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
|
||||
|
||||
const runtimeReady = taskByName(pipelineJson, "runtime-ready");
|
||||
const runtimeReadyScript = runtimeReady.taskSpec.steps[0].script;
|
||||
assert.match(runtimeReadyScript, /status: "degraded", reason: "source-commit-refresh-timeout"/u);
|
||||
assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u);
|
||||
assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u);
|
||||
|
||||
const gitopsPromote = taskByName(pipelineJson, "gitops-promote");
|
||||
const gitopsPromoteScript = gitopsPromote.taskSpec.steps[0].script;
|
||||
assert.match(gitopsPromoteScript, /argo_hard_refresh_v02\(\) \{/u);
|
||||
assert.match(gitopsPromoteScript, /argocd\.argoproj\.io\/refresh": "hard"/u);
|
||||
assert.match(gitopsPromoteScript, /applications\/" \+ encodeURIComponent\(application\) \+ "\?fieldManager=hwlab-v02-gitops-promote/u);
|
||||
|
||||
const removedServiceIds = [
|
||||
"hwlab-router",
|
||||
|
||||
Reference in New Issue
Block a user