mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #615 from pikasTech/fix/v02-runtime-ready-visibility

Browse Source 
fix: expose v02 runtime readiness progress
This commit is contained in:
Lyon
2026-05-31 12:52:24 +08:00
committed by GitHub
parent 88b1e28505 c64c9a0455
commit 6bc24ff360
3 changed files with 77 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/reference/spec-v02-cicd.md
+2
View File
@@ -219,6 +219,8 @@ P1 no-op runtime skip 只跳过无实际 runtime 变化的等待。planner 输
`runtime-ready` 只能观察本轮 `rolloutServices` 对应的 workload readiness，不能把整个 `hwlab-v02` namespace 的全量 workload 都绑定到当前 source commit 后等待。复用 service 的 Deployment metadata 可以记录本次 source commit 作为 GitOps revision 线索，但 Pod template identity 必须来自该 service 的 artifact/runtime commit、boot commit 或配置内容 hash；否则 catalog 复用会被误转成不必要 rollout，`runtime-ready` 会被无关服务拖慢。`hwlab-cloud-api` 的 rollout 允许连带观察 `hwlab-deepseek-proxy`，因为 deepseek bridge 复用 cloud-api 镜像作为运行载体；其他连带关系必须有明确运行依赖后再加入观察集合。基础对象如 FRP、Postgres 的 Pod template 不得写入全局 source commit，应用配置 hash 或 migration hash 表达真实重启边界。
`runtime-ready` 的可见性优先级高于耗时优化。Task 启动后必须立即输出 `runtime-ready started` 事件，明确 `readinessMode`、观察服务数和当前 revision；等待 Argo、source commit refresh 或 workload ready 时必须周期性输出 `progress` 事件，包含 `observedCount`、`workloadCount`、pending/blocked 摘要或 Argo sync/health 状态。service rollout 场景继续用 `rolloutServices` 对应的 Pod template source commit 判定；`rolloutServices=[]` 但 GitOps runtime 真实变化的 infra-only 场景不得黑盒等待 source commit，应改为等待 Argo Application `Synced/Healthy` 和 workload generation ready，并输出 `argo-sync-health` 事件。禁止再出现 runtime-ready 只在 240s timeout 后才输出一条日志的不可观察状态。
Kubernetes label 里的配置 hash 必须使用短 hash，完整 64 位 sha256 只能放 annotation。FRP `config-sha256`、Postgres `migration-sha256` 这类字段如果进入 Pod template label，长度必须小于 Kubernetes label value 的 63 字节上限；annotation 保留完整 hash 用于排障和人工比对。Argo sync 报 `must be no more than 63 bytes` 时，优先检查是否把完整 sha256 写入 label，不要通过删除 hash、放宽 sync 或忽略 OutOfSync 绕过。
快速优化不能绕过 fail-closed 语义。mirror miss、commit ancestry 不合法、boot script 缺失、digest 缺失、Argo observer RBAC 不足、workload 未 ready 或公网 health 不一致，都不能为了追求耗时而降级为 warning。允许跳过的只有已经证明 runtime desired state 没有实际变化的等待。
scripts/g14-gitops-render.mjs
+67 -3
View File
@@ -2368,11 +2368,13 @@ function requiredEnv(name) {
const revision = requiredEnv("HWLAB_SOURCE_REVISION");
const runtimeNamespace = requiredEnv("HWLAB_RUNTIME_NAMESPACE");
const gitopsTarget = requiredEnv("HWLAB_GITOPS_TARGET");
const argoApplication = requiredEnv("HWLAB_ARGO_APPLICATION");
const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null;
const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null;
const task = process.env.HWLAB_TEKTON_TASK || null;
const timeoutMs = 240000;
const pollMs = 1000;
const progressEveryMs = 10000;
function emit(payload) {
console.log(JSON.stringify({
@@ -2474,6 +2476,7 @@ function readObservedServiceIds() {
}
const observedServiceIds = readObservedServiceIds();
const readinessMode = observedServiceIds ? "service-rollout" : "infra-runtime-change";
function observedRuntimeItems(items) {
if (!observedServiceIds) return items;
@@ -2492,8 +2495,57 @@ function readyWorkload(item) {
return { ready: observedGeneration >= generation && ready >= desired && updated >= desired, desired, readyReplicas: ready, updatedReplicas: updated };
}
function maybeEmitProgress(state, stage, startedAt, payload) {
const now = Date.now();
if (now - state.lastProgressAt < progressEveryMs) return;
state.lastProgressAt = now;
emit({ stage, status: "progress", durationMs: now - startedAt, readinessMode, ...payload });
}
async function argoApplicationState() {
const path = "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(argoApplication);
const response = await request("GET", path);
if (response.statusCode === 403) return { status: "rbac-denied", item: null, statusCode: response.statusCode };
if (response.statusCode < 200 || response.statusCode > 299) return { status: "error", item: null, statusCode: response.statusCode, body: response.body.slice(0, 500) };
return { status: "ok", item: response.json, statusCode: response.statusCode };
}
async function waitForArgoSyncedHealthy() {
const startedAt = Date.now();
const progressState = { lastProgressAt: 0 };
for (;;) {
const application = await argoApplicationState();
if (application.status === "rbac-denied") {
emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-rbac-denied", durationMs: Date.now() - startedAt, readinessMode });
process.exitCode = 1;
return { observed: false };
}
if (application.status !== "ok") {
emit({ stage: "argo-sync-health", status: "failed", reason: "argocd-application-read-failed", statusCode: application.statusCode, body: application.body || null, durationMs: Date.now() - startedAt, readinessMode });
process.exitCode = 1;
return { observed: false };
}
const syncStatus = application.item?.status?.sync?.status || null;
const healthStatus = application.item?.status?.health?.status || null;
const operationPhase = application.item?.status?.operationState?.phase || null;
const operationMessage = application.item?.status?.operationState?.message || null;
if (syncStatus === "Synced" && healthStatus === "Healthy") {
emit({ stage: "argo-sync-health", status: "succeeded", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase });
return { observed: true };
}
if (Date.now() - startedAt >= timeoutMs) {
emit({ stage: "argo-sync-health", status: "timeout", durationMs: Date.now() - startedAt, readinessMode, syncStatus, healthStatus, operationPhase, operationMessage });
process.exitCode = 1;
return { observed: false };
}
maybeEmitProgress(progressState, "argo-sync-health", startedAt, { syncStatus, healthStatus, operationPhase, operationMessage });
await sleep(pollMs);
}
}
async function waitForArgoRefresh() {
const startedAt = Date.now();
const progressState = { lastProgressAt: 0 };
for (;;) {
const runtime = await runtimeItems();
if (runtime.status === "rbac-denied") {
@@ -2514,12 +2566,14 @@ async function waitForArgoRefresh() {
process.exitCode = 1;
return { observed: false, skipped: false };
}
maybeEmitProgress(progressState, "argo-refresh", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) });
await sleep(pollMs);
}
}
async function waitForWorkloads() {
const startedAt = Date.now();
const progressState = { lastProgressAt: 0 };
for (;;) {
const runtime = await runtimeItems();
if (runtime.status === "rbac-denied") {
@@ -2538,11 +2592,19 @@ async function waitForWorkloads() {
process.exitCode = 1;
return;
}
maybeEmitProgress(progressState, "workload-ready", startedAt, { workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) });
await sleep(pollMs);
}
}
(async () => {
emit({ stage: "runtime-ready", status: "started", readinessMode, observedServiceCount: observedServiceIds ? observedServiceIds.size : 0 });
if (!observedServiceIds) {
const argo = await waitForArgoSyncedHealthy();
if (!argo.observed) return;
await waitForWorkloads();
return;
}
const refresh = await waitForArgoRefresh();
if (!refresh.observed) return;
await waitForWorkloads();
@@ -2560,7 +2622,7 @@ function runtimeReadyTask({ profile = "dev" } = {}) {
runAfter: ["gitops-promote"],
workspaces: [{ name: "source", workspace: "source" }],
taskSpec: {
params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }],
params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }],
workspaces: [{ name: "source" }],
steps: [{
name: "runtime-ready",
@@ -2568,7 +2630,8 @@ function runtimeReadyTask({ profile = "dev" } = {}) {
env: [
...proxyEnv(),
{ name: "HWLAB_RUNTIME_NAMESPACE", value: "$(params.runtime-namespace)" },
{ name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" }
{ name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" },
{ name: "HWLAB_ARGO_APPLICATION", value: "$(params.argo-application)" }
],
script: runtimeReadyScript()
}]
@@ -2576,7 +2639,8 @@ function runtimeReadyTask({ profile = "dev" } = {}) {
params: [
{ name: "revision", value: "$(params.revision)" },
{ name: "runtime-namespace", value: namespaceNameForProfile(profile) },
{ name: "gitops-target", value: gitopsTargetForProfile(profile) }
{ name: "gitops-target", value: gitopsTargetForProfile(profile) },
{ name: "argo-application", value: argoApplicationName(profile) }
]
};
}
scripts/g14-gitops-render.test.ts
+8
View File
@@ -55,9 +55,17 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
assert.match(runtimeReadyScript, /readObservedServiceIds/u);
assert.match(runtimeReadyScript, /observedRuntimeItems/u);
assert.match(runtimeReadyScript, /observedCount/u);
assert.match(runtimeReadyScript, /readinessMode = observedServiceIds \? "service-rollout" : "infra-runtime-change"/u);
assert.match(runtimeReadyScript, /waitForArgoSyncedHealthy/u);
assert.match(runtimeReadyScript, /stage: "runtime-ready", status: "started"/u);
assert.match(runtimeReadyScript, /status: "progress"/u);
assert.match(runtimeReadyScript, /stage: "argo-sync-health"/u);
assert.match(runtimeReadyScript, /ids\.has\("hwlab-cloud-api"\)\) ids\.add\("hwlab-deepseek-proxy"\)/u);
assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u);
assert.match(runtimeReadyScript, /if \(!observedServiceIds\) \{/u);
assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u);
assert.ok(runtimeReady.taskSpec.params.some((param) => param.name === "argo-application"));
assert.ok(runtimeReady.params.some((param) => param.name === "argo-application" && param.value === "hwlab-g14-v02"));
assert.ok(runtimeReady.workspaces?.some((workspace) => workspace.name === "source" && workspace.workspace === "source"));
assert.ok(runtimeReady.taskSpec.workspaces?.some((workspace) => workspace.name === "source"));