From f937da15588ced2bf4b8fdd615e213c1a2f556d9 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 31 May 2026 12:16:08 +0800 Subject: [PATCH] fix: limit v02 runtime readiness scope --- docs/reference/spec-v02-cicd.md | 4 ++ scripts/artifact-publish.mjs | 11 +-- scripts/g14-gitops-render.mjs | 112 +++++++++++++++++++++--------- scripts/g14-gitops-render.test.ts | 15 ++++ 4 files changed, 107 insertions(+), 35 deletions(-) diff --git a/docs/reference/spec-v02-cicd.md b/docs/reference/spec-v02-cicd.md index f9c93f49..2c15f789 100644 --- a/docs/reference/spec-v02-cicd.md +++ b/docs/reference/spec-v02-cicd.md @@ -166,6 +166,8 @@ Cloud Web 单元测试必须自动发现并执行 repo-owned `web/hwlab-cloud-we | `gitops-promote` no-op | 约 7-9s | 未输出 `skipped-runtime-unchanged`，或写入了 `v0.2-gitops`，说明 runtime 比对未命中。 | | `runtime-ready` | no-op 应跳过；真实 rollout 约 15-20s | no-op 场景出现 TaskRun 即为 P1 退化；真实 rollout 超时则按 Argo/workload 排障。 | +真实 rollout 若超过约 20s，先区分是 workload 本身启动慢，还是 `runtime-ready` 观察集合过大。正常日志应带 `observedCount`，且该值应接近本轮 `rolloutServices` 数量加上明确连带依赖；如果 `workloadCount` 很大但 `observedCount` 缺失，或 FRP、Postgres、device-pod 在无关 cloud-web/cloud-api 变更中重启，通常说明 render 把复用服务的 Pod template 绑定到了全局 source commit。修复方向是收窄 `runtime-ready` 观察集合，并把复用服务 template identity 改成 artifact commit、boot commit、config hash 或 migration hash，而不是加大 timeout 或恢复全 namespace 等待。 + ## 性能优化原理 `prepare-source` 不安装依赖。该 task 只负责 checkout source、读取上一版 catalog、输出 source identity 和基础文件，不运行 `npm ci`、不探测 npm registry、不做与后续 task 重复的工具链检查。需要依赖的语法检查、render test 或 build task 必须使用各自镜像内已有工具或在专属阶段处理，不能把全仓依赖安装塞回 source 准备阶段。 @@ -215,6 +217,8 @@ planner 必须按 service component model 裁剪 build。`tools/` 下的短连 P1 no-op runtime skip 只跳过无实际 runtime 变化的等待。planner 输出 `buildServices=[]` 且 `rolloutServices=[]` 后，`gitops-promote` 会在写入前对旧 `runtime-v02` 与新 render 结果做归一化比较：忽略 source commit、artifact source commit、boot commit 和等价 commit env/annotation 这类 identity-only 字段。如果归一化结果相同，promotion 输出 `skipped-runtime-unchanged`，把 Tekton result `runtime-ready-required=false` 写出，并跳过 GitOps commit、push、Argo hard refresh 和 `runtime-ready`。如果 workload spec、image digest、env image、SecretRef、Service、Ingress、FRP、ConfigMap 或 rollout service 有实际变化，必须保持 `runtime-ready-required=true` 并等待 runtime 收敛。 +`runtime-ready` 只能观察本轮 `rolloutServices` 对应的 workload readiness，不能把整个 `hwlab-v02` namespace 的全量 workload 都绑定到当前 source commit 后等待。复用 service 的 Deployment metadata 可以记录本次 source commit 作为 GitOps revision 线索，但 Pod template identity 必须来自该 service 的 artifact/runtime commit、boot commit 或配置内容 hash；否则 catalog 复用会被误转成不必要 rollout，`runtime-ready` 会被无关服务拖慢。`hwlab-cloud-api` 的 rollout 允许连带观察 `hwlab-deepseek-proxy`，因为 deepseek bridge 复用 cloud-api 镜像作为运行载体；其他连带关系必须有明确运行依赖后再加入观察集合。基础对象如 FRP、Postgres 的 Pod template 不得写入全局 source commit，应用配置 hash 或 migration hash 表达真实重启边界。 + 快速优化不能绕过 fail-closed 语义。mirror miss、commit ancestry 不合法、boot script 缺失、digest 缺失、Argo observer RBAC 不足、workload 未 ready 或公网 health 不一致，都不能为了追求耗时而降级为 warning。允许跳过的只有已经证明 runtime desired state 没有实际变化的等待。 ## 快速排障 diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index 43b31a3b..335fb7e4 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -1501,14 +1501,17 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit const environmentImage = catalogService?.environmentImage ?? service.environmentImage ?? null; const environmentDigest = catalogService?.environmentDigest ?? service.environmentDigest ?? "not_published"; const imageTag = imageTagFromReference(environmentImage); + const bootCommit = service.codeChanged === true || service.envChanged === true + ? service.bootCommit ?? commitId + : catalogService?.bootCommit ?? service.bootCommit ?? commitId; const base = { ...service, runtimeMode: v02EnvReuseRuntimeMode, envReuse: true, envChanged: service.envChanged ?? null, codeChanged: service.codeChanged ?? null, - commitId, - sourceCommitId: commitId, + commitId: bootCommit, + sourceCommitId: bootCommit, image: environmentImage, imageTag, digest: environmentDigest, @@ -1517,13 +1520,13 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit environmentInputHash: service.environmentInputHash ?? catalogService?.environmentInputHash ?? null, codeInputHash: service.codeInputHash ?? null, bootRepo: service.bootRepo ?? catalogService?.bootRepo ?? "git@github.com:pikasTech/HWLAB.git", - bootCommit: service.bootCommit ?? commitId, + bootCommit, bootSh: service.bootSh ?? catalogService?.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`, buildCreatedAt: catalogService?.buildCreatedAt ?? null, buildSource: catalogService?.buildSource ?? null, buildBackend: "reused-env-catalog", repositoryDigest: catalogService?.repositoryDigest ?? repositoryDigestFor(environmentImage, environmentDigest), - bootEnvEvidence: bootEnvEvidence({ service, commitId, environmentImage, environmentDigest }) + bootEnvEvidence: bootEnvEvidence({ service: { ...service, bootCommit }, commitId: bootCommit, environmentImage, environmentDigest }) }; if (!environmentImage || !imageTag || !shaDigestPattern.test(environmentDigest)) { return { diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index 9ce59263..994af532 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -2440,13 +2440,41 @@ async function runtimeItems() { } function runtimeSourceCommit(item) { - return item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || - item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || - item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || + return item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || + item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] || + item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] || null; } +function serviceIdForRuntimeItem(item) { + return item.metadata?.labels?.["hwlab.pikastech.local/service-id"] || + item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || + item.metadata?.labels?.["app.kubernetes.io/name"] || + item.metadata?.name || + null; +} + +function readObservedServiceIds() { + try { + const plan = JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")); + const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices.filter(Boolean) : []; + if (rolloutServices.length === 0) return null; + const ids = new Set(rolloutServices); + if (ids.has("hwlab-cloud-api")) ids.add("hwlab-deepseek-proxy"); + return ids; + } catch { + return null; + } +} + +const observedServiceIds = readObservedServiceIds(); + +function observedRuntimeItems(items) { + if (!observedServiceIds) return items; + return items.filter((item) => observedServiceIds.has(serviceIdForRuntimeItem(item))); +} + function readyWorkload(item) { const specReplicas = item.spec?.replicas ?? 1; if (specReplicas === 0) return { ready: true, skipped: true }; @@ -2468,15 +2496,16 @@ async function waitForArgoRefresh() { process.exitCode = 1; return { observed: false, skipped: true }; } - const pending = runtime.items + const observed = observedRuntimeItems(runtime.items); + const pending = observed .map((item) => ({ kind: item.kind, name: item.metadata?.name, sourceCommit: runtimeSourceCommit(item) })) .filter((item) => item.sourceCommit !== revision); - if (runtime.items.length > 0 && pending.length === 0) { - emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length }); + if (observed.length > 0 && pending.length === 0) { + emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length }); return { observed: true, skipped: false }; } if (Date.now() - startedAt >= timeoutMs) { - emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) }); + emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) }); process.exitCode = 1; return { observed: false, skipped: false }; } @@ -2493,13 +2522,14 @@ async function waitForWorkloads() { process.exitCode = 1; return; } - const blocked = runtime.items.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready); - if (runtime.items.length > 0 && blocked.length === 0) { - emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length }); + const observed = observedRuntimeItems(runtime.items); + const blocked = observed.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready); + if (observed.length > 0 && blocked.length === 0) { + emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length }); return; } if (Date.now() - startedAt >= timeoutMs) { - emit({ stage: "workload-ready", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, blocked: blocked.slice(0, 8) }); + emit({ stage: "workload-ready", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) }); process.exitCode = 1; return; } @@ -2523,8 +2553,10 @@ function runtimeReadyTask({ profile = "dev" } = {}) { return { name: "runtime-ready", runAfter: ["gitops-promote"], + workspaces: [{ name: "source", workspace: "source" }], taskSpec: { params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }], + workspaces: [{ name: "source" }], steps: [{ name: "runtime-ready", image: ciToolsRunnerImage, @@ -3529,20 +3561,7 @@ function g14FrpcManifest({ profile = "dev", source = null } = {}) { }; const sourceCommitLabel = source?.full ? { "hwlab.pikastech.local/source-commit": source.full } : {}; const renderedLabels = { ...labels, ...sourceCommitLabel }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { - name: configName, - namespace, - labels: renderedLabels - }, - data: { - "frpc.toml": `serverAddr = "74.48.78.17" + const configText = `serverAddr = "74.48.78.17" serverPort = 7000 loginFailExit = true @@ -3559,7 +3578,24 @@ type = "tcp" localIP = "hwlab-edge-proxy.${namespace}.svc.cluster.local" localPort = 6667 remotePort = ${edgeRemotePort} -` +`; + const configSha256 = createHash("sha256").update(configText).digest("hex"); + const templateLabels = { ...labels, "hwlab.pikastech.local/config-sha256": configSha256 }; + const templateAnnotations = { "hwlab.pikastech.local/config-sha256": configSha256 }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: configName, + namespace, + labels: renderedLabels + }, + data: { + "frpc.toml": configText } }, { @@ -3574,7 +3610,7 @@ remotePort = ${edgeRemotePort} replicas: 1, selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, template: { - metadata: { labels: renderedLabels, annotations: sourceCommitLabel }, + metadata: { labels: templateLabels, annotations: templateAnnotations }, spec: { containers: [{ name: "frpc", @@ -3607,9 +3643,14 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau "hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy", "hwlab.pikastech.local/source-commit": source.full }; + const templateLabels = { + ...labels, + "hwlab.pikastech.local/source-commit": bridgeCommit, + "hwlab.pikastech.local/bridge-source-commit": bridgeCommit + }; const templateAnnotations = { "hwlab.pikastech.local/bridge-service-id": bridgeServiceId, - "hwlab.pikastech.local/source-commit": source.full, + "hwlab.pikastech.local/source-commit": bridgeCommit, "hwlab.pikastech.local/bridge-source-commit": bridgeCommit, "hwlab.pikastech.local/bridge-image-tag": bridgeImageTag, "hwlab.pikastech.local/bridge-image": bridgeImage @@ -3634,7 +3675,7 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau replicas: 1, selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" } }, template: { - metadata: { labels, annotations: templateAnnotations }, + metadata: { labels: templateLabels, annotations: templateAnnotations }, spec: { initContainers: [{ name: "moonbridge-config", @@ -3988,7 +4029,16 @@ function v02PostgresManifest({ migrationSql, source }) { "hwlab.pikastech.local/profile": "v02", "hwlab.pikastech.local/source-commit": source.full }; - const templateAnnotations = { "hwlab.pikastech.local/source-commit": source.full }; + const migrationSha256 = createHash("sha256").update(migrationSql).digest("hex"); + const templateLabels = { + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/environment": "v02", + "hwlab.pikastech.local/gitops-target": "v02", + "hwlab.pikastech.local/profile": "v02", + "hwlab.pikastech.local/migration-sha256": migrationSha256 + }; + const templateAnnotations = { "hwlab.pikastech.local/migration-sha256": migrationSha256 }; return { apiVersion: "v1", kind: "List", @@ -4014,7 +4064,7 @@ function v02PostgresManifest({ migrationSql, source }) { replicas: 1, selector: { matchLabels: { "app.kubernetes.io/name": name } }, template: { - metadata: { labels, annotations: templateAnnotations }, + metadata: { labels: templateLabels, annotations: templateAnnotations }, spec: { containers: [{ name: "postgres", diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index ff92211e..c40cd23d 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -52,8 +52,14 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.deepEqual(runtimeReady.when, [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }]); const runtimeReadyScript = runtimeReady.taskSpec.steps[0].script; assert.match(runtimeReadyScript, /const pollMs = 1000;/u); + assert.match(runtimeReadyScript, /readObservedServiceIds/u); + assert.match(runtimeReadyScript, /observedRuntimeItems/u); + assert.match(runtimeReadyScript, /observedCount/u); + assert.match(runtimeReadyScript, /ids\.has\("hwlab-cloud-api"\)\) ids\.add\("hwlab-deepseek-proxy"\)/u); assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u); assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u); + assert.ok(runtimeReady.workspaces?.some((workspace) => workspace.name === "source" && workspace.workspace === "source")); + assert.ok(runtimeReady.taskSpec.workspaces?.some((workspace) => workspace.name === "source")); const gitopsPromote = taskByName(pipelineJson, "gitops-promote"); assert.ok(gitopsPromote.taskSpec.results.some((result) => result.name === "runtime-ready-required")); @@ -235,6 +241,15 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots const content = await readFile(path.join(outDir, "runtime-v02", generatedFile), "utf8"); assert.match(content, /"hwlab\.pikastech\.local\/source-commit"/u); } + const frpc = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "g14-frpc.yaml"), "utf8")); + const frpcDeployment = frpc.items.find((item) => item.kind === "Deployment"); + assert.ok(frpcDeployment.spec.template.metadata.labels["hwlab.pikastech.local/config-sha256"]); + assert.equal(frpcDeployment.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined); + + const postgres = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "postgres.yaml"), "utf8")); + const postgresStatefulSet = postgres.items.find((item) => item.kind === "StatefulSet"); + assert.ok(postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/migration-sha256"]); + assert.equal(postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined); await assert.rejects(() => readFile(path.join(outDir, "tekton-v02", "control-plane-reconciler.yaml"), "utf8"), { code: "ENOENT" }); await assert.rejects(() => readFile(path.join(outDir, "tekton-v02", "poller.yaml"), "utf8"), { code: "ENOENT" });