mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #613 from pikasTech/fix/v02-runtime-identity-reuse

Browse Source 
fix: limit v02 runtime readiness scope
This commit is contained in:
Lyon
2026-05-31 12:17:10 +08:00
committed by GitHub
parent 53ffdf386a f937da1558
commit 794bea65e8
4 changed files with 107 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/reference/spec-v02-cicd.md
+4
View File
@@ -166,6 +166,8 @@ Cloud Web 单元测试必须自动发现并执行 repo-owned `web/hwlab-cloud-we
| `gitops-promote` no-op | 约 7-9s | 未输出 `skipped-runtime-unchanged`，或写入了 `v0.2-gitops`，说明 runtime 比对未命中。 |
| `runtime-ready` | no-op 应跳过；真实 rollout 约 15-20s | no-op 场景出现 TaskRun 即为 P1 退化；真实 rollout 超时则按 Argo/workload 排障。 |
真实 rollout 若超过约 20s，先区分是 workload 本身启动慢，还是 `runtime-ready` 观察集合过大。正常日志应带 `observedCount`，且该值应接近本轮 `rolloutServices` 数量加上明确连带依赖；如果 `workloadCount` 很大但 `observedCount` 缺失，或 FRP、Postgres、device-pod 在无关 cloud-web/cloud-api 变更中重启，通常说明 render 把复用服务的 Pod template 绑定到了全局 source commit。修复方向是收窄 `runtime-ready` 观察集合，并把复用服务 template identity 改成 artifact commit、boot commit、config hash 或 migration hash，而不是加大 timeout 或恢复全 namespace 等待。
## 性能优化原理
`prepare-source` 不安装依赖。该 task 只负责 checkout source、读取上一版 catalog、输出 source identity 和基础文件，不运行 `npm ci`、不探测 npm registry、不做与后续 task 重复的工具链检查。需要依赖的语法检查、render test 或 build task 必须使用各自镜像内已有工具或在专属阶段处理，不能把全仓依赖安装塞回 source 准备阶段。
@@ -215,6 +217,8 @@ planner 必须按 service component model 裁剪 build。`tools/` 下的短连
P1 no-op runtime skip 只跳过无实际 runtime 变化的等待。planner 输出 `buildServices=[]` 且 `rolloutServices=[]` 后，`gitops-promote` 会在写入前对旧 `runtime-v02` 与新 render 结果做归一化比较：忽略 source commit、artifact source commit、boot commit 和等价 commit env/annotation 这类 identity-only 字段。如果归一化结果相同，promotion 输出 `skipped-runtime-unchanged`，把 Tekton result `runtime-ready-required=false` 写出，并跳过 GitOps commit、push、Argo hard refresh 和 `runtime-ready`。如果 workload spec、image digest、env image、SecretRef、Service、Ingress、FRP、ConfigMap 或 rollout service 有实际变化，必须保持 `runtime-ready-required=true` 并等待 runtime 收敛。
`runtime-ready` 只能观察本轮 `rolloutServices` 对应的 workload readiness，不能把整个 `hwlab-v02` namespace 的全量 workload 都绑定到当前 source commit 后等待。复用 service 的 Deployment metadata 可以记录本次 source commit 作为 GitOps revision 线索，但 Pod template identity 必须来自该 service 的 artifact/runtime commit、boot commit 或配置内容 hash；否则 catalog 复用会被误转成不必要 rollout，`runtime-ready` 会被无关服务拖慢。`hwlab-cloud-api` 的 rollout 允许连带观察 `hwlab-deepseek-proxy`，因为 deepseek bridge 复用 cloud-api 镜像作为运行载体；其他连带关系必须有明确运行依赖后再加入观察集合。基础对象如 FRP、Postgres 的 Pod template 不得写入全局 source commit，应用配置 hash 或 migration hash 表达真实重启边界。
快速优化不能绕过 fail-closed 语义。mirror miss、commit ancestry 不合法、boot script 缺失、digest 缺失、Argo observer RBAC 不足、workload 未 ready 或公网 health 不一致，都不能为了追求耗时而降级为 warning。允许跳过的只有已经证明 runtime desired state 没有实际变化的等待。
## 快速排障
scripts/artifact-publish.mjs
+7 -4
View File
@@ -1501,14 +1501,17 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit
const environmentImage = catalogService?.environmentImage ?? service.environmentImage ?? null;
const environmentDigest = catalogService?.environmentDigest ?? service.environmentDigest ?? "not_published";
const imageTag = imageTagFromReference(environmentImage);
const bootCommit = service.codeChanged === true || service.envChanged === true
? service.bootCommit ?? commitId
: catalogService?.bootCommit ?? service.bootCommit ?? commitId;
const base = {
...service,
runtimeMode: v02EnvReuseRuntimeMode,
envReuse: true,
envChanged: service.envChanged ?? null,
codeChanged: service.codeChanged ?? null,
commitId,
sourceCommitId: commitId,
commitId: bootCommit,
sourceCommitId: bootCommit,
image: environmentImage,
imageTag,
digest: environmentDigest,
@@ -1517,13 +1520,13 @@ function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commit
environmentInputHash: service.environmentInputHash ?? catalogService?.environmentInputHash ?? null,
codeInputHash: service.codeInputHash ?? null,
bootRepo: service.bootRepo ?? catalogService?.bootRepo ?? "git@github.com:pikasTech/HWLAB.git",
bootCommit: service.bootCommit ?? commitId,
bootCommit,
bootSh: service.bootSh ?? catalogService?.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`,
buildCreatedAt: catalogService?.buildCreatedAt ?? null,
buildSource: catalogService?.buildSource ?? null,
buildBackend: "reused-env-catalog",
repositoryDigest: catalogService?.repositoryDigest ?? repositoryDigestFor(environmentImage, environmentDigest),
bootEnvEvidence: bootEnvEvidence({ service, commitId, environmentImage, environmentDigest })
bootEnvEvidence: bootEnvEvidence({ service: { ...service, bootCommit }, commitId: bootCommit, environmentImage, environmentDigest })
};
if (!environmentImage || !imageTag || !shaDigestPattern.test(environmentDigest)) {
return {
scripts/g14-gitops-render.mjs
+81 -31
View File
@@ -2440,13 +2440,41 @@ async function runtimeItems() {
}
function runtimeSourceCommit(item) {
return item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] ||
item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
return item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
item.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] ||
item.metadata?.labels?.["hwlab.pikastech.local/source-commit"] ||
item.metadata?.annotations?.["hwlab.pikastech.local/source-commit"] ||
null;
}
function serviceIdForRuntimeItem(item) {
return item.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
item.metadata?.labels?.["app.kubernetes.io/name"] ||
item.metadata?.name ||
null;
}
function readObservedServiceIds() {
try {
const plan = JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8"));
const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices.filter(Boolean) : [];
if (rolloutServices.length === 0) return null;
const ids = new Set(rolloutServices);
if (ids.has("hwlab-cloud-api")) ids.add("hwlab-deepseek-proxy");
return ids;
} catch {
return null;
}
}
const observedServiceIds = readObservedServiceIds();
function observedRuntimeItems(items) {
if (!observedServiceIds) return items;
return items.filter((item) => observedServiceIds.has(serviceIdForRuntimeItem(item)));
}
function readyWorkload(item) {
const specReplicas = item.spec?.replicas ?? 1;
if (specReplicas === 0) return { ready: true, skipped: true };
@@ -2468,15 +2496,16 @@ async function waitForArgoRefresh() {
process.exitCode = 1;
return { observed: false, skipped: true };
}
const pending = runtime.items
const observed = observedRuntimeItems(runtime.items);
const pending = observed
.map((item) => ({ kind: item.kind, name: item.metadata?.name, sourceCommit: runtimeSourceCommit(item) }))
.filter((item) => item.sourceCommit !== revision);
if (runtime.items.length > 0 && pending.length === 0) {
emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length });
if (observed.length > 0 && pending.length === 0) {
emit({ stage: "argo-refresh", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length });
return { observed: true, skipped: false };
}
if (Date.now() - startedAt >= timeoutMs) {
emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, pending: pending.slice(0, 8) });
emit({ stage: "argo-refresh", status: "timeout", reason: "source-commit-refresh-timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, pending: pending.slice(0, 8) });
process.exitCode = 1;
return { observed: false, skipped: false };
}
@@ -2493,13 +2522,14 @@ async function waitForWorkloads() {
process.exitCode = 1;
return;
}
const blocked = runtime.items.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready);
if (runtime.items.length > 0 && blocked.length === 0) {
emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length });
const observed = observedRuntimeItems(runtime.items);
const blocked = observed.map((item) => ({ kind: item.kind, name: item.metadata?.name, ...readyWorkload(item) })).filter((item) => !item.ready);
if (observed.length > 0 && blocked.length === 0) {
emit({ stage: "workload-ready", status: "succeeded", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length });
return;
}
if (Date.now() - startedAt >= timeoutMs) {
emit({ stage: "workload-ready", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, blocked: blocked.slice(0, 8) });
emit({ stage: "workload-ready", status: "timeout", durationMs: Date.now() - startedAt, workloadCount: runtime.items.length, observedCount: observed.length, blocked: blocked.slice(0, 8) });
process.exitCode = 1;
return;
}
@@ -2523,8 +2553,10 @@ function runtimeReadyTask({ profile = "dev" } = {}) {
return {
name: "runtime-ready",
runAfter: ["gitops-promote"],
workspaces: [{ name: "source", workspace: "source" }],
taskSpec: {
params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }],
workspaces: [{ name: "source" }],
steps: [{
name: "runtime-ready",
image: ciToolsRunnerImage,
@@ -3529,20 +3561,7 @@ function g14FrpcManifest({ profile = "dev", source = null } = {}) {
};
const sourceCommitLabel = source?.full ? { "hwlab.pikastech.local/source-commit": source.full } : {};
const renderedLabels = { ...labels, ...sourceCommitLabel };
return {
apiVersion: "v1",
kind: "List",
items: [
{
apiVersion: "v1",
kind: "ConfigMap",
metadata: {
name: configName,
namespace,
labels: renderedLabels
},
data: {
"frpc.toml": `serverAddr = "74.48.78.17"
const configText = `serverAddr = "74.48.78.17"
serverPort = 7000
loginFailExit = true
@@ -3559,7 +3578,24 @@ type = "tcp"
localIP = "hwlab-edge-proxy.${namespace}.svc.cluster.local"
localPort = 6667
remotePort = ${edgeRemotePort}
`
`;
const configSha256 = createHash("sha256").update(configText).digest("hex");
const templateLabels = { ...labels, "hwlab.pikastech.local/config-sha256": configSha256 };
const templateAnnotations = { "hwlab.pikastech.local/config-sha256": configSha256 };
return {
apiVersion: "v1",
kind: "List",
items: [
{
apiVersion: "v1",
kind: "ConfigMap",
metadata: {
name: configName,
namespace,
labels: renderedLabels
},
data: {
"frpc.toml": configText
}
},
{
@@ -3574,7 +3610,7 @@ remotePort = ${edgeRemotePort}
replicas: 1,
selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
template: {
metadata: { labels: renderedLabels, annotations: sourceCommitLabel },
metadata: { labels: templateLabels, annotations: templateAnnotations },
spec: {
containers: [{
name: "frpc",
@@ -3607,9 +3643,14 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau
"hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy",
"hwlab.pikastech.local/source-commit": source.full
};
const templateLabels = {
...labels,
"hwlab.pikastech.local/source-commit": bridgeCommit,
"hwlab.pikastech.local/bridge-source-commit": bridgeCommit
};
const templateAnnotations = {
"hwlab.pikastech.local/bridge-service-id": bridgeServiceId,
"hwlab.pikastech.local/source-commit": source.full,
"hwlab.pikastech.local/source-commit": bridgeCommit,
"hwlab.pikastech.local/bridge-source-commit": bridgeCommit,
"hwlab.pikastech.local/bridge-image-tag": bridgeImageTag,
"hwlab.pikastech.local/bridge-image": bridgeImage
@@ -3634,7 +3675,7 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau
replicas: 1,
selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" } },
template: {
metadata: { labels, annotations: templateAnnotations },
metadata: { labels: templateLabels, annotations: templateAnnotations },
spec: {
initContainers: [{
name: "moonbridge-config",
@@ -3988,7 +4029,16 @@ function v02PostgresManifest({ migrationSql, source }) {
"hwlab.pikastech.local/profile": "v02",
"hwlab.pikastech.local/source-commit": source.full
};
const templateAnnotations = { "hwlab.pikastech.local/source-commit": source.full };
const migrationSha256 = createHash("sha256").update(migrationSql).digest("hex");
const templateLabels = {
"app.kubernetes.io/name": name,
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/environment": "v02",
"hwlab.pikastech.local/gitops-target": "v02",
"hwlab.pikastech.local/profile": "v02",
"hwlab.pikastech.local/migration-sha256": migrationSha256
};
const templateAnnotations = { "hwlab.pikastech.local/migration-sha256": migrationSha256 };
return {
apiVersion: "v1",
kind: "List",
@@ -4014,7 +4064,7 @@ function v02PostgresManifest({ migrationSql, source }) {
replicas: 1,
selector: { matchLabels: { "app.kubernetes.io/name": name } },
template: {
metadata: { labels, annotations: templateAnnotations },
metadata: { labels: templateLabels, annotations: templateAnnotations },
spec: {
containers: [{
name: "postgres",
scripts/g14-gitops-render.test.ts
+15
View File
@@ -52,8 +52,14 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
assert.deepEqual(runtimeReady.when, [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }]);
const runtimeReadyScript = runtimeReady.taskSpec.steps[0].script;
assert.match(runtimeReadyScript, /const pollMs = 1000;/u);
assert.match(runtimeReadyScript, /readObservedServiceIds/u);
assert.match(runtimeReadyScript, /observedRuntimeItems/u);
assert.match(runtimeReadyScript, /observedCount/u);
assert.match(runtimeReadyScript, /ids\.has\("hwlab-cloud-api"\)\) ids\.add\("hwlab-deepseek-proxy"\)/u);
assert.match(runtimeReadyScript, /status: "timeout", reason: "source-commit-refresh-timeout"/u);
assert.match(runtimeReadyScript, /const refresh = await waitForArgoRefresh\(\);\n if \(!refresh\.observed\) return;/u);
assert.ok(runtimeReady.workspaces?.some((workspace) => workspace.name === "source" && workspace.workspace === "source"));
assert.ok(runtimeReady.taskSpec.workspaces?.some((workspace) => workspace.name === "source"));
const gitopsPromote = taskByName(pipelineJson, "gitops-promote");
assert.ok(gitopsPromote.taskSpec.results.some((result) => result.name === "runtime-ready-required"));
@@ -235,6 +241,15 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
const content = await readFile(path.join(outDir, "runtime-v02", generatedFile), "utf8");
assert.match(content, /"hwlab\.pikastech\.local\/source-commit"/u);
}
const frpc = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "g14-frpc.yaml"), "utf8"));
const frpcDeployment = frpc.items.find((item) => item.kind === "Deployment");
assert.ok(frpcDeployment.spec.template.metadata.labels["hwlab.pikastech.local/config-sha256"]);
assert.equal(frpcDeployment.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined);
const postgres = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "postgres.yaml"), "utf8"));
const postgresStatefulSet = postgres.items.find((item) => item.kind === "StatefulSet");
assert.ok(postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/migration-sha256"]);
assert.equal(postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined);
await assert.rejects(() => readFile(path.join(outDir, "tekton-v02", "control-plane-reconciler.yaml"), "utf8"), { code: "ENOENT" });
await assert.rejects(() => readFile(path.join(outDir, "tekton-v02", "poller.yaml"), "utf8"), { code: "ENOENT" });