diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 2fa898cd..35d8c2ea 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -61,8 +61,12 @@ lanes: - hwlab-workbench-runtime - hwlab-user-billing - hwlab-project-management + - hwlab-tasktree-api + - hwlab-tasktree-worker - hwlab-harnessrl-api - hwlab-harnessrl-worker + - hwlab-workbench-api + - hwlab-workbench-worker - hwlab-hwpod-api - hwlab-hwpod-worker - hwlab-hwpod-web @@ -314,6 +318,37 @@ lanes: buildkit: sidecarImage: 127.0.0.1:5000/hwlab/buildkit:rootless sourceImage: docker.io/moby/buildkit:rootless + ciResourceGovernance: + build: + matrixTaskName: build-services + maxParallelServices: 2 + taskTimeout: 20m + priorityClassName: unidesk-ci-batch + stepResources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 750m + memory: 768Mi + sidecarResources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 1500m + memory: 1536Mi + dedupe: + identityFields: + - repository + - lane + - sourceCommit + - pipelineIntent + coreProtection: + configRefs: + - config/hwlab-node-control-plane.yaml#targets.NC01.k3s + - config/platform-infra/sub2api.yaml#targets.NC01.resourceProtection + - config/platform-infra/public-edge.yaml#targets.NC01.runtime.resourceProtection stepEnv: HOME: /tekton/home XDG_CONFIG_HOME: /tekton/home/.config diff --git a/config/platform-infra/public-edge.yaml b/config/platform-infra/public-edge.yaml index 9e11b97a..68e84ccd 100644 --- a/config/platform-infra/public-edge.yaml +++ b/config/platform-infra/public-edge.yaml @@ -52,6 +52,10 @@ targets: - h2 runtime: image: caddy:2.10.2-alpine + resourceProtection: + cpuShares: 2048 + memoryReservation: 128m + memoryLimit: 512m containerName: unidesk-public-edge workDir: /opt/unidesk/public-edge composePath: /opt/unidesk/public-edge/docker-compose.yaml diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index b8ecdfca..5cfc3a9f 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -251,6 +251,25 @@ targets: redisReplicas: 1 runtime: autoSetup: false + resourceProtection: + priorityClassName: unidesk-platform-core + priorityValue: 1000000 + batchPriorityClassName: unidesk-ci-batch + batchPriorityValue: -1000000 + app: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 2000m + memory: 2Gi + redis: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi image: repository: weishaw/sub2api tag: 0.1.162 diff --git a/project-management/PJ2026-01/specs/PJ2026-010601-controlled-release.md b/project-management/PJ2026-01/specs/PJ2026-010601-controlled-release.md index 6a0d9e6b..a4e53a73 100644 --- a/project-management/PJ2026-01/specs/PJ2026-010601-controlled-release.md +++ b/project-management/PJ2026-01/specs/PJ2026-010601-controlled-release.md @@ -85,6 +85,7 @@ | PJ2026-01060104 | 验证分层 | 本规格 6.4 | 自测试、综合联调、CLI/API 交互和真实运行面通过口径 | 目标 runtime、业务模块测试需求 | 发布决策 | | PJ2026-01060105 | AgentRun发布 | [PJ2026-01060105 AgentRun发布Lane](PJ2026-01060105-agentrun-v01-release-lane.md) | AgentRun `v0.1` 的发布 lane、Pipeline、runtime namespace 和真实联调细则 | 源码同步、YAML运维、Agent编排 | AgentRun runtime | | PJ2026-01060106 | HWLAB双环境 | [PJ2026-01060106 HWLAB双环境](PJ2026-01060106-hwlab-dev-production-lanes.md) | HWLAB `v0.3` development 与 `release` production 的分支、流水线、公开入口和数据隔离 | 源码同步、YAML运维、Workbench实时权威 | HWLAB runtime | +| PJ2026-01060107 | NC01 CI资源治理 | [PJ2026-01060107 NC01 CI资源治理](PJ2026-01060107-nc01-ci-resource-governance.md) | NC01 构建并发、排队、资源合同、稳定去重和核心运行面保护 | 发布流水、YAML运维、CI/CD目标治理 | NC01 CI 与核心服务 | ## 6. 原子需求 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060107-nc01-ci-resource-governance.md b/project-management/PJ2026-01/specs/PJ2026-01060107-nc01-ci-resource-governance.md new file mode 100644 index 00000000..021d56e6 --- /dev/null +++ b/project-management/PJ2026-01/specs/PJ2026-01060107-nc01-ci-resource-governance.md @@ -0,0 +1,93 @@ +# PJ2026-01060107 NC01 CI资源治理 + +## 修改历史 + +| 版本 | 对应 commit id | 更新日期 | 变更说明 | +| --- | --- | --- | --- | + +当前正文仍在规格治理草稿中；未定稿前不新增版本号，不为单次编辑追加 `待提交` 版本。 + +## 正文 + +## PJ2026-01060107 NC01 CI资源治理需求规格 + +## 1. 文档控制 + +| 字段 | 内容 | +| --- | --- | +| 编号 | PJ2026-01060107 | +| 短名 | NC01 CI资源治理 | +| 层级 | L3 子课题 | +| 状态 | 草稿 | +| 实现引用版本 | draft-2026-07-21-p0-nc01-ci-resource-governance | +| 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) | +| 上级规格 | [PJ2026-010601 发布流水](PJ2026-010601-controlled-release.md) | +| 关联规格 | [PJ2026-010603 YAML运维](PJ2026-010603-yaml-first-ops.md)、[PJ2026-01060308 CI/CD YAML-first目标治理](PJ2026-01060308-cicd-yaml-first-target-governance.md) | +| 跟踪任务 | pikasTech/unidesk#2721、TaskTree `tg_743359ba-58aa-426b-a6f0-4df84fb1d795` | + +本文定义 NC01 上 CI 构建并发、排队、资源合同、稳定去重和核心运行面保护。所有可调事实由 owning YAML 控制，代码只校验、渲染和观察；禁止手改 Pipeline、PipelineRun、TaskRun、Pod 或运行中 workload 来补齐合同。 + +## 2. 事故与目标 + +单个 HWLAB PipelineRun 曾同时展开 14 个 `build-services` TaskRun，使 8 核 NC01 的 load、Swap 和 k3s Kine/SQLite 延迟同时上升，并造成核心 API 约四分钟 502。PaC Repository 的 `concurrencyLimit` 只限制 Repository 触发层的并发 PipelineRun，不能限制一个 PipelineRun 内 matrix 展开的 TaskRun 数量。 + +治理目标是让每个发布意图只产生一个受控运行，并让单次运行中的构建工作按 YAML 预算进入有界 worker 队列。CI 压力下，k3s 控制面、CoreDNS、Sub2API 和 Public Edge 的可用性优先于构建吞吐。 + +## 3. 架构与数据流 + +```mermaid +flowchart LR + Y[owning YAML] --> V[配置校验] + V --> R[HWLAB Pipeline renderer] + R --> D[单 TaskRun 有界 worker 队列] + D --> T[Tekton TaskRun/Pod] + T --> K[Kubernetes 调度] + T --> O[plan/status/diagnosis] + K --> C[核心运行面保护] +``` + +Repository 触发排队和 Pipeline 内构建排队是两个独立层次。前者继续由 PaC Repository `concurrencyLimit` 管理，后者由 renderer 将目标 matrix 收敛为一个受资源约束的 TaskRun，并在 Pod 内按 `maxParallelServices` 执行有界 worker 队列。不得复制大型内联 `taskSpec`，不得把二者合并为同名字段，也不得创建数据库、ConfigMap 或常驻控制器作为第二队列 authority。 + +## 4. 配置合同 + +HWLAB node/lane target 的 `ciResourceGovernance` 至少声明： + +- `build.maxParallelServices`：单个构建 TaskRun 内最多并行服务数。 +- `build.taskTimeout`：单个构建 TaskRun 的 Tekton timeout。 +- `build.stepResources`：构建工具 step 的 CPU/内存 requests 与 limits。 +- `build.sidecarResources`：BuildKit sidecar 的 CPU/内存 requests 与 limits。 +- `build.priorityClassName`：构建 Pod 的可抢占优先级；必须低于核心运行面。 +- `dedupe.identityFields`：稳定意图键字段，固定为 repository、lane、source commit 和 pipeline intent。 +- `coreProtection`：核心服务优先级和最小资源预算的配置引用；各服务事实仍归自身 owning YAML，不复制进 HWLAB lane YAML。 + +资源字段使用 Kubernetes quantity 字符串。requests 不得超过 limits，CPU 和内存必须同时声明。NC01 默认同时运行的构建服务数不得由代码、环境变量或运行面探测回填。 + +## 5. 原子需求 + +### 5.1 CI-RESOURCE-REQ-001 分层并发与排队 + +Renderer 必须只对 YAML 选中的构建 matrix 建立有界 worker 队列。单个构建 TaskRun 同时工作的服务数不得超过 `maxParallelServices`，每个服务使用独立工作目录和结果目录，任一批失败后不得继续下一批。原先依赖完整 matrix 的后续任务仍依赖该构建任务，非构建任务及其既有依赖语义保持不变。 + +### 5.2 CI-RESOURCE-REQ-002 资源合同 + +构建 TaskRun 的工具 step 和 BuildKit sidecar 必须带 YAML 声明的 `computeResources`。构建 Pod 使用低于核心服务的 PriorityClass；资源不足时允许构建排队或被抢占，不允许核心 API、DNS 或控制面为构建让路。 + +### 5.3 CI-RESOURCE-REQ-003 稳定去重 + +自动触发的发布意图键固定为 `repository + lane + source commit + pipeline intent`。相同键已处于 queued、running 或 succeeded 时，自动 poll 不得再创建第二次构建。人工重跑必须使用显式受控入口，携带独立、可审计的 rerun intent；不得通过删除旧 PipelineRun、修改 label 或人工创建 PipelineRun 绕过去重。 + +### 5.4 CI-RESOURCE-REQ-004 核心运行面保护 + +CoreDNS 和 k3s 控制面使用 Kubernetes 系统关键优先级。Sub2API 和集群内 Public Edge 使用平台核心优先级与自身 owning YAML 的 requests/limits。Host 运行的 Public Edge 使用其 owning YAML 声明的 CPU/内存保留或限制。任何保护变更必须通过相应受控 CLI 渲染和部署，禁止从 live workload 反解后强写。 + +### 5.5 CI-RESOURCE-REQ-005 状态与诊断 + +`plan`、`status` 和 `diagnosis` 必须在一次输出中披露：YAML configRef、触发层 concurrency、构建层 max parallel、批次数、运行/排队 TaskRun 数、step/sidecar 资源合同、priority class、稳定意图键状态和核心保护摘要。共享读取超时只能投影为 warning 或 unavailable，不得把全部 consumer 伪装为真实失败。 + +## 6. 验收 + +- L0：配置解析、资源 quantity、requests/limits 关系、matrix 收敛、有界 worker 队列和依赖保持通过轻量验证。 +- L1：受控 `plan/status/diagnosis` 显示 YAML 来源、并发预算、批次、资源和保护状态，不依赖裸 Kubernetes 命令。 +- L2：由正常 HWLAB source PR merge 自动触发，禁止人工补 PipelineRun；观察到同一时刻构建 TaskRun 不超过 YAML 预算，相同 source commit 的自动 poll 不产生第二次构建，所有构建 Pod 带资源合同，并且 NC01 Node、CoreDNS、Sub2API、Public Edge 和公开 API 在构建期间保持健康。 + +验收失败时保留自动链证据并修复 owning YAML 或 renderer。重启 k3s、删除 Pod/PipelineRun、手工 Argo sync、运行时 patch、mirror flush 或人工补跑均不能作为最终通过证据。 diff --git a/scripts/native/hwlab/runtime-pipeline-resource-governance.mjs b/scripts/native/hwlab/runtime-pipeline-resource-governance.mjs new file mode 100644 index 00000000..01a5abc5 --- /dev/null +++ b/scripts/native/hwlab/runtime-pipeline-resource-governance.mjs @@ -0,0 +1,135 @@ +// Injected into the HWLAB control-plane renderer. It intentionally uses the +// surrounding renderer's overlay, stepEnv, patchScript, YAML, and text values. +function patchPipelineTaskRuntime(task) { + const governance = overlay.ciResourceGovernance?.build; + const governed = governance && task.name === governance.matrixTaskName; + if (governed) task.timeout = governance.taskTimeout; + for (const sidecar of task.taskSpec?.sidecars || []) { + if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage; + if (governed) sidecar.computeResources = structuredClone(governance.sidecarResources); + } + for (const step of task.taskSpec?.steps || []) { + if (step.image === overlay.toolsImage && overlay.toolsImagePullPolicy) step.imagePullPolicy = overlay.toolsImagePullPolicy; + if (governed) step.computeResources = structuredClone(governance.stepResources); + if (Array.isArray(step.env)) { + for (const env of step.env) { + if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name]; + } + } + step.env = Array.isArray(step.env) ? step.env : []; + const existingEnv = new Set(step.env.map((env) => env.name)); + for (const [name, value] of Object.entries(stepEnv)) { + if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value }); + } + if (typeof step.script === 'string') step.script = patchScript(step.script); + } +} + +function applyPipelineBuildQueue(doc) { + const governance = overlay.ciResourceGovernance?.build; + if (!governance) return; + const tasks = Array.isArray(doc.spec.tasks) ? doc.spec.tasks : []; + const task = tasks.find((candidate) => candidate?.name === governance.matrixTaskName); + if (!task) return; + const matrixParams = Array.isArray(task.matrix?.params) ? task.matrix.params : []; + const serviceParam = matrixParams.find((param) => param?.name === 'service-id'); + if (!serviceParam || !Array.isArray(serviceParam.value) || serviceParam.value.length === 0) { + throw new Error(`CI resource governance task ${governance.matrixTaskName} must declare a static service-id matrix`); + } + const maxParallel = governance.maxParallelServices; + const publish = (task.taskSpec?.steps || []).find((step) => step?.name === 'publish'); + if (!publish || typeof publish.script !== 'string') throw new Error(`CI resource governance task ${governance.matrixTaskName} must contain a publish script`); + const serviceIds = serviceParam.value.map((value) => String(value)); + let workerScript = publish.script + .replaceAll('$(params.service-id)', '$service_id') + .replaceAll('/workspace/service-work', '/workspace/service-work/$service_id') + .replaceAll('/tekton/results', '/workspace/service-results-tekton/$service_id'); + publish.script = [ + 'set -eu', + 'build_service() {', + ' service_id="$1"', + ' mkdir -p "/workspace/service-work/$service_id" "/workspace/service-results-tekton/$service_id"', + workerScript.split('\n').map((line) => ` ${line}`).join('\n'), + '}', + `max_parallel=${maxParallel}`, + 'service_csv="$(params.services)"', + 'old_ifs="$IFS"', + "IFS=','", + 'set -- $service_csv', + 'IFS="$old_ifs"', + 'failed=0', + 'while [ "$#" -gt 0 ]; do', + ' pids=""', + ' launched=0', + ' while [ "$#" -gt 0 ] && [ "$launched" -lt "$max_parallel" ]; do', + ' service_id="$1"', + ' shift', + ' (build_service "$service_id") &', + ' pids="$pids $!"', + ' launched=$((launched + 1))', + ' done', + ' for pid in $pids; do if ! wait "$pid"; then failed=1; fi; done', + ' if [ "$failed" -ne 0 ]; then exit "$failed"; fi', + 'done', + ].join('\n'); + task.params = (task.params || []).map((param) => param?.name === 'service-id' ? { name: 'services', value: '$(params.services)' } : param); + if (!task.params.some((param) => param?.name === 'services')) task.params.push({ name: 'services', value: '$(params.services)' }); + task.taskSpec.params = (task.taskSpec.params || []).map((param) => param?.name === 'service-id' ? { ...param, name: 'services' } : param); + task.taskSpec.results = []; + delete task.matrix; + doc.metadata.annotations['unidesk.ai/ci-build-queue-mode'] = 'single-taskrun-bounded-workers'; + doc.metadata.annotations['unidesk.ai/ci-build-service-count'] = String(serviceIds.length); + doc.metadata.annotations['unidesk.ai/ci-build-batch-count'] = String(Math.ceil(serviceIds.length / maxParallel)); + doc.metadata.annotations['unidesk.ai/ci-build-max-parallel'] = String(maxParallel); +} + +function patchManifestObject(doc) { + if (!doc || typeof doc !== 'object') return false; + if (doc.kind !== 'Pipeline' || !doc.spec) return false; + const defaults = { + 'git-url': overlay.gitUrl, + 'git-read-url': overlay.gitReadUrl, + 'git-write-url': overlay.gitWriteUrl, + 'catalog-path': overlay.catalogPath, + 'runtime-path': overlay.runtimePath, + 'registry-prefix': overlay.registryPrefix, + }; + for (const param of doc.spec?.params || []) { + if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name]; + } + doc.metadata = doc.metadata || {}; + if (typeof overlay.pipelineName === 'string' && overlay.pipelineName.length > 0) doc.metadata.name = overlay.pipelineName; + doc.metadata.annotations = doc.metadata.annotations || {}; + doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId; + doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId; + if (overlay.ciResourceGovernance) { + doc.metadata.annotations['unidesk.ai/ci-resource-governance-config-ref'] = overlay.ciResourceGovernance.configRef; + doc.metadata.annotations['unidesk.ai/ci-dedupe-identity'] = overlay.ciResourceGovernance.dedupe.identityFields.join('+'); + } + for (const task of doc.spec?.tasks || []) patchPipelineTaskRuntime(task); + applyPipelineBuildQueue(doc); + return true; +} + +function patchStructuredPipeline() { + try { + const doc = JSON.parse(text); + if (!patchManifestObject(doc)) return false; + text = JSON.stringify(doc, null, 2) + '\n'; + return true; + } catch (error) { + if (String(error?.message || '').includes('CI resource governance')) throw error; + } + if (YAML) { + try { + const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null); + const changed = docs.some((doc) => patchManifestObject(doc)); + if (!changed) return false; + text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\n---\n') + '\n'; + return true; + } catch (error) { + if (String(error?.message || '').includes('CI resource governance')) throw error; + } + } + return false; +} diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 2ed870d4..2bb6e36b 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -396,6 +396,28 @@ export interface HwlabRuntimeBuildkitSpec { readonly sourceImage: string; } +export interface HwlabRuntimeComputeResourcesSpec { + readonly requests: { readonly cpu: string; readonly memory: string }; + readonly limits: { readonly cpu: string; readonly memory: string }; +} + +export interface HwlabRuntimeCiResourceGovernanceSpec { + readonly build: { + readonly matrixTaskName: string; + readonly maxParallelServices: number; + readonly taskTimeout: string; + readonly priorityClassName: string; + readonly stepResources: HwlabRuntimeComputeResourcesSpec; + readonly sidecarResources: HwlabRuntimeComputeResourcesSpec; + }; + readonly dedupe: { + readonly identityFields: readonly ["repository", "lane", "sourceCommit", "pipelineIntent"]; + }; + readonly coreProtection: { + readonly configRefs: readonly string[]; + }; +} + export interface HwlabRuntimeObservabilitySpec { readonly prometheusOperator: boolean; readonly traceExplorerUrlTemplate?: string; @@ -830,6 +852,7 @@ export interface HwlabRuntimeLaneSpec { readonly publicApiUrl: string; readonly stepEnv: Record; readonly buildkit?: HwlabRuntimeBuildkitSpec; + readonly ciResourceGovernance?: HwlabRuntimeCiResourceGovernanceSpec; readonly bootstrapAdmin?: HwlabRuntimeBootstrapAdminSpec; readonly codeAgentProvider?: HwlabRuntimeCodeAgentProviderSpec; readonly codeAgentRuntime?: HwlabRuntimeCodeAgentRuntimeSpec; @@ -897,6 +920,7 @@ interface HwlabLaneConfig { readonly public: { readonly webUrl: string; readonly apiUrl: string }; readonly stepEnv: Record; readonly buildkit?: HwlabRuntimeBuildkitSpec; + readonly ciResourceGovernance?: HwlabRuntimeCiResourceGovernanceSpec; readonly bootstrapAdmin?: HwlabRuntimeBootstrapAdminSpec; readonly codeAgentProvider?: HwlabRuntimeCodeAgentProviderSpec; readonly codeAgentRuntime?: HwlabRuntimeCodeAgentRuntimeSpec; @@ -1163,6 +1187,7 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa }, stepEnv: optionalStringRecord(raw.stepEnv, `lanes.${id}.stepEnv`), buildkit: buildkitConfig(raw.buildkit, `lanes.${id}.buildkit`), + ciResourceGovernance: ciResourceGovernanceConfig(raw.ciResourceGovernance, `lanes.${id}.ciResourceGovernance`), bootstrapAdmin: bootstrapAdminConfig(raw.bootstrapAdmin, `lanes.${id}.bootstrapAdmin`), codeAgentProvider: codeAgentProviderConfig(raw.codeAgentProvider, `lanes.${id}.codeAgentProvider`), codeAgentRuntime: codeAgentRuntimeConfig(raw.codeAgentRuntime, `lanes.${id}.codeAgentRuntime`), @@ -1200,6 +1225,7 @@ function laneTargetConfig( public: mergeOptionalRecord(baseRaw.public, targetRaw.public), stepEnv: mergeOptionalRecord(baseRaw.stepEnv, targetRaw.stepEnv) ?? {}, buildkit: mergeOptionalRecord(baseRaw.buildkit, targetRaw.buildkit), + ciResourceGovernance: mergeOptionalRecord(baseRaw.ciResourceGovernance, targetRaw.ciResourceGovernance), bootstrapAdmin: mergeOptionalRecord(baseRaw.bootstrapAdmin, targetRaw.bootstrapAdmin), codeAgentProvider: mergeOptionalRecord(baseRaw.codeAgentProvider, targetRaw.codeAgentProvider), codeAgentRuntime: mergeOptionalRecord(baseRaw.codeAgentRuntime, targetRaw.codeAgentRuntime), @@ -1295,6 +1321,66 @@ function buildkitConfig(value: unknown, path: string): HwlabRuntimeBuildkitSpec }; } +function kubernetesQuantity(value: unknown, path: string, kind: "cpu" | "memory"): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty Kubernetes quantity`); + const pattern = kind === "cpu" ? /^(?:[1-9]\d*|0)(?:\.\d+)?m?$/u : /^(?:[1-9]\d*|0)(?:\.\d+)?(?:Ki|Mi|Gi|Ti)$/u; + if (!pattern.test(value)) throw new Error(`${path} must be a supported Kubernetes ${kind} quantity`); + return value; +} + +function computeResourcesConfig(value: unknown, path: string): HwlabRuntimeComputeResourcesSpec { + const raw = asRecord(value, path); + const requests = asRecord(raw.requests, `${path}.requests`); + const limits = asRecord(raw.limits, `${path}.limits`); + const result = { + requests: { + cpu: kubernetesQuantity(requests.cpu, `${path}.requests.cpu`, "cpu"), + memory: kubernetesQuantity(requests.memory, `${path}.requests.memory`, "memory"), + }, + limits: { + cpu: kubernetesQuantity(limits.cpu, `${path}.limits.cpu`, "cpu"), + memory: kubernetesQuantity(limits.memory, `${path}.limits.memory`, "memory"), + }, + }; + if (kubernetesQuantityValue(result.requests.cpu, "cpu") > kubernetesQuantityValue(result.limits.cpu, "cpu")) throw new Error(`${path}.requests.cpu must not exceed limits.cpu`); + if (kubernetesQuantityValue(result.requests.memory, "memory") > kubernetesQuantityValue(result.limits.memory, "memory")) throw new Error(`${path}.requests.memory must not exceed limits.memory`); + return result; +} + +function kubernetesQuantityValue(value: string, kind: "cpu" | "memory"): number { + if (kind === "cpu") return value.endsWith("m") ? Number(value.slice(0, -1)) / 1000 : Number(value); + const units = { Ki: 1024, Mi: 1024 ** 2, Gi: 1024 ** 3, Ti: 1024 ** 4 } as const; + const unit = value.slice(-2) as keyof typeof units; + return Number(value.slice(0, -2)) * units[unit]; +} + +function ciResourceGovernanceConfig(value: unknown, path: string): HwlabRuntimeCiResourceGovernanceSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + const build = asRecord(raw.build, `${path}.build`); + const dedupe = asRecord(raw.dedupe, `${path}.dedupe`); + const coreProtection = asRecord(raw.coreProtection, `${path}.coreProtection`); + const identityFields = stringArrayField(dedupe, "identityFields", `${path}.dedupe`); + const expectedIdentityFields = ["repository", "lane", "sourceCommit", "pipelineIntent"] as const; + if (JSON.stringify(identityFields) !== JSON.stringify(expectedIdentityFields)) { + throw new Error(`${path}.dedupe.identityFields must be repository, lane, sourceCommit, pipelineIntent in that order`); + } + const taskTimeout = stringField(build, "taskTimeout", `${path}.build`); + if (!/^(?:\d+h)?(?:\d+m)?(?:\d+s)?$/u.test(taskTimeout) || !/\d/u.test(taskTimeout)) throw new Error(`${path}.build.taskTimeout must be a Tekton duration`); + return { + build: { + matrixTaskName: stringField(build, "matrixTaskName", `${path}.build`), + maxParallelServices: boundedIntegerField(build, "maxParallelServices", `${path}.build`, 1, 32), + taskTimeout, + priorityClassName: stringField(build, "priorityClassName", `${path}.build`), + stepResources: computeResourcesConfig(build.stepResources, `${path}.build.stepResources`), + sidecarResources: computeResourcesConfig(build.sidecarResources, `${path}.build.sidecarResources`), + }, + dedupe: { identityFields: expectedIdentityFields }, + coreProtection: { configRefs: nonEmptyStringArrayField(coreProtection, "configRefs", `${path}.coreProtection`) }, + }; +} + function sourceRefField(obj: Record, key: string, path: string): string { const value = stringField(obj, key, path); if (!/^[A-Za-z0-9_./-]+$/u.test(value)) throw new Error(`${path}.${key} has an unsupported format`); @@ -2819,6 +2905,7 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec { publicApiUrl: config.public.apiUrl, stepEnv: config.stepEnv, ...(config.buildkit === undefined ? {} : { buildkit: config.buildkit }), + ...(config.ciResourceGovernance === undefined ? {} : { ciResourceGovernance: config.ciResourceGovernance }), ...(config.bootstrapAdmin === undefined ? {} : { bootstrapAdmin: config.bootstrapAdmin }), ...(config.codeAgentProvider === undefined ? {} : { codeAgentProvider: config.codeAgentProvider }), ...(config.codeAgentRuntime === undefined ? {} : { codeAgentRuntime: config.codeAgentRuntime }), diff --git a/scripts/src/hwlab-node/plan.ts b/scripts/src/hwlab-node/plan.ts index 92ee3940..dcad8372 100644 --- a/scripts/src/hwlab-node/plan.ts +++ b/scripts/src/hwlab-node/plan.ts @@ -120,6 +120,15 @@ export function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record 0) doc.metadata.name = overlay.pipelineName;", - " doc.metadata.annotations = doc.metadata.annotations || {};", - " doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;", - " doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;", - " for (const task of doc.spec?.tasks || []) {", - " for (const sidecar of task.taskSpec?.sidecars || []) {", - " if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;", - " }", - " for (const step of task.taskSpec?.steps || []) {", - " if (step.image === overlay.toolsImage && overlay.toolsImagePullPolicy) step.imagePullPolicy = overlay.toolsImagePullPolicy;", - " if (Array.isArray(step.env)) {", - " for (const env of step.env) {", - " if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];", - " }", - " }", - " step.env = Array.isArray(step.env) ? step.env : [];", - " const existingEnv = new Set(step.env.map((env) => env.name));", - " for (const [name, value] of Object.entries(stepEnv)) {", - " if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });", - " }", - " if (typeof step.script === 'string') step.script = patchScript(step.script);", - " }", - " }", - " return true;", - "}", - "function patchStructuredPipeline() {", - " try {", - " const doc = JSON.parse(text);", - " if (!patchManifestObject(doc)) return false;", - " text = JSON.stringify(doc, null, 2) + '\\n';", - " return true;", - " } catch {}", - " if (YAML) {", - " try {", - " const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);", - " const changed = docs.some((doc) => patchManifestObject(doc));", - " if (!changed) return false;", - " text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';", - " return true;", - " } catch {}", - " }", - " return false;", - "}", + ...runtimePipelineResourceGovernanceNativeScript.split("\n"), "function patchGitMirrorTransportYaml() {", " const mirror = overlay.gitMirror || {};", " const transport = mirror.githubTransport || {};", diff --git a/scripts/src/hwlab-node/web-probe.ts b/scripts/src/hwlab-node/web-probe.ts index 2d48588a..050b3f2d 100644 --- a/scripts/src/hwlab-node/web-probe.ts +++ b/scripts/src/hwlab-node/web-probe.ts @@ -117,6 +117,10 @@ export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record 63) throw new Error(`HWLAB deterministic PipelineRun name exceeds 63 characters for ${binding.consumer.id}`); + annotations["unidesk.ai/ci-dedupe-mode"] = "deterministic-kubernetes-name"; + } Object.assign(annotations, { "hwlab.pikastech.local/ci-contract": "tekton-native-primitive-tasks", "hwlab.pikastech.local/download-profile": spec.downloadProfileId, @@ -928,7 +934,7 @@ function remotePipelineRun(binding: PacSourceArtifactBinding, pipeline: Record { release: pac.release, repository: repositorySummary(repository), consumer: consumerObservationSummary(consumer), + ciResourceGovernance: ciResourceGovernanceSummary(consumer, repository), secrets, policy: policyChecks(repository), warnings: pac.validationWarnings, @@ -1600,6 +1605,13 @@ async function status(config: UniDeskConfig, options: CommonOptions, collectArti const observed = await capturePacReadOnly(config, pac, "status", target, consumer, remoteScript("status", pac, target, repository, consumer, { ...options, confirm: false, dryRun: true, wait: false }, secrets, "", [consumer], collectArtifactLogs), `${stringValue(next.status)} --full`); const { result, parsed, observation } = observed; const summary = parsed === null ? null : statusSummary(parsed); + const ciResourceGovernance = ciResourceGovernanceSummary( + consumer, + repository, + arrayRecords(summary?.taskRuns), + arrayRecords(summary?.pipelineRuns), + observation.ok, + ); const deliveryAuthority = resolveCicdDeliveryAuthority({ consumerId: consumer.id, node: consumer.node, lane: consumer.lane }); const deliveryBudget = observePacStatusDeliveryBudget({ policy: pac.deliveryTiming, targetId: target.id, consumerId: consumer.id, summary }); const deliveryWarning = record(deliveryBudget.warning); @@ -1613,6 +1625,7 @@ async function status(config: UniDeskConfig, options: CommonOptions, collectArti deliveryAuthority, coverage: consumerCoverage(pac, target.id), summary, + ciResourceGovernance, deliveryBudget, observation, remote: parsed === null ? compactCapture(result, { full: true }) : options.raw ? parsed : undefined, @@ -1917,6 +1930,7 @@ async function diagnoseRegression(config: UniDeskConfig, options: HistoryOptions const pac = readPacConfig({ consumerId: options.consumerId }); const target = resolveTarget(pac, options.targetId); const consumer = resolveConsumer(pac, options.consumerId); + const repository = resolveRepository(pac, consumer.repositoryRef); if (consumer.node.toLowerCase() !== target.id.toLowerCase()) { throw new Error(`Pipelines-as-Code consumer ${consumer.id} belongs to ${consumer.node}, not target ${target.id}`); } @@ -1955,6 +1969,7 @@ async function diagnoseRegression(config: UniDeskConfig, options: HistoryOptions observation: historyResult.observation, historyErrors: historyResult.historyErrors, warnings: historyResult.warnings, + ciResourceGovernance: ciResourceGovernanceSummary(consumer, repository), valuesPrinted: false, }; } @@ -2637,6 +2652,72 @@ function repositorySummary(repository: PacRepository): Record { }; } +function ciResourceGovernanceSummary( + consumer: PacConsumer, + repository: PacRepository, + taskRuns: readonly Record[] = [], + pipelineRuns: readonly Record[] = [], + liveObservationAvailable = false, +): Record | null { + if (!isHwlabRuntimeLane(consumer.lane)) return null; + const lane = hwlabRuntimeLaneSpecForNode(consumer.lane, consumer.node); + const governance = lane.ciResourceGovernance; + if (governance === undefined) return null; + const runningTaskRuns = taskRuns.filter((item) => { + const status = stringValue(item.status).toLowerCase(); + const reason = stringValue(item.reason).toLowerCase(); + return status === "running" || status === "unknown" || reason === "running"; + }).length; + const queuedTaskRuns = taskRuns.filter((item) => { + const status = stringValue(item.status).toLowerCase(); + const reason = stringValue(item.reason).toLowerCase(); + return status === "pending" || reason.includes("pending") || reason.includes("queued"); + }).length; + const activeOrSucceeded = pipelineRuns.filter((item) => { + const status = stringValue(item.status).toLowerCase(); + const reason = stringValue(item.reason).toLowerCase(); + return ["running", "unknown", "succeeded", "true"].includes(status) + || ["running", "succeeded", "completed"].includes(reason); + }); + const identityCounts = new Map(); + for (const item of activeOrSucceeded) { + const sourceCommit = stringValue(item.sourceCommit, ""); + if (sourceCommit.length === 0 || sourceCommit === "-") continue; + const key = [repository.id, consumer.lane, sourceCommit, consumer.pipeline].join("+"); + identityCounts.set(key, (identityCounts.get(key) ?? 0) + 1); + } + const duplicateIntentCount = [...identityCounts.values()].filter((count) => count > 1).length; + return { + configRef: `config/hwlab-node-lanes.yaml#lanes.${consumer.lane}.targets.${consumer.node}.ciResourceGovernance`, + triggerConcurrencyLimit: repository.concurrencyLimit, + queueMode: "single-taskrun-bounded-workers", + dedupeMode: "deterministic-kubernetes-name", + automaticNameTemplate: `${consumer.pipelineRunPrefix}-{{ revision }}`, + buildTaskName: governance.build.matrixTaskName, + maxParallelServices: governance.build.maxParallelServices, + serviceCount: lane.serviceIds.length, + batchCount: Math.ceil(lane.serviceIds.length / governance.build.maxParallelServices), + taskTimeout: governance.build.taskTimeout, + taskRuns: { + status: liveObservationAvailable ? "observed" : "unavailable", + running: liveObservationAvailable ? runningTaskRuns : null, + queued: liveObservationAvailable ? queuedTaskRuns : null, + observed: liveObservationAvailable ? taskRuns.length : null, + }, + stepResources: governance.build.stepResources, + sidecarResources: governance.build.sidecarResources, + priorityClassName: governance.build.priorityClassName, + intentIdentity: { + fields: governance.dedupe.identityFields, + state: !liveObservationAvailable ? "not-observed" : duplicateIntentCount === 0 ? "unique" : "duplicate-observed", + duplicateCount: liveObservationAvailable ? duplicateIntentCount : null, + blocking: false, + }, + coreProtectionConfigRefs: governance.coreProtection.configRefs, + valuesPrinted: false, + }; +} + function consumerObservationSummary(consumer: PacConsumer): Record { return { id: consumer.id, @@ -2697,6 +2778,7 @@ function compactPlanJson(result: Record): Record): Record): RenderedCliResult { const secrets = arrayRecords(result.secrets); const policy = arrayRecords(result.policy); const warnings = arrayRecords(result.warnings); + const ciResourceGovernance = record(result.ciResourceGovernance); const lines = [ "PLATFORM-INFRA PIPELINES-AS-CODE PLAN", ...table(["TARGET", "NAMESPACE", "RELEASE", "REPOSITORY", "CONSUMER"], [[stringValue(target.id), stringValue(repository.namespace), stringValue(record(config.release).version), stringValue(repository.name), stringValue(consumer.id)]]), @@ -3004,6 +3088,7 @@ function renderPlan(result: Record): RenderedCliResult { ["cd", "Argo", stringValue(consumer.argoApplication)], ["runtime evidence", "owning YAML", `${stringValue(deliveryObservation.runtimeEvidence)} (${stringValue(deliveryObservation.configPath)})`], ]), + ...renderCiResourceGovernance(ciResourceGovernance), "", "SECRETS", ...table(["ID", "PRESENT", "FINGERPRINT", "VALUES"], secrets.map((item) => [stringValue(item.id), boolText(item.present), stringValue(item.fingerprint), "false"])), @@ -3078,6 +3163,7 @@ export function renderStatus(result: Record): RenderedCliResult const deliveryPlan = record(sourceObservation.plan); const repository = record(summary.repository); const webhooks = arrayRecords(summary.webhooks); + const ciResourceGovernance = record(result.ciResourceGovernance); const lines = [ "PLATFORM-INFRA PIPELINES-AS-CODE STATUS", ...renderPacConfigWarnings(warnings), @@ -3102,6 +3188,7 @@ export function renderStatus(result: Record): RenderedCliResult ` argocd-repository-secret: name=${stringValue(bootstrapArgo.secretName)} exists=${boolText(bootstrapArgo.exists)} label=${boolText(bootstrapArgo.labelReady)} keys=${boolText(bootstrapArgo.keysReady)} url=${boolText(bootstrapArgo.urlReady)} password-present=${boolText(bootstrapArgo.passwordPresent)} password-decoded=${boolText(bootstrapArgo.passwordDecoded)}`, ` consumer-bootstrap-reasons: ${Array.isArray(consumerBootstrap.reasons) ? consumerBootstrap.reasons.join(",") || "-" : "-"}`, ]), + ...renderCiResourceGovernance(ciResourceGovernance), "", "GITEA HOOKS", ...(webhooks.length === 0 ? ["-"] : table(["HOOK", "ACTIVE", "EVENTS", "URL"], webhooks.map((item) => [stringValue(item.id), boolText(item.active), Array.isArray(item.events) ? item.events.join(",") : stringValue(item.events), short(stringValue(item.url), 56)]))), @@ -3169,6 +3256,34 @@ export function renderStatus(result: Record): RenderedCliResult return rendered(result, "platform-infra pipelines-as-code status", lines); } +function renderCiResourceGovernance(governance: Record): string[] { + if (Object.keys(governance).length === 0) return []; + const taskRuns = record(governance.taskRuns); + const intentIdentity = record(governance.intentIdentity); + const stepResources = record(governance.stepResources); + const sidecarResources = record(governance.sidecarResources); + return [ + "", + "CI RESOURCE GOVERNANCE", + ` config-ref: ${stringValue(governance.configRef)}`, + ...table(["TRIGGER_MAX", "QUEUE_MODE", "BUILD_TASK", "SERVICE_MAX", "BATCHES", "RUNNING", "QUEUED", "PRIORITY"], [[ + stringValue(governance.triggerConcurrencyLimit), + stringValue(governance.queueMode), + stringValue(governance.buildTaskName), + stringValue(governance.maxParallelServices), + stringValue(governance.batchCount), + stringValue(taskRuns.running), + stringValue(taskRuns.queued), + stringValue(governance.priorityClassName), + ]]), + ` step-resources: ${compactLine(JSON.stringify(stepResources))}`, + ` sidecar-resources: ${compactLine(JSON.stringify(sidecarResources))}`, + ` dedupe: mode=${stringValue(governance.dedupeMode)} name=${stringValue(governance.automaticNameTemplate)}`, + ` intent-identity: fields=${Array.isArray(intentIdentity.fields) ? intentIdentity.fields.join("+") : "-"} state=${stringValue(intentIdentity.state)} duplicates=${stringValue(intentIdentity.duplicateCount)} blocking=false`, + ` core-protection: ${Array.isArray(governance.coreProtectionConfigRefs) ? governance.coreProtectionConfigRefs.join(",") : "-"}`, + ]; +} + function renderPacConfigWarnings(warnings: readonly Record[]): string[] { if (warnings.length === 0) return []; return [ diff --git a/scripts/src/platform-infra-public-edge.ts b/scripts/src/platform-infra-public-edge.ts index 174e793a..80ae666d 100644 --- a/scripts/src/platform-infra-public-edge.ts +++ b/scripts/src/platform-infra-public-edge.ts @@ -103,6 +103,11 @@ export interface PublicEdgeTarget { configDir: string; statePath: string; lockPath: string; + resourceProtection: { + cpuShares: number; + memoryReservation: string; + memoryLimit: string; + }; legacy: { containerName: string; composePath: string; @@ -155,11 +160,11 @@ export async function runPlatformInfraPublicEdgeCommand(config: UniDeskConfig, a const resolution = resolveSites(target); if (options.action === "plan") { const result = planResult(options.configPath, target, resolution, "plan"); - return renderMachine("platform-infra public-edge plan", result, "json", result.ok === true); + return renderMachine("platform-infra public-edge plan", compactPlan(result), "json", result.ok === true); } if (options.action === "apply") { const result = planResult(options.configPath, target, resolution, "dry-run"); - return renderMachine("platform-infra public-edge apply", result, "json", result.ok === true); + return renderMachine("platform-infra public-edge apply", compactPlan(result), "json", result.ok === true); } if (options.action === "status") { const result = await status(config, options.configPath, delivery, target, resolution); @@ -266,6 +271,7 @@ function readTarget(root: Record, requestedTargetId: string | n const listener = record(target.listener, `targets.${targetId}.listener`); const isolation = record(target.isolation, `targets.${targetId}.isolation`); const runtime = record(target.runtime, `targets.${targetId}.runtime`); + const resourceProtection = record(runtime.resourceProtection, `targets.${targetId}.runtime.resourceProtection`); const legacy = record(runtime.legacy, `targets.${targetId}.runtime.legacy`); const sites = array(target.sites, `targets.${targetId}.sites`); const parsed: PublicEdgeTarget = { @@ -291,6 +297,11 @@ function readTarget(root: Record, requestedTargetId: string | n configDir: absolutePath(runtime.configDir, `targets.${targetId}.runtime.configDir`), statePath: absolutePath(runtime.statePath, `targets.${targetId}.runtime.statePath`), lockPath: absolutePath(runtime.lockPath, `targets.${targetId}.runtime.lockPath`), + resourceProtection: { + cpuShares: positiveInteger(resourceProtection.cpuShares, `targets.${targetId}.runtime.resourceProtection.cpuShares`), + memoryReservation: dockerMemory(resourceProtection.memoryReservation, `targets.${targetId}.runtime.resourceProtection.memoryReservation`), + memoryLimit: dockerMemory(resourceProtection.memoryLimit, `targets.${targetId}.runtime.resourceProtection.memoryLimit`), + }, legacy: { containerName: string(legacy.containerName, `targets.${targetId}.runtime.legacy.containerName`), composePath: absolutePath(legacy.composePath, `targets.${targetId}.runtime.legacy.composePath`), @@ -311,6 +322,9 @@ function readTarget(root: Record, requestedTargetId: string | n }; }), }; + if (dockerMemoryBytes(parsed.runtime.resourceProtection.memoryReservation) > dockerMemoryBytes(parsed.runtime.resourceProtection.memoryLimit)) { + throw inputError(`targets.${targetId}.runtime.resourceProtection.memoryReservation 不得大于 memoryLimit`, "invalid-resource-protection", `targets.${targetId}.runtime.resourceProtection`); + } const routeId = parsed.route.split(":", 1)[0]!; if (parsed.isolation.excludedRoutes.includes(routeId) || parsed.isolation.excludedRoutes.includes(parsed.id)) { throw inputError(`target route 命中 excludedRoutes：${parsed.route}`, "excluded-target-route", `targets.${targetId}.route`); @@ -797,6 +811,33 @@ function compactStatus(result: Record): Record }; } +function compactPlan(result: Record): Record { + const target = optionalRecord(result.target) ?? {}; + const rendered = optionalRecord(result.rendered) ?? {}; + const sites = Array.isArray(result.sites) ? result.sites : []; + const unresolved = Array.isArray(result.unresolved) ? result.unresolved : []; + return { + ok: result.ok === true, + action: result.action, + mode: result.mode, + mutation: false, + configRef: result.configRef, + target: { + id: target.id ?? null, + route: target.route ?? null, + publicAddress: target.publicAddress ?? null, + listeners: target.listeners ?? [], + containerName: target.containerName ?? null, + resourceProtection: target.resourceProtection ?? null, + }, + siteSummary: { resolved: sites.length, unresolved: unresolved.length }, + rendered, + warning: unresolved.length === 0 ? null : compactStatusWarning(unresolved[0]), + complete: result.complete === true, + valuesPrinted: false, + }; +} + function firstStatusFailure(runtime: Record): Record | null { if (optionalRecord(runtime.capture) !== null) { return { stage: "remoteStatus", code: "public-edge-status-capture-failed" }; @@ -850,7 +891,7 @@ export function renderPublicEdgeArtifacts(target: PublicEdgeTarget, sites: Resol const authFingerprintBlock = authMaterialFingerprint === null ? "" : ` environment:\n UNIDESK_PUBLIC_EDGE_AUTH_MATERIAL_FINGERPRINT: ${authMaterialFingerprint}\n`; - const compose = `services:\n caddy:\n image: ${target.runtime.image}\n container_name: ${target.runtime.containerName}\n network_mode: host\n restart: unless-stopped\n${envFileBlock}${authFingerprintBlock} dns:\n${dnsServers}\n volumes:\n - ${target.runtime.caddyfilePath}:/etc/caddy/Caddyfile:ro\n - ${target.runtime.dataDir}:/data\n - ${target.runtime.configDir}:/config\n`; + const compose = `services:\n caddy:\n image: ${target.runtime.image}\n container_name: ${target.runtime.containerName}\n network_mode: host\n restart: unless-stopped\n cpu_shares: ${target.runtime.resourceProtection.cpuShares}\n mem_reservation: ${target.runtime.resourceProtection.memoryReservation}\n mem_limit: ${target.runtime.resourceProtection.memoryLimit}\n${envFileBlock}${authFingerprintBlock} dns:\n${dnsServers}\n volumes:\n - ${target.runtime.caddyfilePath}:/etc/caddy/Caddyfile:ro\n - ${target.runtime.dataDir}:/data\n - ${target.runtime.configDir}:/config\n`; return { caddyfile: `${global}\n${blocks}`, compose }; } @@ -1306,6 +1347,7 @@ function targetSummary(target: PublicEdgeTarget): Record { configDir: target.runtime.configDir, statePath: target.runtime.statePath, lockPath: target.runtime.lockPath, + resourceProtection: target.runtime.resourceProtection, legacyContainerName: target.runtime.legacy.containerName, dnsServers: target.runtime.dnsServers, }; @@ -1402,6 +1444,17 @@ function positiveInteger(value: unknown, path: string): number { return value as number; } +function dockerMemory(value: unknown, path: string): string { + const parsed = string(value, path); + if (!/^[1-9]\d*(?:[bkmg])$/iu.test(parsed)) throw inputError(`${path} 必须是 Docker 内存值，例如 128m`, "invalid-docker-memory", path); + return parsed; +} + +function dockerMemoryBytes(value: string): number { + const units: Record = { b: 1, k: 1024, m: 1024 ** 2, g: 1024 ** 3 }; + return Number(value.slice(0, -1)) * units[value.slice(-1).toLowerCase()]; +} + function port(value: unknown, path: string): number { const result = positiveInteger(value, path); if (result > 65_535) throw new Error(`${path} 必须小于等于 65535`); diff --git a/scripts/src/platform-infra/actions.ts b/scripts/src/platform-infra/actions.ts index 58d3a154..5d05eb06 100644 --- a/scripts/src/platform-infra/actions.ts +++ b/scripts/src/platform-infra/actions.ts @@ -200,7 +200,11 @@ export function plan(options: TargetOptions): Record { ? `Node-local HTTPS ${target.publicExposure.publicBaseUrl} through ${target.id} Docker Caddy to the ClusterIP Sub2API Service; no PK01 Caddy/FRP and no Kubernetes Ingress/NodePort/LoadBalancer.` : `Public HTTPS ${target.publicExposure.publicBaseUrl} through PK01 Caddy and ${target.id} frpc; no master server forwarding and no Kubernetes Ingress/NodePort/LoadBalancer.` : "ClusterIP only; no public ingress or node-level exposure.", - resourcePolicy: isHostDockerTarget(target) ? "PK01 host-Docker compose is controlled by YAML; no Kubernetes resources are rendered." : "No Kubernetes CPU/memory requests or limits, matching issue #220.", + resourcePolicy: isHostDockerTarget(target) + ? "PK01 host-Docker compose is controlled by YAML; no Kubernetes resources are rendered." + : target.resourceProtection === null + ? "No explicit Kubernetes resource protection is declared for this target." + : `PriorityClass ${target.resourceProtection.priorityClassName} and app/Redis requests and limits are controlled by config/platform-infra/sub2api.yaml.`, imageVersionControl: "Sub2API image repository/tag/pullPolicy are controlled by config/platform-infra/sub2api.yaml in the UniDesk repository.", urlAllowlistControl: "Sub2API upstream URL validation options are controlled by config/platform-infra/sub2api.yaml and rendered to SECURITY_URL_ALLOWLIST_* env vars.", networkPolicy: isHostDockerTarget(target) ? "Not applicable for PK01 host-Docker deployment." : "NetworkPolicy/allow-all is rendered with the deployment so kube-router cannot silently default-deny Sub2API cross-pod traffic.", diff --git a/scripts/src/platform-infra/config.ts b/scripts/src/platform-infra/config.ts index d4c1e1a4..54904512 100644 --- a/scripts/src/platform-infra/config.ts +++ b/scripts/src/platform-infra/config.ts @@ -182,6 +182,7 @@ export function parseTargets(root: Record, defaultTargetId: str const appReplicas = integerField(record, "appReplicas", path); const redisReplicas = integerField(record, "redisReplicas", path); const runtime = parseTargetRuntime(record.runtime, path); + const resourceProtection = parseResourceProtection(record.resourceProtection, path, runtimeMode); const image = targetImageOverride(record, path); const dependencyImages = targetDependencyImageOverride(record, path); const hostDocker = parseHostDockerConfig(record.hostDocker, path, runtimeMode); @@ -198,7 +199,7 @@ export function parseTargets(root: Record, defaultTargetId: str if (runtimeMode === "host-docker" && accountLocalProxy?.enabled === true) throw new Error(`${configPath}.${path}.accountLocalProxy must be disabled or omitted for runtimeMode=host-docker`); if (runtimeMode === "host-docker" && publicExposure?.enabled === true && publicExposure.mode !== "pk01-local") throw new Error(`${configPath}.${path}.publicExposure.mode must be pk01-local for runtimeMode=host-docker`); if (runtimeMode === "k3s" && publicExposure?.enabled === true && publicExposure.mode === "pk01-local") throw new Error(`${configPath}.${path}.publicExposure.mode=pk01-local is only supported for runtimeMode=host-docker`); - return { id, route, namespace: targetNamespace, runtimeMode, role, enabled, databaseMode, redisMode, appReplicas, redisReplicas, runtime, image, dependencyImages, hostDocker, publicExposure, egressProxy, accountLocalProxy }; + return { id, route, namespace: targetNamespace, runtimeMode, role, enabled, databaseMode, redisMode, appReplicas, redisReplicas, runtime, resourceProtection, image, dependencyImages, hostDocker, publicExposure, egressProxy, accountLocalProxy }; }); const ids = new Set(); for (const target of targets) { @@ -209,6 +210,57 @@ export function parseTargets(root: Record, defaultTargetId: str return targets; } +function resourceQuantity(value: unknown, path: string, kind: "cpu" | "memory"): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`${configPath}.${path} must be a non-empty Kubernetes quantity`); + const pattern = kind === "cpu" ? /^(?:[1-9]\d*|0)(?:\.\d+)?m?$/u : /^(?:[1-9]\d*|0)(?:\.\d+)?(?:Ki|Mi|Gi|Ti)$/u; + if (!pattern.test(value)) throw new Error(`${configPath}.${path} has an unsupported Kubernetes ${kind} quantity`); + return value; +} + +function containerResources(value: unknown, path: string): NonNullable["app"] { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${configPath}.${path} must be an object`); + const raw = value as Record; + const requests = objectField(raw, "requests", path); + const limits = objectField(raw, "limits", path); + const result = { + requests: { cpu: resourceQuantity(requests.cpu, `${path}.requests.cpu`, "cpu"), memory: resourceQuantity(requests.memory, `${path}.requests.memory`, "memory") }, + limits: { cpu: resourceQuantity(limits.cpu, `${path}.limits.cpu`, "cpu"), memory: resourceQuantity(limits.memory, `${path}.limits.memory`, "memory") }, + }; + if (resourceQuantityValue(result.requests.cpu, "cpu") > resourceQuantityValue(result.limits.cpu, "cpu")) throw new Error(`${configPath}.${path}.requests.cpu must not exceed limits.cpu`); + if (resourceQuantityValue(result.requests.memory, "memory") > resourceQuantityValue(result.limits.memory, "memory")) throw new Error(`${configPath}.${path}.requests.memory must not exceed limits.memory`); + return result; +} + +function resourceQuantityValue(value: string, kind: "cpu" | "memory"): number { + if (kind === "cpu") return value.endsWith("m") ? Number(value.slice(0, -1)) / 1000 : Number(value); + const units = { Ki: 1024, Mi: 1024 ** 2, Gi: 1024 ** 3, Ti: 1024 ** 4 } as const; + const unit = value.slice(-2) as keyof typeof units; + return Number(value.slice(0, -2)) * units[unit]; +} + +function parseResourceProtection(value: unknown, path: string, runtimeMode: Sub2ApiTargetConfig["runtimeMode"]): Sub2ApiTargetConfig["resourceProtection"] { + if (value === undefined || value === null) return null; + if (runtimeMode !== "k3s") throw new Error(`${configPath}.${path}.resourceProtection is only supported for runtimeMode=k3s`); + if (typeof value !== "object" || Array.isArray(value)) throw new Error(`${configPath}.${path}.resourceProtection must be an object`); + const raw = value as Record; + const priorityClassName = stringField(raw, "priorityClassName", `${path}.resourceProtection`); + const priorityValue = integerField(raw, "priorityValue", `${path}.resourceProtection`); + const batchPriorityClassName = stringField(raw, "batchPriorityClassName", `${path}.resourceProtection`); + const batchPriorityValue = integerField(raw, "batchPriorityValue", `${path}.resourceProtection`); + if (!isKubernetesName(priorityClassName)) throw new Error(`${configPath}.${path}.resourceProtection.priorityClassName must be a Kubernetes name`); + if (!isKubernetesName(batchPriorityClassName)) throw new Error(`${configPath}.${path}.resourceProtection.batchPriorityClassName must be a Kubernetes name`); + if (priorityValue < 1 || priorityValue >= 1000000000) throw new Error(`${configPath}.${path}.resourceProtection.priorityValue must be between 1 and 999999999`); + if (batchPriorityValue >= 0 || batchPriorityValue <= -1000000000) throw new Error(`${configPath}.${path}.resourceProtection.batchPriorityValue must be between -999999999 and -1`); + return { + priorityClassName, + priorityValue, + batchPriorityClassName, + batchPriorityValue, + app: containerResources(raw.app, `${path}.resourceProtection.app`), + redis: containerResources(raw.redis, `${path}.resourceProtection.redis`), + }; +} + export function parseTargetRuntime(value: unknown, path: string): Sub2ApiTargetConfig["runtime"] { if (value === undefined || value === null) return { autoSetup: true }; if (typeof value !== "object" || Array.isArray(value)) throw new Error(`${configPath}.${path}.runtime must be an object`); diff --git a/scripts/src/platform-infra/entry.ts b/scripts/src/platform-infra/entry.ts index 59b2b648..4bef9377 100644 --- a/scripts/src/platform-infra/entry.ts +++ b/scripts/src/platform-infra/entry.ts @@ -127,6 +127,14 @@ export interface Sub2ApiTargetConfig { runtime: { autoSetup: boolean; }; + resourceProtection: { + priorityClassName: string; + priorityValue: number; + batchPriorityClassName: string; + batchPriorityValue: number; + app: Sub2ApiContainerResources; + redis: Sub2ApiContainerResources; + } | null; image: Partial; dependencyImages: Partial; hostDocker: Sub2ApiHostDockerConfig | null; @@ -135,6 +143,11 @@ export interface Sub2ApiTargetConfig { accountLocalProxy: Sub2ApiAccountLocalProxyConfig | null; } +export interface Sub2ApiContainerResources { + requests: { cpu: string; memory: string }; + limits: { cpu: string; memory: string }; +} + export interface Sub2ApiHostDockerConfig { projectName: string; workDir: string; diff --git a/scripts/src/platform-infra/manifest.ts b/scripts/src/platform-infra/manifest.ts index 58fe9c3a..ed61e8ff 100644 --- a/scripts/src/platform-infra/manifest.ts +++ b/scripts/src/platform-infra/manifest.ts @@ -304,7 +304,36 @@ export function externalPendingManifest(sub2api: Sub2ApiConfig, target: Sub2ApiT const runtimeConfigInitContainer = renderRuntimeConfigInitContainer(target, database, dependencyImages); const accountLocalProxyContainer = renderAccountLocalProxyContainer(target.accountLocalProxy); const accountLocalProxyVolume = renderAccountLocalProxyVolume(target.accountLocalProxy); - return `apiVersion: v1 + const protection = target.resourceProtection; + const priorityClass = protection === null ? "" : `apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: ${protection.priorityClassName} + labels: + app.kubernetes.io/managed-by: unidesk + app.kubernetes.io/part-of: platform-infra +value: ${protection.priorityValue} +globalDefault: false +preemptionPolicy: PreemptLowerPriority +description: UniDesk platform core workloads protected from batch CI pressure. +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: ${protection.batchPriorityClassName} + labels: + app.kubernetes.io/managed-by: unidesk + app.kubernetes.io/part-of: platform-infra +value: ${protection.batchPriorityValue} +globalDefault: false +preemptionPolicy: Never +description: UniDesk batch CI yields resources to platform core workloads. +--- +`; + const priorityClassName = protection === null ? "" : ` priorityClassName: ${protection.priorityClassName}\n`; + const redisResources = protection === null ? "" : renderContainerResources(protection.redis, 10); + const appResources = protection === null ? "" : renderContainerResources(protection.app, 10); + return `${priorityClass}apiVersion: v1 kind: Namespace metadata: name: ${target.namespace} @@ -462,13 +491,13 @@ spec: app.kubernetes.io/component: redis app.kubernetes.io/part-of: platform-infra spec: - securityContext: +${priorityClassName} securityContext: fsGroup: 999 containers: - name: redis image: ${dependencyImages.redis} imagePullPolicy: IfNotPresent - command: +${redisResources} command: - sh - -c args: @@ -535,7 +564,7 @@ spec: app.kubernetes.io/component: app app.kubernetes.io/part-of: platform-infra spec: - securityContext: +${priorityClassName} securityContext: fsGroup: 1000 initContainers: ${runtimeConfigInitContainer} @@ -557,7 +586,7 @@ ${runtimeConfigInitContainer} - name: sub2api image: ${imageRef(sub2api, target)} imagePullPolicy: ${image.pullPolicy} - ports: +${appResources} ports: - name: http containerPort: 8080 envFrom: @@ -620,6 +649,11 @@ ${egressProxy} `; } +function renderContainerResources(resources: { requests: { cpu: string; memory: string }; limits: { cpu: string; memory: string } }, spaces: number): string { + const indent = " ".repeat(spaces); + return `${indent}resources:\n${indent} requests:\n${indent} cpu: ${resources.requests.cpu}\n${indent} memory: ${resources.requests.memory}\n${indent} limits:\n${indent} cpu: ${resources.limits.cpu}\n${indent} memory: ${resources.limits.memory}\n`; +} + export function sub2ApiProxyEnv(target: Sub2ApiTargetConfig): { httpProxy: string; noProxy: string } { const proxy = target.egressProxy; if (proxy === null || !proxy.enabled || !proxy.applyToSub2Api) return { httpProxy: "", noProxy: "" }; diff --git a/scripts/src/platform-infra/policy.ts b/scripts/src/platform-infra/policy.ts index 834cb277..c9f570e0 100644 --- a/scripts/src/platform-infra/policy.ts +++ b/scripts/src/platform-infra/policy.ts @@ -118,9 +118,11 @@ export function policyChecks(sub2api: Sub2ApiConfig, yaml: string, target: Sub2A detail: "Pods must not expose host ports.", }, { - name: "no-cpu-memory-resources", - ok: !/^\s*(cpu|memory):\s*/mu.test(yaml), - detail: "Issue #220 requires no Kubernetes CPU/memory requests or limits.", + name: "core-resource-protection", + ok: target.resourceProtection === null || (/^\s*kind:\s*PriorityClass\s*$/mu.test(yaml) && /^\s*priorityClassName:\s*/mu.test(yaml) && /^\s*resources:\s*$/mu.test(yaml)), + detail: target.resourceProtection === null + ? "This target does not declare a core resource protection contract." + : `YAML renders PriorityClass ${target.resourceProtection.priorityClassName} and explicit app/Redis requests and limits.`, }, { name: "no-resource-quota-or-limit-range", diff --git a/scripts/src/platform-infra/status-script.ts b/scripts/src/platform-infra/status-script.ts index 3cc9543f..705a3d8c 100644 --- a/scripts/src/platform-infra/status-script.ts +++ b/scripts/src/platform-infra/status-script.ts @@ -249,7 +249,7 @@ def is_allow_all_network_policy(item): and spec.get("egress") == [{}] ) -def resource_findings(kind, collection): +def boundary_violations(kind, collection): findings = [] for item in collection: spec = item.get("spec") or {} @@ -258,11 +258,6 @@ def resource_findings(kind, collection): findings.append({"kind": kind, "name": item["metadata"]["name"], "field": "hostNetwork"}) all_containers = [(container, "containers") for container in template_spec.get("containers", [])] + [(container, "initContainers") for container in template_spec.get("initContainers", [])] for container, container_group in all_containers: - resources = container.get("resources") or {} - if resources.get("requests"): - findings.append({"kind": kind, "name": item["metadata"]["name"], "container": container.get("name"), "containerGroup": container_group, "field": "resources.requests"}) - if resources.get("limits"): - findings.append({"kind": kind, "name": item["metadata"]["name"], "container": container.get("name"), "containerGroup": container_group, "field": "resources.limits"}) for port in container.get("ports", []): if "hostPort" in port: findings.append({"kind": kind, "name": item["metadata"]["name"], "container": container.get("name"), "containerGroup": container_group, "field": "hostPort", "value": port.get("hostPort")}) @@ -297,7 +292,23 @@ for svc in services: for port in spec.get("ports", []): if "nodePort" in port: service_violations.append({"name": svc["metadata"]["name"], "nodePort": port.get("nodePort")}) -resource_violations = resource_findings("Deployment", deployments) + resource_findings("StatefulSet", statefulsets) +resource_violations = boundary_violations("Deployment", deployments) + boundary_violations("StatefulSet", statefulsets) +expected_resource_protection = json.loads(${JSON.stringify(JSON.stringify(target.resourceProtection))}) +def workload_resource_contract(name, container_name): + workload = next((item for item in deployments if item.get("metadata", {}).get("name") == name), None) + pod_spec = (((workload or {}).get("spec") or {}).get("template") or {}).get("spec") or {} + container = next((item for item in pod_spec.get("containers", []) if item.get("name") == container_name), None) + return {"priorityClassName": pod_spec.get("priorityClassName"), "resources": (container or {}).get("resources") or {}} +resource_contracts = { + "app": workload_resource_contract("${serviceName}", "${serviceName}"), + "redis": workload_resource_contract("${redisService}", "redis"), +} +resource_protection_aligned = expected_resource_protection is None or ( + resource_contracts["app"]["priorityClassName"] == expected_resource_protection["priorityClassName"] + and resource_contracts["redis"]["priorityClassName"] == expected_resource_protection["priorityClassName"] + and resource_contracts["app"]["resources"] == {"requests": expected_resource_protection["app"]["requests"], "limits": expected_resource_protection["app"]["limits"]} + and resource_contracts["redis"]["resources"] == {"requests": expected_resource_protection["redis"]["requests"], "limits": expected_resource_protection["redis"]["limits"]} +) expected_image = "${expectedImage}" expected_url_allowlist = json.loads(${JSON.stringify(JSON.stringify(expectedUrlAllowlist))}) sub2api_deployment = next((deployment_summary(item) for item in deployments if item["metadata"]["name"] == "${serviceName}"), None) @@ -412,7 +423,7 @@ else: state_model_ok = local_postgres_present and sub2api_desired_aligned and redis_desired_aligned status_label = "pending-external-db" if external_pending else "external-db-active" if external_active else "active" payload = { - "ok": rc("ns") == 0 and workload_ready and image_aligned and url_allowlist_aligned and proxy_env_aligned and network_policy["ok"] and boundary["internalOnly"] and len(resource_violations) == 0 and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and secret_ok and state_model_ok and rc("local-https") == 0, + "ok": rc("ns") == 0 and workload_ready and image_aligned and url_allowlist_aligned and proxy_env_aligned and network_policy["ok"] and boundary["internalOnly"] and len(resource_violations) == 0 and resource_protection_aligned and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and secret_ok and state_model_ok and rc("local-https") == 0, "target": "${target.id}", "route": "${target.route}", "namespace": "${target.namespace}", @@ -428,6 +439,7 @@ payload = { "services": [service_summary(item) for item in services], "pvcs": [pvc_summary(item) for item in pvcs], "networkPolicy": network_policy, + "resourceProtection": {"expected": expected_resource_protection, "observed": resource_contracts, "aligned": resource_protection_aligned}, "localHttpsExposure": { "enabled": ${localHttps === null ? "False" : "True"}, "action": text("local-https-action.txt"),