fix(cicd): 收敛手动发布就绪与耗时观测

This commit is contained in:
pikastech
2026-07-21 14:27:10 +02:00
parent d269c6b043
commit ffd8316f6a
6 changed files with 139 additions and 53 deletions
+2 -1
View File
@@ -129,7 +129,8 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help
- 唯一推荐入口是 `platform-infra pipelines-as-code delivery-timing --target <NODE> --consumer <id>`
- 一次调用直接披露 GitHub PR `mergedAt`、trigger wait、PipelineRun/TaskRun、端到端耗时、Argo、runtime health 和 `evidenceGaps`
- 证据不完整时保留已成立的耗时字段并返回 `partial` 与明确缺口,不得手工串联第二状态源;
- 端到端耗时严格超过 owning YAML 预算时输出 `pac-automatic-delivery-over-budget`,固定 `blocking=false``mutation=false`
- PaC migrated consumer 从 CLI webhook durable admission 开始计时,PR merge 到人工 trigger 的等待不计入发布耗时
- 端到端耗时严格超过 owning YAML 预算时输出 `pac-manual-delivery-over-budget`,固定 `blocking=false``mutation=false`,并直接披露最长 TaskRun 与 budget evidence gaps
- 超预算提示只下钻 env identity、依赖缓存、BuildKit/cache 和 stage timing,不给出人工恢复命令;
- 该入口固定只读且 `mutation=false`,不得用于同步、触发、补跑或运行面写入。
- 验收 registry cache 性能时,按 [Gitea + PaC](references/gitea-pac.md) 的冷热两次正常 PR 流程执行,不用人工 PipelineRun 或重复 status/timing 试探。
+29 -22
View File
@@ -699,7 +699,9 @@ function evaluatePacStatus(inputValue) {
const runtimeMatches = expectedDigest === null || runtimeDigest === expectedDigest;
const runtimeReady = Number.isInteger(runtime.replicas)
&& runtime.replicas > 0
&& runtime.readyReplicas === runtime.replicas;
&& runtime.readyReplicas === runtime.replicas
&& (!Number.isInteger(runtime.updatedReplicas) || runtime.updatedReplicas === runtime.replicas)
&& (!Number.isInteger(runtime.generation) || !Number.isInteger(runtime.observedGeneration) || runtime.observedGeneration >= runtime.generation);
const selectedArtifactDigests = [...new Set([
artifactDigest,
...(Array.isArray(artifact.digests) ? artifact.digests : []),
@@ -772,11 +774,6 @@ function evaluatePacStatus(inputValue) {
code = "pac-retained-gitops-missing";
phase = "retained-gitops-missing";
hint = "the no-runtime-change observation is valid but the retained Argo GitOps revision is missing";
} else if (noRuntimeChange && (argo.sync !== "Synced" || argo.health !== "Healthy")) {
ok = false;
code = "pac-retained-argo-not-ready";
phase = "retained-argo-not-ready";
hint = "the no-runtime-change observation is valid but retained Argo is not Synced/Healthy";
} else if (noRuntimeChange && gitopsRelation !== "retained") {
ok = false;
code = "pac-retained-revision-unproven";
@@ -800,7 +797,7 @@ function evaluatePacStatus(inputValue) {
} else if (noRuntimeChange) {
code = "pac-ready-no-runtime-change";
phase = "ready-no-runtime-change";
hint = "the successful source observation requires no artifact, GitOps or runtime change; retained Argo and runtime provenance remain healthy";
hint = "the successful source observation requires no artifact, GitOps or runtime change; retained runtime provenance and minimal health remain ready";
} else if (catalogPresent && !catalogSourceMatches) {
ok = false;
code = "pac-artifact-catalog-source-mismatch";
@@ -836,11 +833,6 @@ function evaluatePacStatus(inputValue) {
code = "pac-gitops-missing";
phase = "artifact-ready-gitops-missing";
hint = "PipelineRun completed but its GitOps commit is not visible";
} else if (argo.sync !== "Synced" || argo.health !== "Healthy") {
ok = false;
code = "pac-argo-not-ready";
phase = "gitops-ready-argo-pending";
hint = "GitOps exists but Argo is not Synced/Healthy";
} else if (!gitopsRevisionAligned) {
ok = false;
code = `pac-argo-revision-${gitopsRelation}`;
@@ -881,8 +873,8 @@ function evaluatePacStatus(inputValue) {
hint = deliverySkipped
? "YAML-declared runtime inputs are unchanged; the existing image, GitOps manifest, runtime and health remain aligned"
: gitopsRelation === "descendant"
? "Argo is on a commit-graph-proven GitOps descendant and the selected artifact, runtime, Argo and health are aligned"
: "Argo is on the exact selected GitOps commit and the artifact, runtime, Argo and health are aligned";
? "the observed GitOps revision is a commit-graph-proven descendant and the selected artifact, runtime and minimal health are aligned"
: "the observed GitOps revision exactly matches and the artifact, runtime and minimal health are aligned";
}
return {
@@ -1149,23 +1141,29 @@ function evaluatePacNodeStatusSnapshot(inputValue) {
const digest = digestFromImage(image);
const desired = Number.isInteger(deploymentSpec.replicas) ? deploymentSpec.replicas : null;
const readyReplicas = Number.isInteger(deploymentStatus.readyReplicas) ? deploymentStatus.readyReplicas : 0;
const updatedReplicas = Number.isInteger(deploymentStatus.updatedReplicas) ? deploymentStatus.updatedReplicas : 0;
const generation = Number.isInteger(record(deployment?.metadata).generation) ? deployment.metadata.generation : null;
const observedGeneration = Number.isInteger(deploymentStatus.observedGeneration) ? deploymentStatus.observedGeneration : null;
const pipelineReadReady = !readErrorStages.has("pipeline-runs");
const argoReadReady = !readErrorStages.has("argo-applications");
const runtimeReadReady = !readErrorStages.has("deployments");
const ciReady = pipelineReadReady && selected !== null && pipelineCondition.status === "True";
const argoReady = argoReadReady && application !== null && appSync.status === "Synced" && appHealth.status === "Healthy";
const argoReady = argoReadReady && application !== null;
const runtimeNotApplicable = target === null;
const runtimeReady = runtimeReadReady && (runtimeNotApplicable || (deployment !== null && desired !== null && desired > 0 && readyReplicas === desired));
const ready = ciReady && argoReady && runtimeReady;
const runtimeReady = runtimeReadReady && (runtimeNotApplicable || (deployment !== null
&& desired !== null
&& desired > 0
&& readyReplicas === desired
&& updatedReplicas === desired
&& generation !== null
&& observedGeneration !== null
&& observedGeneration >= generation));
const ready = ciReady && runtimeReady;
const reason = !pipelineReadReady
? "pipeline-read-failed"
: !ciReady
? selected === null ? "pipeline-missing" : `ci:${pipelineCondition.reason || pipelineCondition.status || "unknown"}`
: !argoReadReady
? "argo-read-failed"
: !argoReady
? "argo-not-ready"
: !runtimeReadReady
: !runtimeReadReady
? "runtime-read-failed"
: !runtimeReady
? "runtime-not-ready"
@@ -1202,7 +1200,10 @@ function evaluatePacNodeStatusSnapshot(inputValue) {
runtime: {
deployment: firstString(record(deployment?.metadata).name),
readyReplicas,
updatedReplicas,
replicas: desired,
generation,
observedGeneration,
digest,
image,
...(expectedSourceCommit === null ? {} : {
@@ -1360,6 +1361,12 @@ function runPacStatusFixtureChecks() {
expectedCode: "pac-ready-gitops-exact",
input: fixtureInput({ artifact: { imageStatus: "built", digest: digestA, digests: [digestA], gitopsCommit: revision, sourceObservation: delivery, catalog: publishedCatalog() }, argo: exactArgo, runtime: { image: `registry/service@${digestA}`, digest: digestA, replicas: 1, readyReplicas: 1 } }),
},
{
id: "delivery-ready-ignores-argo-aggregate-health",
expectedOk: true,
expectedCode: "pac-ready-gitops-exact",
input: fixtureInput({ artifact: { imageStatus: "built", digest: digestA, digests: [digestA], gitopsCommit: revision, sourceObservation: delivery, catalog: publishedCatalog() }, argo: { ...exactArgo, sync: "OutOfSync", health: "Degraded" }, runtime: { image: `registry/service@${digestA}`, digest: digestA, replicas: 1, readyReplicas: 1 } }),
},
{
id: "equal-revisions-ignore-owning-git-fetch-failure",
expectedOk: true,
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
import { observePacDeliveryBudget, observePacHistoryDeliveryBudget, observePacStatusDeliveryBudget } from "./platform-infra-pac-delivery-timing";
import { observePacDeliveryBudget, observePacHistoryDeliveryBudget, observePacStatusDeliveryBudget, pacDeliveryTimingWindow } from "./platform-infra-pac-delivery-timing";
const policy = {
endToEndBudgetSeconds: 120,
@@ -17,7 +17,31 @@ function observe(endToEndSeconds: number | null, complete: boolean) {
});
}
describe("PaC automatic delivery budget warning", () => {
describe("PaC manual delivery budget warning", () => {
test("manual delivery timing starts at PaC webhook admission instead of PR merge", () => {
expect(pacDeliveryTimingWindow({
triggeredAt: "2026-07-21T11:35:42Z",
startTime: "2026-07-21T11:35:52Z",
completionTime: "2026-07-21T11:37:42Z",
durationSeconds: 110,
})).toEqual({
deliveryStart: "2026-07-21T11:35:42Z",
deliveryStartSource: "pac-webhook-admission",
pipelineStart: "2026-07-21T11:35:52Z",
pipelineCompletion: "2026-07-21T11:37:42Z",
triggerWaitSeconds: 10,
pipelineDurationSeconds: 110,
endToEndSeconds: 120,
});
});
test("missing webhook admission stays unknown instead of falling back to PR merge", () => {
expect(pacDeliveryTimingWindow({
startTime: "2026-07-21T11:35:52Z",
completionTime: "2026-07-21T11:37:42Z",
})).toMatchObject({ deliveryStart: null, deliveryStartSource: null, triggerWaitSeconds: null, endToEndSeconds: null });
});
test("120 seconds stays within budget without a warning", () => {
const result = observe(120, true);
expect(result).toMatchObject({ state: "complete", overBudget: false, warning: null, blocking: false, mutation: false });
@@ -27,7 +51,7 @@ describe("PaC automatic delivery budget warning", () => {
const result = observe(121, true);
expect(result.overBudget).toBe(true);
expect(result.warning).toMatchObject({
code: "pac-automatic-delivery-over-budget",
code: "pac-manual-delivery-over-budget",
severity: "warning",
blocking: false,
mutation: false,
@@ -49,11 +49,48 @@ function durationBetween(start: unknown, end: unknown): number | null {
return Math.max(0, Math.round((b - a) / 1000));
}
export function pacDeliveryTimingWindow(row: Record<string, unknown>): Record<string, unknown> {
const deliveryStart = typeof row.triggeredAt === "string" ? row.triggeredAt : null;
const pipelineStart = typeof row.startTime === "string" ? row.startTime : null;
const pipelineCompletion = typeof row.completionTime === "string" ? row.completionTime : null;
return {
deliveryStart,
deliveryStartSource: deliveryStart === null ? null : "pac-webhook-admission",
pipelineStart,
pipelineCompletion,
triggerWaitSeconds: durationBetween(deliveryStart, pipelineStart),
pipelineDurationSeconds: typeof row.durationSeconds === "number"
? row.durationSeconds
: durationBetween(pipelineStart, pipelineCompletion),
endToEndSeconds: durationBetween(deliveryStart, pipelineCompletion),
};
}
function summarizeTaskRuns(value: unknown): Record<string, unknown> {
const taskRuns = record(value);
const details = records(taskRuns.details).map((item) => ({
name: item.name ?? null,
status: item.status ?? null,
reason: item.reason ?? null,
durationSeconds: item.durationSeconds ?? null,
}));
const timed = details.filter((item) => typeof item.durationSeconds === "number");
const longest = timed.reduce<Record<string, unknown> | null>((selected, item) => (
selected === null || Number(item.durationSeconds) > Number(selected.durationSeconds) ? item : selected
), null);
return {
count: typeof taskRuns.count === "number" ? taskRuns.count : details.length,
stages: details,
longest,
valuesPrinted: false,
};
}
export function observePacDeliveryBudget(input: PacDeliveryBudgetObservationInput): Record<string, unknown> {
const evidenceState = input.complete && input.endToEndSeconds !== null ? "complete" : "partial";
const overBudget = evidenceState === "complete" && (input.endToEndSeconds as number) > input.policy.endToEndBudgetSeconds;
const warning = overBudget ? {
code: "pac-automatic-delivery-over-budget",
code: "pac-manual-delivery-over-budget",
severity: "warning",
blocking: false,
mutation: false,
@@ -161,10 +198,12 @@ export async function observePacDeliveryTiming(
const latest = record(statusSummary.latestPipelineRun);
const commit = typeof row.commit === "string" ? row.commit : typeof latest.sourceCommit === "string" ? latest.sourceCommit : null;
const evidenceGaps: string[] = [];
const budgetEvidenceGaps: string[] = [];
let pullRequest: Record<string, unknown> | null = null;
let githubEvidence: Record<string, unknown> = { repository: binding.consumer.repository, source: "github-commit-pulls", mutation: false };
if (commit === null) {
evidenceGaps.push("source-commit-missing");
budgetEvidenceGaps.push("source-commit-missing");
} else {
const tokenInfo = resolveToken(binding.consumer.repository, false);
if (tokenInfo.token === null) {
@@ -190,19 +229,25 @@ export async function observePacDeliveryTiming(
}
}
const mergedAt = pullRequest?.mergedAt ?? null;
const startTime = row.startTime ?? null;
const completionTime = row.completionTime ?? null;
const timing = pacDeliveryTimingWindow(row);
const deliveryStart = timing.deliveryStart;
const startTime = timing.pipelineStart;
const completionTime = timing.pipelineCompletion;
const argo = record(statusSummary.argo);
const runtime = record(statusSummary.runtime);
const provenance = record(statusSummary.provenance);
if (mergedAt === null) evidenceGaps.push("mergedAt-missing");
if (startTime === null || completionTime === null) evidenceGaps.push("pipeline-timestamps-missing");
if (argo.sync === undefined || argo.health === undefined) evidenceGaps.push("argo-closeout-missing");
if (runtime.readyReplicas === undefined) evidenceGaps.push("runtime-health-missing");
if (provenance.relation === "unknown") evidenceGaps.push("runtime-provenance-unknown");
if (statusSummary.ready !== true) evidenceGaps.push("runtime-closeout-not-ready");
const state = evidenceGaps.length === 0 ? "complete" : "partial";
const endToEndSeconds = durationBetween(mergedAt, completionTime);
if (deliveryStart === null) budgetEvidenceGaps.push("manual-webhook-admission-missing");
if (startTime === null || completionTime === null) budgetEvidenceGaps.push("pipeline-timestamps-missing");
if (runtime.readyReplicas === undefined) budgetEvidenceGaps.push("runtime-health-missing");
const replicas = typeof runtime.replicas === "number" ? runtime.replicas : null;
const readyReplicas = typeof runtime.readyReplicas === "number" ? runtime.readyReplicas : null;
if (replicas !== null && (replicas <= 0 || readyReplicas === null || readyReplicas < replicas)) {
budgetEvidenceGaps.push("runtime-minimal-readiness-not-ready");
}
evidenceGaps.push(...budgetEvidenceGaps);
const state = budgetEvidenceGaps.length === 0 ? "complete" : "partial";
const endToEndSeconds = typeof timing.endToEndSeconds === "number" ? timing.endToEndSeconds : null;
const pipelineRunId = typeof row.id === "string"
? row.id
: typeof latest.name === "string"
@@ -225,19 +270,17 @@ export async function observePacDeliveryTiming(
target: binding.target,
consumer: binding.consumer,
source: { commit, pullRequest, github: githubEvidence },
timing: {
mergedAt,
pipelineStart: startTime,
pipelineCompletion: completionTime,
triggerWaitSeconds: durationBetween(mergedAt, startTime),
pipelineDurationSeconds: row.durationSeconds ?? durationBetween(startTime, completionTime),
endToEndSeconds,
},
timing: { mergedAt, ...timing },
deliveryBudget,
warnings: Object.keys(warning).length === 0 ? [] : [warning],
pipeline: { id: pipelineRunId, status: row.status ?? latest.status ?? null, taskRuns: row.taskRuns ?? statusSummary.taskRuns ?? null },
pipeline: {
id: pipelineRunId,
status: row.status ?? latest.status ?? null,
taskRuns: summarizeTaskRuns(row.taskRuns ?? statusSummary.taskRuns ?? null),
},
runtime: { argo, health: runtime, provenance },
evidenceGaps: [...new Set(evidenceGaps)],
budgetEvidenceGaps: [...new Set(budgetEvidenceGaps)],
valuesPrinted: false,
};
}
@@ -1537,7 +1537,10 @@ const workload = (deploy) => {
namespace: deploy.metadata?.namespace || null,
deployment: deploy.metadata?.name || null,
readyReplicas: deploy.status?.readyReplicas ?? 0,
updatedReplicas: deploy.status?.updatedReplicas ?? 0,
replicas: deploy.spec?.replicas ?? null,
generation: deploy.metadata?.generation ?? null,
observedGeneration: deploy.status?.observedGeneration ?? null,
sourceCommit: sourceCommit(deploy),
image,
digest,
@@ -2061,20 +2064,23 @@ const argo = JSON.parse(process.env.UNIDESK_PAC_ARGO_JSON || '{}');
const runtime = JSON.parse(process.env.UNIDESK_PAC_RUNTIME_JSON || '{}');
const output = { ...diagnostics, admissionProvenance, consumerBootstrap, valuesPrinted: false };
const latest = Array.isArray(pipelineRuns) ? pipelineRuns[0] || {} : {};
const runtimeReady = Number.isInteger(runtime.readyReplicas)
&& Number.isInteger(runtime.replicas)
&& runtime.replicas > 0
&& runtime.readyReplicas >= runtime.replicas;
const sourceWorkloads = Array.isArray(runtime.workloads)
? runtime.workloads.filter((item) => item?.sourceCommit === latest.sourceCommit)
: [];
const runtimeReady = sourceWorkloads.length > 0 && sourceWorkloads.every((item) => Number.isInteger(item.readyReplicas)
&& Number.isInteger(item.replicas)
&& item.replicas > 0
&& item.readyReplicas >= item.replicas
&& item.updatedReplicas >= item.replicas
&& item.observedGeneration >= item.generation);
if (['pac-gitops-missing', 'pac-source-observation-inconsistent'].includes(output.code)
&& artifact.collector?.mode === 'taskrun-status-only'
&& latest.status === 'True'
&& argo.sync === 'Synced'
&& argo.health === 'Healthy'
&& runtimeReady) {
output.ok = true;
output.code = 'pac-artifact-log-evidence-deferred';
output.phase = 'read-only-evidence-deferred';
output.hint = 'PipelineRun, Argo, and runtime are ready; use id-specific history when synchronous TaskRun log evidence is required';
output.hint = 'PipelineRun and exact-source workloads are minimally ready; use id-specific history when synchronous TaskRun log evidence is required';
output.warnings = [...(Array.isArray(output.warnings) ? output.warnings : []), {
code: 'pac-artifact-log-evidence-deferred',
blocking: false,
@@ -3656,16 +3656,21 @@ function renderDeliveryTiming(result: Record<string, unknown>): RenderedCliResul
const runtime = record(result.runtime);
const argo = record(runtime.argo);
const health = record(runtime.health);
const taskRuns = record(record(result.pipeline).taskRuns);
const longestTask = record(taskRuns.longest);
const deliveryBudget = record(result.deliveryBudget);
const warnings = arrayRecords(result.warnings);
const lines = [
"PLATFORM-INFRA DELIVERY TIMING",
`STATE: ${stringValue(result.state)} MUTATION: ${boolText(result.mutation)}`,
`SOURCE: ${stringValue(record(result.consumer).repository)}@${stringValue(source.commit)} PR#${stringValue(pr.number)}`,
`MERGED_AT: ${stringValue(timing.mergedAt)} PIPELINE_START: ${stringValue(timing.pipelineStart)} PIPELINE_DONE: ${stringValue(timing.pipelineCompletion)}`,
`MERGED_AT: ${stringValue(timing.mergedAt)} DELIVERY_START: ${stringValue(timing.deliveryStart)} (${stringValue(timing.deliveryStartSource)})`,
`PIPELINE_START: ${stringValue(timing.pipelineStart)} PIPELINE_DONE: ${stringValue(timing.pipelineCompletion)}`,
`TRIGGER_WAIT_S: ${stringValue(timing.triggerWaitSeconds)} PIPELINE_S: ${stringValue(timing.pipelineDurationSeconds)} END_TO_END_S: ${stringValue(timing.endToEndSeconds)}`,
`BUDGET_S: ${stringValue(deliveryBudget.budgetSeconds)} OVER_BUDGET: ${stringValue(deliveryBudget.overBudget)} BUDGET_STATE: ${stringValue(deliveryBudget.state)}`,
`LONGEST_TASK: ${stringValue(longestTask.name)} ${stringValue(longestTask.durationSeconds)}s status=${stringValue(longestTask.reason ?? longestTask.status)}`,
`ARGO: ${stringValue(argo.sync)}/${stringValue(argo.health)} RUNTIME_READY: ${stringValue(health.readyReplicas)}/${stringValue(health.replicas)}`,
`BUDGET_EVIDENCE_GAPS: ${(Array.isArray(result.budgetEvidenceGaps) ? result.budgetEvidenceGaps.map(String) : []).join(", ") || "-"}`,
`EVIDENCE_GAPS: ${(Array.isArray(result.evidenceGaps) ? result.evidenceGaps.map(String) : []).join(", ") || "-"}`,
...renderPacConfigWarnings(warnings),
];