From a0491e9dc55820c3793167961b3c84f5e2ffd71d Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 03:11:03 +0200 Subject: [PATCH] =?UTF-8?q?fix(cicd):=20=E6=81=A2=E5=A4=8D=20PaC=20?= =?UTF-8?q?=E5=85=AC=E5=85=B1=E7=8A=B6=E6=80=81=E8=A7=82=E5=AF=9F=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .agents/skills/unidesk-cicd/SKILL.md | 10 ++ .../references/incident-recovery.md | 36 +++++ .../R4.1.1_Task_Report.md | 47 +++++- ...latform-infra-pac-failure-evidence.test.ts | 15 +- ...-infra-pipelines-as-code-bootstrap.test.ts | 26 ++++ ...platform-infra-pipelines-as-code-remote.sh | 142 +++++++++++------- .../src/platform-infra-pipelines-as-code.ts | 14 +- 7 files changed, 226 insertions(+), 64 deletions(-) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 5fd8754b..046171ee 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -116,6 +116,11 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 事故处理中曾在故障基线上推进共享 Repository 多分支改造，`#2223` 与 `#2226` 的不完整实现分别通过 `#2225` 与 `#2228` 回退；误合并的无关 PR `#2227` 通过 `#2229` 回退，不得让事故恢复夹带无关语义； - 基线恢复后，再通过独立 PR `#2230`、`#2231`、`#2233` 完整实现共享 Repository、多分支参数、release Argo 自动同步和资源镜像 ref，并分别通过正常自动事件与 release canary 验收； - 本案例的固定结论是：生产自动链故障的第一目标是缩小变更面并恢复已知健康状态，不是让当前失败变更通过更多补丁勉强工作。 +- PaC 公共 status renderer 故障必须先区分交付失败与观察失败： + - 先用 PipelineRun、Argo 与 runtime 独立事实确认自动交付是否成功，不得把 `parse-failure` 或 status timeout 直接表述为交付链失败； + - 引用缺失变量导致非法 JSON 时，优先从最后已知正确实现恢复该赋值，不得整体回退并撤销仍然正确的 fail-closed 业务语义； + - `status` 不得把慢 TaskRun 日志作为同步必需项，必须复用一次 TaskRun 读取并输出有界阶段标记；精确日志证据留给 id-specific `history` 或 `debug-step`； + - PipelineRun 成功、Argo `Synced/Healthy` 且 runtime ready 时，同步日志证据延后只能是 `blocking=false` warning，不能制造 false outage；详细案例见 [references/incident-recovery.md](references/incident-recovery.md)。 - PK01 的 CI/CD 默认只允许只读诊断： - 普通 PR merge、自动滚动、环境恢复或跨服务交付不构成 PK01 变更授权； - 只有用户在当前请求中明确要求修改 PK01，才允许提交或推广会改变 PK01 版本、配置、Secret 绑定、边缘路由、容器或运行面状态的 source 变更。 @@ -164,6 +169,11 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 仍要求 PipelineRun 成功、原 Argo `Synced/Healthy`、runtime ready、sha256 digest pin 与 health ready； - 证据缺失、冲突或 source commit 不匹配必须 fail-closed，不得等价为 delivery disabled，也不得伪造新 artifact、GitOps revision 或 runtime source commit。 - PaC `status|closeout|history` 的 `--json` 是有界 machine contract；`--full` 展开单条详情，`--raw` 才展开目标侧原始 payload。`history --id --full` 不得重复嵌套同一 rows/consumer payload或依赖 `/tmp/unidesk-cli-output` 才可读。 +- PaC `status` 必须在 `observability.readOnlyCapture.timeoutMs` 内完成： + - 目标侧通过 `pac-status-progress` 披露最后阶段，CLI 将其投影为 `observation.remoteStage`； + - PipelineRun 与 TaskRun 原始对象各只读取一次并复用，禁止 status 为同一 run 串行读取每个 Pod 日志； + - status 的 TaskRun 日志证据模式固定为 `taskrun-status-only`，需要日志时使用 id-specific 下钻； + - 不得通过提高 YAML timeout 掩盖重复采集、同步日志或无界远端查询。 - PaC `debug-step --id ` 必须读取目标侧真实 PipelineRun、TaskRun condition/result 与合同 Task 日志： - 默认输出每个 terminal role 的 presence、status、source 与首个断点； - 同时披露 artifact catalog、digest、GitOps commit 和 collector 读错； diff --git a/.agents/skills/unidesk-cicd/references/incident-recovery.md b/.agents/skills/unidesk-cicd/references/incident-recovery.md index fdca23e3..521b09ed 100644 --- a/.agents/skills/unidesk-cicd/references/incident-recovery.md +++ b/.agents/skills/unidesk-cicd/references/incident-recovery.md @@ -107,3 +107,39 @@ - 先恢复最后已知健康状态； - 再做独立前向修复； - 每个 PR 必须单一职责、可独立验证、可独立回退。 + +## 案例：PaC status 公共 renderer 生成非法 JSON + +- 现象分层： + - SelfMedia PR `#82` 合并后 `14s` 自动触发 PipelineRun； + - PipelineRun 对 source commit `8c03bcca554457315bc16bad01e5a96764276472` 执行成功，耗时 `256s`； + - Argo 与 runtime 正常，`pac-read-parse-failure` 只发生在共享 status 观察器； + - 因此这是公共观察面事故，不是 SelfMedia 私有 YAML 或自动交付失败。 +- 直接根因： + - `80ca187c` 中 `bootstrap_ready` 的赋值原本正确； + - `1f3283ff` 在回退 admission 业务阻断时删除该赋值； + - `2b385f08` 恢复 consumer bootstrap fail-closed 引用，却没有恢复赋值； + - shell 最终拼出非法 `"ok":,`，影响所有复用共享 renderer 的 PaC consumer。 +- 回退决策： + - 不整体回退 `2b385f08`，因为整体回退会再次丢失正确的 bootstrap fail-closed 语义； + - 只从最后已知正确实现恢复 `bootstrap_ready` 赋值，并实际执行三种 bootstrap fixture 后 `JSON.parse` 输出； + - 这是对致因遗漏的最小语义回退，不是在错误基础上继续扩展 admission 设计。 +- 恢复期间暴露的第二断点： + - status 在 `25000ms` YAML 预算内超时，阶段标记锁定在 `artifact-evidence`； + - 共享 collector 对同一 TaskRun 重复读取，并为成功 run 同步串行执行 `kubectl logs`； + - NC01 上同一条有界日志读取实测需要 `18s` 到 `38.5s`，不能作为 status 同步必需项。 +- 公共面修复： + - status 只读取一次 TaskRun JSON，并在 task summary 与 artifact terminal record 间复用； + - status 使用 `taskrun-status-only`，不再同步读取 Pod 日志； + - history collector 收敛为一次 selector 聚合，保留失败日志下钻； + - 目标侧输出无敏感值的 `pac-status-progress`，CLI 在 timeout observation 中显示 `remoteStage`； + - PipelineRun 成功、Argo `Synced/Healthy` 且 runtime ready 时，缺少同步日志证据降级为 `pac-artifact-log-evidence-deferred` 非阻断告警。 +- 验收结论： + - 原 `status --target NC01 --consumer selfmedia-nc01 --json` 在 `14.5s` 内返回合法 JSON； + - 输出 `ok=true`、`remoteStage=complete`、PipelineRun `Succeeded`、Argo `Synced/Healthy`、runtime `1/1`； + - 全程未修改 SelfMedia 私有 YAML，未人工执行 PipelineRun、mirror、trigger、sync、flush、Argo sync 或 runtime patch。 +- 长期结论： + - 先区分 delivery failure 与 observer failure； + - 修复共享 renderer 必须取得公共服务面授权； + - 先恢复最后已知正确语义，再删除重复、无界和同步日志读取； + - status 负责快速结构化摘要，id-specific 入口负责慢日志证据。 diff --git a/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R4.1.1_Task_Report.md b/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R4.1.1_Task_Report.md index dbd24b02..da56c41c 100644 --- a/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R4.1.1_Task_Report.md +++ b/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R4.1.1_Task_Report.md @@ -5,7 +5,7 @@ - MDTODO：`R4.1.1` - GitHub issue：`pikasTech/unidesk#2039` - 状态：`in_progress` -- 原因：源码修复与定向验证已完成，尚未合并 PR，也未执行合并后的 NC01 原入口运行面验收。 +- 原因：公共 status renderer 源码修复、定向验证与合并前 NC01 canary 已完成，尚未合并 PR，也未执行合并后的原入口验收。 ## 问题事实 @@ -56,7 +56,7 @@ - `sh -n scripts/src/platform-infra-pipelines-as-code-remote.sh` - `bun test scripts/src/platform-infra-pac-provenance.test.ts scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts` - - 结果：`23 pass, 0 fail, 239 expect()`。 + - 历史结果：`23 pass, 0 fail, 239 expect()`。 - `node` 调用 `runPacStatusFixtureChecks()`： - 结果：`ok=true, checks=34`。 - `bun scripts/cli.ts platform-infra pipelines-as-code plan --target NC01` @@ -76,8 +76,8 @@ ## 当前阻塞 -- 无源码或定向验证阻塞。 -- 运行面验收仍等待 PR 合并及主代理执行受控 bootstrap，因此任务保持 `in_progress`。 +- 无源码、定向验证或合并前 canary 阻塞。 +- 仍等待本次公共 status renderer PR 合并及合并后原入口 canary，因此任务保持 `in_progress`。 ## 启动 backlog 后续 @@ -103,3 +103,42 @@ - `bun scripts/cli.ts platform-infra pipelines-as-code plan --target NC01 --consumer hwlab-nc01-v03 --json`：成功披露 watcher probes，`warnings=[]`； - `git diff --check`：通过。 - 合并后只允许由 PaC migrated consumer 的正常 PR merge 自动链触发交付；不得人工执行 PipelineRun、mirror、trigger、sync、flush、Argo sync、runtime patch 或删除历史业务对象。 + +## 公共 status renderer 事故 + +- SelfMedia 自动交付事实： + - PR `#82` 合并后 `14s` 自动触发 PipelineRun； + - source commit 为 `8c03bcca554457315bc16bad01e5a96764276472`； + - PipelineRun `Succeeded`，耗时 `256s`，Argo 与 runtime 正常； + - 因此 `pac-read-parse-failure` 是公共观察面事故，不是 SelfMedia 私有 YAML 或自动交付失败。 +- 源码溯源： + - `80ca187c` 中 `bootstrap_ready` 赋值正确； + - `1f3283ff` 在回退 admission 业务阻断时删除赋值； + - `2b385f08` 恢复 consumer bootstrap fail-closed 引用，但漏恢复赋值； + - renderer 最终生成非法 `"ok":,`，影响所有复用共享脚本的 PaC consumer。 +- 最小语义回退： + - 不整体回退 `2b385f08`，避免再次丢失正确的 bootstrap fail-closed 语义； + - 只恢复最后已知正确的 `bootstrap_ready` 赋值； + - 实际执行未配置、ready 和 not-ready 三种 fixture，并对最终 JSON 调用 `JSON.parse`。 +- 第二断点与恢复： + - 原 status 连续在 YAML `25000ms` 预算超时； + - `pac-status-progress` 将最后阶段锁定为 `artifact-evidence`； + - collector 对同一 TaskRun 重复读取，并同步拉取成功 run 日志； + - 同一 NC01 日志查询实测耗时约 `18s` 至 `38.5s`。 +- 公共面修复： + - status 只读取一次 TaskRun JSON，并由 task summary 与 artifact terminal record 复用； + - status 使用 `taskrun-status-only`，不再同步拉 Pod 日志； + - history 日志采集收敛为一次 selector 聚合，精确日志继续由 id-specific 入口承担； + - CLI 将最后阶段投影为 `observation.remoteStage`； + - PipelineRun 成功、Argo `Synced/Healthy` 且 runtime ready 时，缺少同步日志证据降级为 `pac-artifact-log-evidence-deferred` 非阻断告警； + - 默认 compact JSON 删除静态 capabilities 冗余，保持在 CLI 输出预算内。 +- 合并前 canary： + - `bun scripts/cli.ts platform-infra pipelines-as-code status --target NC01 --consumer selfmedia-nc01 --json`； + - 结果：`ok=true`、`remoteStage=complete`，远端读取约 `14.5s`； + - PipelineRun `Succeeded`，Argo `Synced/Healthy`，runtime `1/1`； + - 全程未修改 SelfMedia 私有 YAML，未人工执行 PipelineRun、mirror、trigger、sync、flush、Argo sync 或 runtime patch。 +- 本轮验证： + - bootstrap 与 failure evidence 定向测试：`23 pass, 0 fail, 156 expect()`； + - 加入 provenance 测试后：`33 pass, 1 fail`； + - 唯一失败是旧 consumer 列表断言缺少现有 `agentrun-nc01-release` 与 `pikaoa-nc01`，与本次补丁无关，未扩大范围修改； + - `sh -n`、`git diff --check` 与 skill `quick_validate.py` 均通过。 diff --git a/scripts/src/platform-infra-pac-failure-evidence.test.ts b/scripts/src/platform-infra-pac-failure-evidence.test.ts index d877dc70..56462759 100644 --- a/scripts/src/platform-infra-pac-failure-evidence.test.ts +++ b/scripts/src/platform-infra-pac-failure-evidence.test.ts @@ -61,7 +61,7 @@ describe("PaC 失败证据合同", () => { action: "history", targetId: "NC01", consumerId: "agentrun-nc01-v02", - result: { exitCode: 124, stdout: "", stderr: "UNIDESK_SSH_RUNTIME_TIMEOUT" }, + result: { exitCode: 124, stdout: "", stderr: '{"event":"pac-status-progress","stage":"artifact-evidence","valuesPrinted":false}\nUNIDESK_SSH_RUNTIME_TIMEOUT' }, parsed: null, elapsedMs: 25001, timeoutMs: 25000, @@ -74,6 +74,7 @@ describe("PaC 失败证据合同", () => { target: "NC01", consumer: "agentrun-nc01-v02", stage: "history-remote-observe", + remoteStage: "artifact-evidence", elapsedMs: 25001, timeoutMs: 25000, configPath: "config/platform-infra/pipelines-as-code.yaml#observability.readOnlyCapture.timeoutMs", @@ -156,10 +157,18 @@ describe("PaC 失败证据合同", () => { expect(JSON.stringify(observation)).not.toContain("process.env"); }); - test("collector 遍历全部 matching TaskRun，terminal record 仍限定合同任务", () => { + test("collector 以单次有界 selector 拉取 PipelineRun 日志，terminal record 仍限定合同任务", () => { const remote = readFileSync(resolve(import.meta.dir, "platform-infra-pipelines-as-code-remote.sh"), "utf8"); - expect(remote).toContain("for (const item of matching)"); expect(remote).toContain("const contractTaskRuns = matching.filter((item) => taskTerminalRecord(item) !== null)"); + expect(remote).toContain("'--tail=320', '--max-log-requests=20'"); + expect(remote.match(/cp\.spawnSync\('kubectl', \['-n', namespace, 'logs'/gu)).toHaveLength(2); + expect(remote).toContain("if (process.env.UNIDESK_PAC_ACTION !== 'status') {"); + expect(remote).toContain("let mode = 'taskrun-status-only';"); + expect(remote).toContain('payload_file="${UNIDESK_PAC_TASKRUN_FILE:-}"'); + expect(remote).toContain('task_file="${UNIDESK_PAC_TASKRUN_FILE:-}"'); + expect(remote).toContain('export UNIDESK_PAC_TASKRUN_FILE="$task_file"'); + expect(remote).not.toContain("for (const item of matching)"); + expect(remote).not.toContain("for (const item of taskRuns)"); expect(remote).toContain("Object.keys(record(row.firstBreak)).length > 0"); expect(remote).toContain("traceId: row.traceId || null"); }); diff --git a/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts b/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts index b149c1ff..3b81f0a3 100644 --- a/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts +++ b/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts @@ -1,4 +1,5 @@ import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import type { GiteaConfig, GiteaMirrorRepository } from "./platform-infra-gitea-config"; @@ -216,11 +217,36 @@ test("PaC apply checks the Gitea repository before dry-run return and cluster mu expect(remote).toContain('"admissionProvenance":%s'); expect(remote).toContain('"error":"gitea-repository-missing"'); const statusAction = remote.slice(remote.indexOf("status_action() {"), remote.indexOf("history_action() {")); + const bootstrapReadyAssignment = statusAction.split("\n").find((line) => line.includes("bootstrap_ready=$(")); + expect(bootstrapReadyAssignment).toBeDefined(); + expect(remote).toContain("status_progress() {"); + expect(statusAction).toContain("status_progress artifact-evidence"); + expect(statusAction).toContain("status_progress diagnostics"); + expect(statusAction).toContain("status_progress complete"); + expect(statusAction).toContain("output.code === 'pac-gitops-missing'"); + expect(statusAction).toContain("output.code = 'pac-artifact-log-evidence-deferred'"); + expect(statusAction).toContain("blocking: false"); expect(statusAction).toContain('[ "$bootstrap_ready" = "true" ]'); expect(statusAction).toContain("output.code = 'pac-consumer-bootstrap-not-ready'"); expect(statusAction).toContain("output.ok = false"); expect(statusAction).toContain("code: 'pac-admission-provenance-not-ready'"); expect(statusAction).toContain("blocking: false"); + for (const fixture of [ + { state: { configured: false, ready: null }, expected: true }, + { state: { configured: true, ready: true }, expected: true }, + { state: { configured: true, ready: false }, expected: false }, + ]) { + const smoke = spawnSync("sh", ["-eu", "-c", [ + "consumer_bootstrap=$UNIDESK_PAC_BOOTSTRAP_STATE", + bootstrapReadyAssignment!.trim(), + `printf '{"ok":%s}\\n' "$( [ "$bootstrap_ready" = "true" ] && echo true || echo false )"`, + ].join("\n")], { + encoding: "utf8", + env: { ...process.env, UNIDESK_PAC_BOOTSTRAP_STATE: JSON.stringify(fixture.state) }, + }); + expect(smoke).toMatchObject({ status: 0, stderr: "" }); + expect(JSON.parse(smoke.stdout)).toEqual({ ok: fixture.expected }); + } }); test("PaC release manifest renders YAML-owned watcher startup backlog probes", () => { diff --git a/scripts/src/platform-infra-pipelines-as-code-remote.sh b/scripts/src/platform-infra-pipelines-as-code-remote.sh index 2f0ddc63..230c3b3c 100644 --- a/scripts/src/platform-infra-pipelines-as-code-remote.sh +++ b/scripts/src/platform-infra-pipelines-as-code-remote.sh @@ -777,33 +777,15 @@ function envReuseForPipelineRun(namespace, name, taskRuns) { const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean); const logErrors = []; let logs = ''; - for (const item of matching) { - const podName = item.status?.podName; - if (!podName) { - logErrors.push({ taskRun: item.metadata?.name || null, reason: 'pod-name-missing' }); - continue; - } - const result = cp.spawnSync('kubectl', ['-n', namespace, 'logs', podName, '--all-containers=true'], { - encoding: 'utf8', - timeout: 12000, - maxBuffer: 4 * 1024 * 1024, - }); - if (result.error || result.status !== 0) { - logErrors.push({ taskRun: item.metadata?.name || null, reason: result.error ? String(result.error.message || result.error) : String(result.stderr || '').slice(0, 240) }); - continue; - } - logs += `${result.stdout || ''}\n`; - } - if (contractTaskRuns.length === 0) { - state.source = 'pipeline-run-log-fallback'; - const result = cp.spawnSync('kubectl', ['-n', namespace, 'logs', '-l', `tekton.dev/pipelineRun=${name}`, '--all-containers=true', '--tail=320'], { - encoding: 'utf8', - timeout: 12000, - maxBuffer: 2 * 1024 * 1024, - }); - if (result.error || result.status !== 0) { - return { ...state, source: 'logs-unavailable', collector: { mode: state.source, contractTaskCount: 0, terminalRecordCount: 0, logErrorCount: 1, valuesPrinted: false } }; - } + const result = cp.spawnSync('kubectl', ['-n', namespace, 'logs', '-l', `tekton.dev/pipelineRun=${name}`, '--all-containers=true', '--tail=320', '--max-log-requests=20'], { + encoding: 'utf8', + timeout: 12000, + maxBuffer: 4 * 1024 * 1024, + }); + if (result.error || result.status !== 0) { + state.source = 'logs-unavailable'; + logErrors.push({ taskRun: null, reason: result.error ? String(result.error.message || result.error) : String(result.stderr || '').slice(0, 240) }); + } else { logs = result.stdout || ''; } const lines = logs.split(/\r?\n/u); @@ -1085,8 +1067,13 @@ NODE } task_rows() { - payload_file=$(mktemp) - kubectl -n "$UNIDESK_PAC_TARGET_NAMESPACE" get taskrun -o json >"$payload_file" 2>/dev/null || printf '{"items":[]}' >"$payload_file" + payload_file="${UNIDESK_PAC_TASKRUN_FILE:-}" + remove_payload=false + if [ -z "$payload_file" ]; then + payload_file=$(mktemp) + remove_payload=true + kubectl -n "$UNIDESK_PAC_TARGET_NAMESPACE" get taskrun -o json >"$payload_file" 2>/dev/null || printf '{"items":[]}' >"$payload_file" + fi node - "$payload_file" <<'NODE' const fs = require('node:fs'); const input = fs.readFileSync(process.argv[2], 'utf8') || '{"items":[]}'; @@ -1127,7 +1114,7 @@ const rows = (data.items || []) }); process.stdout.write(JSON.stringify(rows)); NODE - rm -f "$payload_file" + if [ "$remove_payload" = true ]; then rm -f "$payload_file"; fi } artifact_summary() { @@ -1135,8 +1122,13 @@ artifact_summary() { printf '{}' return fi - task_file=$(mktemp) - kubectl -n "$UNIDESK_PAC_TARGET_NAMESPACE" get taskrun -l "tekton.dev/pipelineRun=$UNIDESK_PAC_TARGET_PIPELINERUN" -o json >"$task_file" 2>/dev/null || printf '{"items":[]}' >"$task_file" + task_file="${UNIDESK_PAC_TASKRUN_FILE:-}" + remove_task_file=false + if [ -z "$task_file" ]; then + task_file=$(mktemp) + remove_task_file=true + kubectl -n "$UNIDESK_PAC_TARGET_NAMESPACE" get taskrun -l "tekton.dev/pipelineRun=$UNIDESK_PAC_TARGET_PIPELINERUN" -o json >"$task_file" 2>/dev/null || printf '{"items":[]}' >"$task_file" + fi node - "$task_file" <<'NODE' const fs = require('node:fs'); const cp = require('node:child_process'); @@ -1149,33 +1141,19 @@ const contractTaskRuns = taskRuns.filter((item) => taskTerminalRecord(item) !== const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean); const logErrors = []; let logText = ''; -for (const item of taskRuns) { - const podName = item.status?.podName; - if (!podName) { - logErrors.push({ taskRun: item.metadata?.name || null, reason: 'pod-name-missing' }); - continue; - } - const result = cp.spawnSync('kubectl', ['-n', namespace, 'logs', podName, '--all-containers=true'], { +let mode = 'taskrun-status-only'; +if (process.env.UNIDESK_PAC_ACTION !== 'status') { + mode = 'pipeline-run-logs'; + const result = cp.spawnSync('kubectl', ['-n', namespace, 'logs', '-l', `tekton.dev/pipelineRun=${pipelineRun}`, '--all-containers=true', '--tail=320', '--max-log-requests=20'], { encoding: 'utf8', timeout: 12000, maxBuffer: 4 * 1024 * 1024, }); - if (result.error || result.status !== 0) { - logErrors.push({ taskRun: item.metadata?.name || null, reason: result.error ? String(result.error.message || result.error) : String(result.stderr || '').slice(0, 240) }); - continue; + if (!result.error && result.status === 0) logText = result.stdout || ''; + else { + mode = 'logs-unavailable'; + logErrors.push({ taskRun: null, reason: result.error ? String(result.error.message || result.error) : String(result.stderr || '').slice(0, 240) }); } - logText += `${result.stdout || ''}\n`; -} -let mode = 'pipeline-task-logs'; -if (contractTaskRuns.length === 0) { - mode = 'pipeline-run-log-fallback'; - const fallback = cp.spawnSync('kubectl', ['-n', namespace, 'logs', '-l', `tekton.dev/pipelineRun=${pipelineRun}`, '--all-containers=true', '--tail=320'], { - encoding: 'utf8', - timeout: 12000, - maxBuffer: 2 * 1024 * 1024, - }); - if (!fallback.error && fallback.status === 0) logText = fallback.stdout || ''; - else logErrors.push({ taskRun: null, reason: fallback.error ? String(fallback.error.message || fallback.error) : String(fallback.stderr || '').slice(0, 240) }); } const records = [...parsePacLogRecords(logText), ...terminalRecords]; const out = extractPacArtifactEvidence(records, logText); @@ -1216,7 +1194,7 @@ out.collector = { }; process.stdout.write(JSON.stringify(out)); NODE - rm -f "$task_file" + if [ "$remove_task_file" = true ]; then rm -f "$task_file"; fi } runtime_summary() { @@ -1694,20 +1672,43 @@ NODE rm -f "$sa_file" "$role_binding_file" "$argo_secret_file" } +status_progress() { + printf '{"event":"pac-status-progress","stage":"%s","valuesPrinted":false}\n' "$1" >&2 +} + status_action() { + status_progress control-plane crd=$(kubectl get crd repositories.pipelinesascode.tekton.dev -o name 2>/dev/null || true) controller_ready=$(kubectl -n "$UNIDESK_PAC_RELEASE_NAMESPACE" get deploy "$UNIDESK_PAC_CONTROLLER_SERVICE_NAME" -o jsonpath='{.status.readyReplicas}/{.spec.replicas}' 2>/dev/null || echo "0/0") repository_condition=$(condition_status "$UNIDESK_PAC_TARGET_NAMESPACE" repository "$UNIDESK_PAC_REPOSITORY_NAME") + status_progress admission-provenance prepare_admission_provenance_state + status_progress consumer-bootstrap consumer_bootstrap=$(consumer_bootstrap_state) + bootstrap_ready=$(UNIDESK_PAC_BOOTSTRAP_STATE="$consumer_bootstrap" node -e 'const s=JSON.parse(process.env.UNIDESK_PAC_BOOTSTRAP_STATE||"{}"); process.stdout.write(s.configured===true&&s.ready!==true?"false":"true")') + status_progress pipeline-runs pipelines=$(pipeline_rows) latest=$(printf '%s' "$pipelines" | node -e 'const fs=require("fs"); const a=JSON.parse(fs.readFileSync(0,"utf8")||"[]"); process.stdout.write(a[0]?.name||"")') export UNIDESK_PAC_TARGET_PIPELINERUN="$latest" + status_progress task-runs + task_file=$(mktemp) + if [ -n "$latest" ]; then + kubectl -n "$UNIDESK_PAC_TARGET_NAMESPACE" get taskrun -l "tekton.dev/pipelineRun=$latest" -o json >"$task_file" 2>/dev/null || printf '{"items":[]}' >"$task_file" + else + printf '{"items":[]}' >"$task_file" + fi + export UNIDESK_PAC_TASKRUN_FILE="$task_file" tasks=$(task_rows) + status_progress artifact-evidence artifact=$(json_normalize "$(artifact_summary)") + unset UNIDESK_PAC_TASKRUN_FILE + rm -f "$task_file" + status_progress webhooks hooks=$(hook_summary) + status_progress argo argo=$(kubectl -n "$UNIDESK_PAC_ARGO_NAMESPACE" get application "$UNIDESK_PAC_ARGO_APPLICATION" -o json 2>/dev/null | node -e 'const fs=require("fs"); const s=fs.readFileSync(0,"utf8").trim(); if(!s){process.stdout.write("{}"); process.exit(0)} const a=JSON.parse(s); const source=a.spec?.source || a.spec?.sources?.[0] || {}; let repoURL=source.repoURL||null; try { const url=new URL(repoURL); url.username=""; url.password=""; repoURL=url.toString(); } catch {} process.stdout.write(JSON.stringify({sync:a.status?.sync?.status||null, health:a.status?.health?.status||null, revision:a.status?.sync?.revision||null, repoURL, targetRevision:source.targetRevision||null}))' || echo '{}') argo=$(json_normalize "$argo") + status_progress gitops-revision revision_relation=$(gitops_revision_relation "$artifact" "$argo") artifact=$(UNIDESK_PAC_ARTIFACT_JSON="$artifact" UNIDESK_PAC_REVISION_RELATION_JSON="$revision_relation" node <<'NODE' const artifact = JSON.parse(process.env.UNIDESK_PAC_ARTIFACT_JSON || '{}'); @@ -1721,13 +1722,41 @@ const revisionRelation = JSON.parse(process.env.UNIDESK_PAC_REVISION_RELATION_JS process.stdout.write(JSON.stringify({ ...argo, revisionRelation })); NODE ) + status_progress runtime runtime=$(json_normalize "$(runtime_summary)") + status_progress diagnostics diagnostics=$(cicd_diagnostics "$pipelines" "$artifact" "$argo" "$runtime") - diagnostics=$(UNIDESK_PAC_DIAGNOSTICS="$diagnostics" UNIDESK_PAC_STATE="$UNIDESK_PAC_ADMISSION_STATE_JSON" UNIDESK_PAC_BOOTSTRAP_STATE="$consumer_bootstrap" node <<'NODE' + diagnostics=$(UNIDESK_PAC_DIAGNOSTICS="$diagnostics" UNIDESK_PAC_STATE="$UNIDESK_PAC_ADMISSION_STATE_JSON" UNIDESK_PAC_BOOTSTRAP_STATE="$consumer_bootstrap" UNIDESK_PAC_PIPELINES_JSON="$pipelines" UNIDESK_PAC_ARTIFACT_JSON="$artifact" UNIDESK_PAC_ARGO_JSON="$argo" UNIDESK_PAC_RUNTIME_JSON="$runtime" node <<'NODE' const diagnostics = JSON.parse(process.env.UNIDESK_PAC_DIAGNOSTICS || '{}'); const admissionProvenance = JSON.parse(process.env.UNIDESK_PAC_STATE || '{}'); const consumerBootstrap = JSON.parse(process.env.UNIDESK_PAC_BOOTSTRAP_STATE || '{}'); +const pipelineRuns = JSON.parse(process.env.UNIDESK_PAC_PIPELINES_JSON || '[]'); +const artifact = JSON.parse(process.env.UNIDESK_PAC_ARTIFACT_JSON || '{}'); +const argo = JSON.parse(process.env.UNIDESK_PAC_ARGO_JSON || '{}'); +const runtime = JSON.parse(process.env.UNIDESK_PAC_RUNTIME_JSON || '{}'); const output = { ...diagnostics, admissionProvenance, consumerBootstrap, valuesPrinted: false }; +const latest = Array.isArray(pipelineRuns) ? pipelineRuns[0] || {} : {}; +const runtimeReady = Number.isInteger(runtime.readyReplicas) + && Number.isInteger(runtime.replicas) + && runtime.replicas > 0 + && runtime.readyReplicas >= runtime.replicas; +if (output.code === 'pac-gitops-missing' + && artifact.collector?.mode === 'taskrun-status-only' + && latest.status === 'True' + && argo.sync === 'Synced' + && argo.health === 'Healthy' + && runtimeReady) { + output.ok = true; + output.code = 'pac-artifact-log-evidence-deferred'; + output.phase = 'read-only-evidence-deferred'; + output.hint = 'PipelineRun, Argo, and runtime are ready; use id-specific history when synchronous TaskRun log evidence is required'; + output.warnings = [...(Array.isArray(output.warnings) ? output.warnings : []), { + code: 'pac-artifact-log-evidence-deferred', + blocking: false, + pipelineRun: latest.name || null, + valuesPrinted: false, + }]; +} if (process.env.UNIDESK_PAC_DELIVERY_PROVENANCE_REQUIRED === '1' && admissionProvenance.ready !== true) { output.warnings = [...(Array.isArray(output.warnings) ? output.warnings : []), { code: 'pac-admission-provenance-not-ready', @@ -1745,6 +1774,7 @@ if (consumerBootstrap.configured === true && consumerBootstrap.ready !== true) { process.stdout.write(JSON.stringify(output)); NODE ) + status_progress complete printf '{"ok":%s,"crdPresent":%s,"controllerReady":"%s","admissionProvenance":%s,"consumerBootstrap":%s,"repository":{"name":"%s","repo":"%s/%s","url":"%s","condition":"%s"},"repositoryCondition":"%s","webhooks":%s,"pipelineRuns":%s,"taskRuns":%s,"artifact":%s,"argo":%s,"runtime":%s,"diagnostics":%s,"valuesPrinted":false}\n' \ "$( [ -n "$crd" ] && [ "$controller_ready" != "0/0" ] && [ "$bootstrap_ready" = "true" ] && echo true || echo false )" \ "$( [ -n "$crd" ] && echo true || echo false )" \ diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index 4a4507db..8703bc2a 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -308,6 +308,7 @@ interface PacReadOnlyCaptureObservation { readonly target: string; readonly consumer: string; readonly stage: string; + readonly remoteStage: string | null; readonly elapsedMs: number; readonly timeoutMs: number; readonly configPath: string; @@ -1922,12 +1923,24 @@ export function pacReadOnlyCaptureObservation(options: { ? "parse-failure" : "succeeded"; const firstBreakCode = status === "succeeded" ? "none" : `pac-read-${status}`; + const remoteStage = options.result.stderr.split(/\r?\n/u).reduce((last, line) => { + if (!line.startsWith("{")) return last; + try { + const event = JSON.parse(line) as Record; + return event.event === "pac-status-progress" && typeof event.stage === "string" && /^[a-z0-9-]{1,80}$/u.test(event.stage) + ? event.stage + : last; + } catch { + return last; + } + }, null); return { ok: status === "succeeded", status, target: options.targetId, consumer: options.consumerId, stage: `${options.action}-remote-observe`, + remoteStage, elapsedMs: options.elapsedMs, timeoutMs: options.timeoutMs, configPath: options.configPath, @@ -2595,7 +2608,6 @@ function compactStatusJson(result: Record): Record