diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index d38b7350..5fd8754b 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -32,6 +32,7 @@ bun scripts/cli.ts platform-infra pipelines-as-code status --target JD01 bun scripts/cli.ts platform-infra pipelines-as-code status --target JD01 --consumer hwlab-jd01-v03 bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 --limit 10 bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 --id +bun scripts/cli.ts platform-infra pipelines-as-code diagnose-regression --target NC01 --consumer bun scripts/cli.ts platform-infra pipelines-as-code delivery-timing --target NC01 --consumer selfmedia-nc01 bun scripts/cli.ts platform-infra pipelines-as-code closeout --target --consumer --wait --timeout 120s bun scripts/cli.ts platform-infra pipelines-as-code debug-step --target JD01 --consumer @@ -47,6 +48,15 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 单 consumer 再用 `platform-infra pipelines-as-code status --target --consumer ` 下钻; - 状态输出与 registry applicability 的长期判定见 [references/gitea-pac.md](references/gitea-pac.md)。 +- 生产事故回退诊断: + - 唯一推荐入口是 `platform-infra pipelines-as-code diagnose-regression --target --consumer `; + - 命令在目标侧只读取 PipelineRun 摘要,一次输出最后成功、首次失败、当前失败连续段、共同失败指纹、source commit 边界和回退候选; + - 默认共同指纹范围只覆盖 owning YAML 中与选中 consumer 共享 `repositoryRef` 的 lanes,避免扫描节点上无关业务仓库; + - `ok` 只表示读取成功,`incident` 表示是否存在当前事故,`read-failed` 不得伪装成健康; + - 默认窗口是每 consumer 20 条,可用 `--limit 1..50` 有界扩大;窗口不足时命令给出 `expand-window`,不得无限拉取; + - 首次失败需要 TaskRun 或日志证据时,只执行输出中的 `debug-first-failure` 精确下钻; + - 命令固定 `mutation=false`、`automatic=false`,只提供 review-required 回退候选,绝不执行回退或生成补链动作。 + - CI/CD 只读等待: - 默认使用 `closeout --wait --timeout 120s` 观察 Tekton、Argo 和 runtime 收敛;已处于成功或失败终态时立即返回。 - 未完成时由目标控制面的 Kubernetes watch 持有单次等待,不得在调用侧每 `1s`、`5s` 重复执行 `status`。 @@ -159,6 +169,13 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 同时披露 artifact catalog、digest、GitOps commit 和 collector 读错; - 该入口只读且 `mutation=false`,不能创建 PipelineRun 或写 GitOps/runtime; - 未带 `--id` 时才只运行 evaluator fixture。 +- PaC `diagnose-regression` 必须保持只读、轻量和有界: + - 目标侧只聚合 PipelineRun 元数据,不默认读取所有 TaskRun 日志; + - 默认只比较同一 `repositoryRef` 的 consumer lanes,并从 owning YAML 解析 owner,禁止按名称猜测; + - 必须区分 `ok`、`incident` 与 `state=read-failed`,timeout、空响应和读取错误均在默认输出中可见; + - `scopeAssessment=public-service` 或作用域未知时必须返回 `authorizationRequired=true`; + - `rollbackCandidate` 只能包含 compare 范围、候选 commit、置信度与 review 要求,固定 `automatic=false`; + - 禁止提供自动 revert、人工 PipelineRun、mirror sync、Argo sync、bootstrap 或 apply。 - GitHub/Git 相关 egress 必须走 YAML-first host proxy/sourceRef:branch-follower controller 读 `config/cicd-branch-followers.yaml#controller.source.githubSsh`,runtime legacy git-mirror 读 owning lane/control-plane YAML 的 host proxy 和 `githubTransport`,Gitea/PaC 迁移 lane 读 `config/platform-infra/gitea.yaml` 与 `config/platform-infra/pipelines-as-code.yaml`;禁止依赖未声明 host env、trans proxy、裸直连 GitHub 或 CLI 输出解析。 - Gitea/PaC lane 的 GitHub -> Gitea 自动同步由 owning YAML 渲染的 webhook bridge 与 durable inbox worker 承担: - receiver 验签并校验 GitHub payload 后,只有在 delivery 通过 fsync 与 atomic rename 写入 YAML 声明的 PVC inbox 后才能返回 HTTP `202 Accepted`; diff --git a/.agents/skills/unidesk-cicd/references/incident-recovery.md b/.agents/skills/unidesk-cicd/references/incident-recovery.md index e8741f67..fdca23e3 100644 --- a/.agents/skills/unidesk-cicd/references/incident-recovery.md +++ b/.agents/skills/unidesk-cicd/references/incident-recovery.md @@ -33,16 +33,22 @@ bun scripts/cli.ts cicd status --node ``` -2. 读取选中 consumer 最近事件,锁定首个失败 PipelineRun 和最后成功 PipelineRun: +2. 一次计算选中 consumer 的回退窗口与同 source authority 共同指纹: ```bash - bun scripts/cli.ts platform-infra pipelines-as-code history \ + bun scripts/cli.ts platform-infra pipelines-as-code diagnose-regression \ --target \ - --consumer \ - --limit 10 + --consumer ``` -3. 对首个失败 PipelineRun 读取 terminal role、TaskRun condition 和首个断点: + - `ok` 表示只读采集是否成功; + - `incident` 表示最新终态是否处于失败连续段; + - `window.lastKnownGood` 与 `window.firstFailed` 给出 source commit 边界; + - `commonFailure` 只在共享 `repositoryRef` 的 lanes 内聚类; + - `state=read-failed` 时先处理读取可见性,不得按健康解释; + - 窗口不足时只执行输出中的有界 `expand-window`。 + +3. 仅在需要精确 TaskRun 或日志证据时,对首次失败执行输出中的 `debug-first-failure`: ```bash bun scripts/cli.ts platform-infra pipelines-as-code debug-step \ @@ -51,7 +57,7 @@ --id ``` -4. 用 Git 比较最后成功 source commit 与首个失败 source commit: +4. 使用命令输出的 `sourceRange.compareUrl` 比较最后成功 source commit 与首次失败 source commit: - 只检查两者之间的提交; - 优先寻找与共同失败指纹直接相关的最近变更; - 记录致因文件、提交、影响 consumer 和可回退性; diff --git a/scripts/src/platform-infra-pac-regression-diagnosis.test.ts b/scripts/src/platform-infra-pac-regression-diagnosis.test.ts new file mode 100644 index 00000000..d55c7cad --- /dev/null +++ b/scripts/src/platform-infra-pac-regression-diagnosis.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import { diagnosePacRegression, pacFailureFingerprint, renderPacRegressionDiagnosis } from "./platform-infra-pac-regression-diagnosis"; + +const base = "a".repeat(40); +const failed = "b".repeat(40); + +function row(input: { + id: string; + consumer?: string; + status: "True" | "False" | "Unknown"; + at: string; + commit: string; + code?: string; + phase?: string; +}): Record { + return { + id: input.id, + consumer: input.consumer ?? "agentrun-nc01-v02", + deliveryAuthorityEligible: true, + status: input.status, + triggeredAt: input.at, + completionTime: input.at, + commit: input.commit, + traceId: input.status === "False" ? "1".repeat(32) : null, + firstBreak: input.status === "False" ? { + code: input.code ?? "child-process-exit-nonzero", + phase: input.phase ?? "collect-artifacts", + reason: "getcwd failed for run 123 at commit bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + } : null, + error: input.status === "False" ? { type: "ChildProcessError", code: "EXIT_NONZERO", exitCode: 1 } : null, + }; +} + +function diagnose(rows: Record[], bindings = [ + { id: "agentrun-nc01-v02", repositoryRef: "unidesk-nc01" }, +]): Record { + return diagnosePacRegression({ + targetId: "NC01", + consumerId: "agentrun-nc01-v02", + sourceRepository: "pikasTech/unidesk", + limitPerConsumer: 20, + rows, + consumerBindings: bindings, + }); +} + +describe("PaC regression diagnosis", () => { + test("finds the last known good and first failure in the active streak", () => { + const result = diagnose([ + row({ id: "fail-2", status: "False", at: "2026-07-16T03:00:00Z", commit: failed }), + row({ id: "fail-1", status: "False", at: "2026-07-16T02:00:00Z", commit: failed }), + row({ id: "good-1", status: "True", at: "2026-07-16T01:00:00Z", commit: base }), + ]); + expect(result).toMatchObject({ state: "active-regression", incident: true, mutation: false }); + expect(result.window).toMatchObject({ complete: true, failureStreak: 2, lastKnownGood: { id: "good-1" }, firstFailed: { id: "fail-1" } }); + expect(result.sourceRange).toMatchObject({ baseCommit: base, failedCommit: failed, candidateCommit: failed }); + expect(result.rollbackCandidate).toMatchObject({ state: "review-required", automatic: false, requiresReview: true, confidence: "high" }); + expect(JSON.stringify(result)).not.toMatch(/trigger-current|bootstrap|apply|argo sync/iu); + }); + + test("latest successful terminal run reports healthy after an older failure", () => { + const result = diagnose([ + row({ id: "good-2", status: "True", at: "2026-07-16T03:00:00Z", commit: failed }), + row({ id: "fail-1", status: "False", at: "2026-07-16T02:00:00Z", commit: failed }), + ]); + expect(result).toMatchObject({ state: "healthy", incident: false, mutation: false }); + expect(result.rollbackCandidate).toMatchObject({ state: "insufficient-evidence", automatic: false }); + }); + + test("missing successful baseline remains explicit and suggests a bounded wider query", () => { + const result = diagnose([ + row({ id: "fail-2", status: "False", at: "2026-07-16T03:00:00Z", commit: failed }), + row({ id: "fail-1", status: "False", at: "2026-07-16T02:00:00Z", commit: failed }), + ]); + expect(result).toMatchObject({ state: "active-regression-baseline-missing", incident: true }); + expect(result.window).toMatchObject({ complete: false, lastKnownGood: null }); + expect((result.next as Record).expandWindow).toContain("--limit 40"); + }); + + test("maximum window does not suggest an identical retry", () => { + const result = diagnosePacRegression({ + targetId: "NC01", + consumerId: "agentrun-nc01-v02", + sourceRepository: "pikasTech/agentrun", + limitPerConsumer: 50, + rows: [row({ id: "fail-1", status: "False", at: "2026-07-16T02:00:00Z", commit: failed })], + consumerBindings: [{ id: "agentrun-nc01-v02", repositoryRef: "agentrun-nc01-v02" }], + }); + expect(result.next).toMatchObject({ expandWindow: null, windowLimitReached: true }); + }); + + test("same fingerprint across consumers marks the diagnosis as public service", () => { + const result = diagnose([ + row({ id: "fail-a", status: "False", at: "2026-07-16T03:00:00Z", commit: failed }), + row({ id: "fail-b", consumer: "hwlab-nc01-v03", status: "False", at: "2026-07-16T02:30:00Z", commit: failed }), + row({ id: "good-a", status: "True", at: "2026-07-16T02:00:00Z", commit: base }), + ], [ + { id: "agentrun-nc01-v02", repositoryRef: "unidesk-nc01" }, + { id: "hwlab-nc01-v03", repositoryRef: "unidesk-nc01" }, + ]); + expect(result.commonFailure).toMatchObject({ matchingRuns: 2, affectedConsumers: ["agentrun-nc01-v02", "hwlab-nc01-v03"] }); + expect(result.scopeAssessment).toMatchObject({ scope: "public-service", confirmed: true }); + expect(result.rollbackCandidate).toMatchObject({ authorizationRequired: true, automatic: false }); + }); + + test("fingerprint normalizes run numbers and commit ids", () => { + const first = pacFailureFingerprint(row({ id: "a", status: "False", at: "2026-07-16T01:00:00Z", commit: failed })); + const secondRow = row({ id: "b", status: "False", at: "2026-07-16T02:00:00Z", commit: failed }); + (secondRow.firstBreak as Record).reason = `getcwd failed for run 999 at commit ${"c".repeat(40)}`; + expect(pacFailureFingerprint(secondRow)).toMatchObject({ id: (first as Record).id }); + }); + + test("failure summaries redact credential-like values", () => { + const failedRow = row({ id: "fail-secret", status: "False", at: "2026-07-16T02:00:00Z", commit: failed }); + (failedRow.firstBreak as Record).reason = "authorization=Bearer exposed token=plain https://user:pass@example.test/path?key=value"; + const result = diagnose([ + failedRow, + row({ id: "good-1", status: "True", at: "2026-07-16T01:00:00Z", commit: base }), + ]); + const serialized = JSON.stringify(result); + expect(serialized).toContain("[REDACTED]"); + expect(serialized).not.toMatch(/exposed|token=plain|user:pass|key=value/u); + }); + + test("default text is bounded and exposes the recovery window", () => { + const output = renderPacRegressionDiagnosis(diagnose([ + row({ id: "fail-1", status: "False", at: "2026-07-16T02:00:00Z", commit: failed }), + row({ id: "good-1", status: "True", at: "2026-07-16T01:00:00Z", commit: base }), + ])); + expect(output).toContain("LAST_KNOWN_GOOD: good-1"); + expect(output).toContain("FIRST_FAILED: fail-1"); + expect(output).toContain("MUTATION: false"); + expect(output.split("\n").length).toBeLessThan(24); + }); + + test("read failures remain distinct from an incident and visible in default text", () => { + const result = diagnose([]); + const output = renderPacRegressionDiagnosis({ + ...result, + state: "read-failed", + incident: null, + observation: { status: "timeout", elapsedMs: 25001 }, + historyErrors: [{ context: "agentrun-ci:pipelinerun", error: "target read timed out" }], + }); + expect(output).toContain("STATE: read-failed INCIDENT: -"); + expect(output).toContain("READ: timeout elapsed-ms=25001 errors=1"); + expect(output).toContain("READ_ERROR: context=agentrun-ci:pipelinerun"); + }); + + test("CLI route is discoverable and rejects detail ids before remote access", () => { + const help = spawnSync("bun", ["scripts/cli.ts", "platform-infra", "pipelines-as-code", "--help", "--json"], { encoding: "utf8" }); + expect(help.status).toBe(0); + expect(help.stdout).toContain("diagnose-regression"); + const invalid = spawnSync("bun", ["scripts/cli.ts", "platform-infra", "pipelines-as-code", "diagnose-regression", "--target", "NC01", "--consumer", "agentrun-nc01-v02", "--id", "fixture", "--json"], { encoding: "utf8" }); + expect(invalid.status).not.toBe(0); + expect(invalid.stdout).toContain("diagnose-regression does not accept --id"); + expect(invalid.stderr).not.toContain("read.progress"); + }); + + test("target collector uses PipelineRun summaries and exposes no mutation action", () => { + const remote = readFileSync(resolve(import.meta.dir, "platform-infra-pipelines-as-code-remote.sh"), "utf8"); + const collector = remote.slice(remote.indexOf("regression_rows() {"), remote.indexOf("task_rows() {")); + expect(collector).toContain("get', 'pipelinerun'"); + expect(collector).toContain("pipelinerun-summary"); + expect(collector).not.toContain("'taskrun'"); + expect(collector).not.toMatch(/kubectl[^\n]+(?:create|apply|patch|delete)|argo[^\n]+sync/iu); + }); +}); diff --git a/scripts/src/platform-infra-pac-regression-diagnosis.ts b/scripts/src/platform-infra-pac-regression-diagnosis.ts new file mode 100644 index 00000000..b7f8fde9 --- /dev/null +++ b/scripts/src/platform-infra-pac-regression-diagnosis.ts @@ -0,0 +1,320 @@ +import { createHash } from "node:crypto"; + +export interface PacRegressionConsumerBinding { + readonly id: string; + readonly repositoryRef: string; +} + +export interface PacRegressionDiagnosisInput { + readonly targetId: string; + readonly consumerId: string; + readonly sourceRepository: string; + readonly limitPerConsumer: number; + readonly rows: readonly Record[]; + readonly consumerBindings: readonly PacRegressionConsumerBinding[]; +} + +type TerminalState = "success" | "failure" | "pending"; + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function records(value: unknown): Record[] { + return Array.isArray(value) ? value.map(record) : []; +} + +function text(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function number(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function safeSummary(value: unknown): string | null { + const source = text(value); + if (source === null) return null; + return source + .replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/giu, "$1[REDACTED]") + .replace(/(token|password|secret)(\s*[:=]\s*)\S+/giu, "$1$2[REDACTED]") + .replace(/https?:\/\/[^\s/]+@/giu, "https://[REDACTED]@") + .replace(/([?&][^=\s]+)=([^&\s]+)/gu, "$1=[REDACTED]") + .replace(/\s+/gu, " ") + .trim() + .slice(0, 480); +} + +function safeFailureRecord(value: unknown): Record | null { + const source = record(value); + if (Object.keys(source).length === 0) return null; + const safeToken = (item: unknown): string | null => { + const candidate = text(item); + return candidate !== null && /^[A-Za-z0-9_.-]{1,100}$/u.test(candidate) ? candidate : null; + }; + return { + type: safeToken(source.type), + code: safeToken(source.code), + phase: safeToken(source.phase) ?? safeToken(source.stage), + exitCode: number(source.exitCode), + reason: safeSummary(source.reason), + message: safeSummary(source.message), + stderrSummary: safeSummary(source.stderrSummary), + valuesPrinted: false, + }; +} + +function terminalState(row: Record): TerminalState { + if (row.deliveryAuthorityEligible !== true) return "pending"; + if (row.status === "True") return "success"; + if (row.status === "False") return "failure"; + if (Object.keys(record(row.firstBreak)).length > 0 || Object.keys(record(row.error)).length > 0) return "failure"; + return "pending"; +} + +function normalizedReason(value: unknown): string | null { + const source = safeSummary(value); + if (source === null) return null; + return source + .toLowerCase() + .replace(/\b[0-9a-f]{40,64}\b/gu, "") + .replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gu, "") + .replace(/\b\d+\b/gu, "") + .replace(/\s+/gu, " ") + .trim() + .slice(0, 240); +} + +export function pacFailureFingerprint(row: Record): Record | null { + if (terminalState(row) !== "failure") return null; + const firstBreak = record(row.firstBreak); + const error = record(row.error); + const failedTask = record(record(row.taskRuns).failed); + const failedStep = record(failedTask.failedStep); + const code = text(firstBreak.code) ?? text(error.code) ?? "pipeline-run-failed"; + const phase = text(firstBreak.phase) ?? text(firstBreak.stage) ?? text(error.phase) ?? text(failedTask.name) ?? "pipeline-run"; + const errorType = text(error.type); + const exitCode = number(error.exitCode) ?? number(failedStep.exitCode); + const reason = normalizedReason(firstBreak.reason) + ?? normalizedReason(error.stderrSummary) + ?? normalizedReason(error.message) + ?? normalizedReason(error.reason) + ?? normalizedReason(failedStep.reason) + ?? normalizedReason(row.reason); + const signature = [code, phase, errorType ?? "-", exitCode ?? "-", reason ?? "-"].join("|"); + return { + id: `sha256:${createHash("sha256").update(signature).digest("hex").slice(0, 16)}`, + code, + phase, + errorType, + exitCode, + reason, + valuesPrinted: false, + }; +} + +function rowSummary(row: Record): Record | null { + if (Object.keys(row).length === 0) return null; + return { + id: row.id ?? row.pipelineRun ?? null, + consumer: row.consumer ?? null, + status: row.status ?? null, + reason: row.reason ?? null, + triggeredAt: row.triggeredAt ?? null, + completionTime: row.completionTime ?? null, + durationSeconds: row.durationSeconds ?? null, + sourceCommit: row.commit ?? null, + branch: row.branch ?? null, + traceId: row.traceId ?? null, + firstBreak: safeFailureRecord(row.firstBreak), + error: safeFailureRecord(row.error), + fingerprint: pacFailureFingerprint(row), + valuesPrinted: false, + }; +} + +function time(value: unknown): number { + const parsed = typeof value === "string" ? Date.parse(value) : Number.NaN; + return Number.isFinite(parsed) ? parsed : 0; +} + +function compareUrl(repository: string, base: string | null, head: string | null): string | null { + if (!/^[^/]+\/[^/]+$/u.test(repository) || base === null || head === null) return null; + if (!/^[0-9a-f]{40}$/iu.test(base) || !/^[0-9a-f]{40}$/iu.test(head) || base === head) return null; + return `https://github.com/${repository}/compare/${base}...${head}`; +} + +export function diagnosePacRegression(input: PacRegressionDiagnosisInput): Record { + const selectedRows = input.rows + .filter((row) => row.consumer === input.consumerId && row.deliveryAuthorityEligible === true) + .sort((left, right) => time(right.triggeredAt) - time(left.triggeredAt)); + const terminalRows = selectedRows.filter((row) => terminalState(row) !== "pending"); + const latestTerminal = terminalRows[0] ?? null; + const activeRegression = latestTerminal !== null && terminalState(latestTerminal) === "failure"; + const failureStreak: Record[] = []; + let lastSuccess: Record | null = null; + if (activeRegression) { + for (const row of terminalRows) { + if (terminalState(row) === "failure") { + failureStreak.push(row); + continue; + } + lastSuccess = row; + break; + } + } else if (latestTerminal !== null && terminalState(latestTerminal) === "success") { + lastSuccess = latestTerminal; + } + const firstFailure = failureStreak.at(-1) ?? null; + const anchorFingerprint = firstFailure === null ? null : pacFailureFingerprint(firstFailure); + const anchorFingerprintId = text(record(anchorFingerprint).id); + const comparisonWindowStart = lastSuccess === null ? 0 : time(lastSuccess.triggeredAt); + const matchingFailures = anchorFingerprintId === null ? [] : input.rows.filter((row) => { + if (row.deliveryAuthorityEligible !== true || terminalState(row) !== "failure") return false; + if (comparisonWindowStart > 0 && time(row.triggeredAt) < comparisonWindowStart) return false; + return text(record(pacFailureFingerprint(row)).id) === anchorFingerprintId; + }); + const affectedConsumers = [...new Set(matchingFailures.map((row) => text(row.consumer)).filter((value): value is string => value !== null))].sort(); + const selectedBinding = input.consumerBindings.find((consumer) => consumer.id === input.consumerId) ?? null; + const repositoryOwners = selectedBinding === null + ? [] + : input.consumerBindings.filter((consumer) => consumer.repositoryRef === selectedBinding.repositoryRef).map((consumer) => consumer.id).sort(); + const scope = affectedConsumers.length > 1 || repositoryOwners.length > 1 + ? "public-service" + : activeRegression && affectedConsumers.length === 1 + ? "consumer-private-candidate" + : "unknown"; + const baseCommit = text(lastSuccess?.commit); + const failedCommit = text(firstFailure?.commit); + const sourceCompareUrl = compareUrl(input.sourceRepository, baseCommit, failedCommit); + const windowComplete = !activeRegression || lastSuccess !== null; + const state = latestTerminal === null + ? "no-terminal-evidence" + : !activeRegression + ? "healthy" + : windowComplete + ? "active-regression" + : "active-regression-baseline-missing"; + const confidence = !activeRegression || firstFailure === null + ? "none" + : sourceCompareUrl !== null && failureStreak.length >= 2 && anchorFingerprintId !== null + ? "high" + : sourceCompareUrl !== null + ? "medium" + : "low"; + const firstFailureId = text(firstFailure?.id) ?? text(firstFailure?.pipelineRun); + return { + state, + incident: activeRegression, + mutation: false, + target: input.targetId, + consumer: input.consumerId, + window: { + complete: windowComplete, + limitPerConsumer: input.limitPerConsumer, + selectedRows: selectedRows.length, + terminalRows: terminalRows.length, + failureStreak: failureStreak.length, + lastKnownGood: rowSummary(lastSuccess ?? {}), + firstFailed: rowSummary(firstFailure ?? {}), + latestTerminal: rowSummary(latestTerminal ?? {}), + valuesPrinted: false, + }, + commonFailure: { + fingerprint: anchorFingerprint, + matchingRuns: matchingFailures.length, + affectedConsumers, + valuesPrinted: false, + }, + sourceRange: { + repository: input.sourceRepository, + baseCommit, + failedCommit, + compareUrl: sourceCompareUrl, + candidateCommit: sourceCompareUrl === null ? null : failedCommit, + valuesPrinted: false, + }, + scopeAssessment: { + scope, + confirmed: scope === "public-service" && affectedConsumers.length > 1, + repositoryRef: selectedBinding?.repositoryRef ?? null, + repositoryOwners, + reason: affectedConsumers.length > 1 + ? "同一失败指纹影响多个 consumer,按公共服务面处理。" + : repositoryOwners.length > 1 + ? "选中 consumer 使用由多个 consumer 共享的 repositoryRef,按公共服务面处理。" + : scope === "consumer-private-candidate" + ? "当前只观察到单 consumer 失败;仍需用 changed-file 证据确认私有配置归属。" + : "现有证据不足以确认配置归属。", + valuesPrinted: false, + }, + rollbackCandidate: { + state: sourceCompareUrl === null ? "insufficient-evidence" : "review-required", + commit: sourceCompareUrl === null ? null : failedCommit, + confidence, + automatic: false, + requiresReview: true, + authorizationRequired: scope !== "consumer-private-candidate", + reason: sourceCompareUrl === null + ? "必须先取得最后成功与首次失败之间的不同 source commit。" + : "只提供精确回退候选;必须审查变更和不可逆状态,并通过单一职责 PR 回退。", + valuesPrinted: false, + }, + next: { + debugFirstFailure: firstFailureId === null + ? null + : `bun scripts/cli.ts platform-infra pipelines-as-code debug-step --target ${input.targetId} --consumer ${input.consumerId} --id ${firstFailureId}`, + expandWindow: windowComplete || input.limitPerConsumer >= 50 + ? null + : `bun scripts/cli.ts platform-infra pipelines-as-code diagnose-regression --target ${input.targetId} --consumer ${input.consumerId} --limit ${Math.min(50, Math.max(input.limitPerConsumer + 10, input.limitPerConsumer * 2))}`, + windowLimitReached: !windowComplete && input.limitPerConsumer >= 50, + valuesPrinted: false, + }, + evidence: { + matchingFailures: matchingFailures.map((row) => rowSummary(row)), + valuesPrinted: false, + }, + valuesPrinted: false, + }; +} + +function value(value: unknown): string { + if (value === null || value === undefined || value === "") return "-"; + return String(value); +} + +export function renderPacRegressionDiagnosis(result: Record): string { + const window = record(result.window); + const lastKnownGood = record(window.lastKnownGood); + const firstFailed = record(window.firstFailed); + const commonFailure = record(result.commonFailure); + const fingerprint = record(commonFailure.fingerprint); + const sourceRange = record(result.sourceRange); + const scope = record(result.scopeAssessment); + const rollback = record(result.rollbackCandidate); + const next = record(result.next); + const observation = record(result.observation); + const historyErrors = records(result.historyErrors); + const firstReadError = historyErrors[0] ?? {}; + return [ + "PLATFORM-INFRA PIPELINES-AS-CODE REGRESSION DIAGNOSIS", + `STATE: ${value(result.state)} INCIDENT: ${value(result.incident)} MUTATION: false`, + `TARGET: ${value(result.target)} CONSUMER: ${value(result.consumer)}`, + `READ: ${value(observation.status)} elapsed-ms=${value(observation.elapsedMs)} errors=${historyErrors.length}`, + ...(historyErrors.length === 0 ? [] : [`READ_ERROR: context=${value(firstReadError.context)} error=${value(firstReadError.error ?? firstReadError.stderr)}`]), + `WINDOW: complete=${value(window.complete)} rows=${value(window.selectedRows)} terminal=${value(window.terminalRows)} failure-streak=${value(window.failureStreak)}`, + `LAST_KNOWN_GOOD: ${value(lastKnownGood.id)} commit=${value(lastKnownGood.sourceCommit)}`, + `FIRST_FAILED: ${value(firstFailed.id)} commit=${value(firstFailed.sourceCommit)} trace=${value(firstFailed.traceId)}`, + `FINGERPRINT: ${value(fingerprint.id)} code=${value(fingerprint.code)} phase=${value(fingerprint.phase)} matches=${value(commonFailure.matchingRuns)}`, + `AFFECTED_CONSUMERS: ${Array.isArray(commonFailure.affectedConsumers) ? commonFailure.affectedConsumers.join(",") || "-" : "-"}`, + `SOURCE_RANGE: ${value(sourceRange.baseCommit)}...${value(sourceRange.failedCommit)}`, + `COMPARE: ${value(sourceRange.compareUrl)}`, + `SCOPE: ${value(scope.scope)} confirmed=${value(scope.confirmed)} authorization-required=${value(rollback.authorizationRequired)}`, + `ROLLBACK_CANDIDATE: ${value(rollback.state)} commit=${value(rollback.commit)} confidence=${value(rollback.confidence)} automatic=false`, + `SCOPE_REASON: ${value(scope.reason)}`, + "NEXT:", + ` debug-first-failure: ${value(next.debugFirstFailure)}`, + ` expand-window: ${value(next.expandWindow)}`, + ` window-limit-reached: ${value(next.windowLimitReached)}`, + ].join("\n") + "\n"; +} diff --git a/scripts/src/platform-infra-pipelines-as-code-remote.sh b/scripts/src/platform-infra-pipelines-as-code-remote.sh index 4d8b8c6f..2f0ddc63 100644 --- a/scripts/src/platform-infra-pipelines-as-code-remote.sh +++ b/scripts/src/platform-infra-pipelines-as-code-remote.sh @@ -973,6 +973,117 @@ process.stdout.write(JSON.stringify({ rows, consumers: consumerSummaries, errors NODE } +regression_rows() { + node <<'NODE' +const cp = require('node:child_process'); +const fs = require('node:fs'); +const { classifyPacPipelineRun, pipelineRunMatchesConsumer } = require(process.env.UNIDESK_PAC_EVALUATOR_PATH); +const limit = Math.max(1, Math.min(50, Number.parseInt(process.env.UNIDESK_PAC_HISTORY_LIMIT || '20', 10) || 20)); +const consumers = JSON.parse(Buffer.from(process.env.UNIDESK_PAC_HISTORY_CONSUMERS_B64 || 'W10=', 'base64').toString('utf8')); +const errors = []; +const namespaceItems = new Map(); +function pipelineRuns(namespace) { + if (namespaceItems.has(namespace)) return namespaceItems.get(namespace); + const tmpDir = fs.mkdtempSync('/tmp/unidesk-pac-regression-'); + const outPath = `${tmpDir}/pipelineruns.json`; + const outFd = fs.openSync(outPath, 'w'); + const result = cp.spawnSync('kubectl', ['-n', namespace, 'get', 'pipelinerun', '-o', 'json'], { stdio: ['ignore', outFd, 'pipe'], encoding: 'utf8', timeout: 8000 }); + fs.closeSync(outFd); + let items = []; + if (result.error || result.status !== 0) { + errors.push({ context: `${namespace}:pipelinerun`, status: result.status, error: result.error ? String(result.error.message || result.error).slice(0, 480) : null, stderr: String(result.stderr || '').slice(0, 480) }); + } else { + try { items = JSON.parse(fs.readFileSync(outPath, 'utf8') || '{"items":[]}').items || []; } + catch (error) { errors.push({ context: `${namespace}:pipelinerun-parse`, status: null, error: String(error && error.message ? error.message : error).slice(0, 480), stderr: null }); } + } + try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {} + namespaceItems.set(namespace, items); + return items; +} +function firstString(...values) { + for (const value of values) if (typeof value === 'string' && value.length > 0) return value; + return null; +} +function params(items) { + const out = {}; + for (const item of items || []) if (item && typeof item.name === 'string') out[item.name] = typeof item.value === 'string' ? item.value : item.value == null ? null : JSON.stringify(item.value); + return out; +} +function condition(item) { + const value = (item.status?.conditions || []).find((candidate) => candidate.type === 'Succeeded') || {}; + return { status: value.status || null, reason: value.reason || null, message: value.message || null }; +} +function safeSummary(value) { + return typeof value !== 'string' ? null : value + .replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/giu, '$1[REDACTED]') + .replace(/(token|password|secret)(\s*[:=]\s*)\S+/giu, '$1$2[REDACTED]') + .replace(/https?:\/\/[^\s/]+@/giu, 'https://[REDACTED]@') + .replace(/([?&][^=\s]+)=([^&\s]+)/gu, '$1=[REDACTED]') + .replace(/\s+/gu, ' ').trim().slice(0, 480); +} +function commit(item, values) { + const labels = item.metadata?.labels || {}; + const annotations = item.metadata?.annotations || {}; + return firstString(labels['pipelinesascode.tekton.dev/sha'], labels['pipelinesascode.tekton.dev/commit'], labels['unidesk.ai/source-commit'], labels['hwlab.pikastech.local/source-commit'], labels['agentrun.pikastech.local/source-commit'], annotations['pipelinesascode.tekton.dev/sha'], annotations['pipelinesascode.tekton.dev/commit'], annotations['unidesk.ai/source-commit'], values.revision, values.source_revision, values.sourceCommit, values.source_commit, values.git_sha, values.sha, values.commit); +} +function branch(item, values) { + const labels = item.metadata?.labels || {}; + const annotations = item.metadata?.annotations || {}; + return firstString(labels['pipelinesascode.tekton.dev/branch'], annotations['pipelinesascode.tekton.dev/branch'], values.source_branch, values.branch, values.git_branch); +} +function durationSeconds(item) { + const start = Date.parse(item.status?.startTime || item.metadata?.creationTimestamp || ''); + const end = Date.parse(item.status?.completionTime || ''); + return Number.isFinite(start) && Number.isFinite(end) ? Math.max(0, Math.round((end - start) / 1000)) : null; +} +const rows = []; +const summaries = []; +for (const consumer of consumers) { + const matches = pipelineRuns(consumer.namespace) + .filter((item) => pipelineRunMatchesConsumer(consumer, item)) + .map((item) => ({ item, classification: classifyPacPipelineRun(consumer, item) })) + .filter(({ classification }) => classification.deliveryAuthorityEligible === true) + .sort((left, right) => Date.parse(right.item.metadata?.creationTimestamp || 0) - Date.parse(left.item.metadata?.creationTimestamp || 0)) + .slice(0, limit); + for (const { item, classification } of matches) { + const c = condition(item); + const values = params(item.spec?.params); + const failed = c.status === 'False'; + rows.push({ + id: item.metadata?.name || null, + pipelineRun: item.metadata?.name || null, + consumer: consumer.id, + repository: consumer.repository, + repo: consumer.repo, + pipeline: consumer.pipeline, + triggeredAt: item.metadata?.creationTimestamp || item.status?.startTime || null, + startTime: item.status?.startTime || null, + completionTime: item.status?.completionTime || null, + durationSeconds: durationSeconds(item), + status: c.status, + reason: c.reason, + commit: commit(item, values), + branch: branch(item, values), + deliveryClass: classification.deliveryClass, + deliveryAuthorityEligible: classification.deliveryAuthorityEligible, + deliveryOwner: classification.deliveryOwner, + executionOwner: classification.executionOwner, + parentRelation: classification.parentRelation, + classification, + traceId: firstString(item.metadata?.annotations?.['unidesk.ai/trace-id']), + firstBreak: failed ? { code: 'tekton-pipelinerun-failed', phase: c.reason || 'pipeline-run', reason: safeSummary(c.message || c.reason || 'PipelineRun failed'), valuesPrinted: false } : null, + error: failed ? { type: 'TektonPipelineRunError', code: c.reason || 'PipelineRunFailed', phase: c.reason || 'pipeline-run', exitCode: null, valuesPrinted: false } : null, + source: 'pipelinerun-summary', + valuesPrinted: false, + }); + } + summaries.push({ id: consumer.id, namespace: consumer.namespace, repository: consumer.repository, pipelineRunPrefix: consumer.pipelineRunPrefix, matched: matches.length, valuesPrinted: false }); +} +rows.sort((left, right) => Date.parse(right.triggeredAt || 0) - Date.parse(left.triggeredAt || 0)); +process.stdout.write(JSON.stringify({ rows, consumers: summaries, errors })); +NODE +} + task_rows() { payload_file=$(mktemp) kubectl -n "$UNIDESK_PAC_TARGET_NAMESPACE" get taskrun -o json >"$payload_file" 2>/dev/null || printf '{"items":[]}' >"$payload_file" @@ -1674,6 +1785,18 @@ history_action() { "$consumers" "$rows" "$errors" "$hooks" } +diagnose_regression_action() { + history=$(regression_rows) + rows=$(printf '%s' "$history" | node -e 'const fs=require("fs"); const h=JSON.parse(fs.readFileSync(0,"utf8")||"{}"); process.stdout.write(JSON.stringify(h.rows||[]));') + consumers=$(printf '%s' "$history" | node -e 'const fs=require("fs"); const h=JSON.parse(fs.readFileSync(0,"utf8")||"{}"); process.stdout.write(JSON.stringify(h.consumers||[]));') + errors=$(printf '%s' "$history" | node -e 'const fs=require("fs"); const h=JSON.parse(fs.readFileSync(0,"utf8")||"{}"); process.stdout.write(JSON.stringify(h.errors||[]));') + printf '{"ok":%s,"consumer":"%s","display":{"timeZone":"%s"},"consumerRows":%s,"rows":%s,"historyErrors":%s,"webhooks":[],"source":"gitea-pac-pipelinerun-summary","historyStore":"none","valuesPrinted":false}\n' \ + "$( [ "$errors" = "[]" ] && echo true || echo false )" \ + "$(json_string "$UNIDESK_PAC_CONSUMER_ID")" \ + "$(json_string "$UNIDESK_PAC_DISPLAY_TIME_ZONE")" \ + "$consumers" "$rows" "$errors" +} + debug_step_action() { fixtures=$(node <<'NODE' const { runPacStatusFixtureChecks } = require(process.env.UNIDESK_PAC_EVALUATOR_PATH); @@ -1814,6 +1937,7 @@ case "$UNIDESK_PAC_ACTION" in apply) apply_action ;; status) status_action ;; history) history_action ;; + diagnose-regression) diagnose_regression_action ;; debug-step) debug_step_action ;; *) printf '{"ok":false,"error":"unsupported-action","valuesPrinted":false}\n'; exit 2 ;; esac diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index d857d117..4a4507db 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -45,6 +45,7 @@ import { observePacStatusDeliveryBudget, type PacDeliveryTimingPolicy, } from "./platform-infra-pac-delivery-timing"; +import { diagnosePacRegression, renderPacRegressionDiagnosis } from "./platform-infra-pac-regression-diagnosis"; const configFile = rootPath("config", "platform-infra", "pipelines-as-code.yaml"); const configLabel = "config/platform-infra/pipelines-as-code.yaml"; @@ -263,6 +264,8 @@ export interface PipelinesAsCodeNodeStatusOptions { interface HistoryOptions extends CommonOptions { limit: number; detailId: string | null; + summaryOnly?: boolean; + diagnosisConsumerIds?: readonly string[]; } export interface PacHistoryConsumerIdentity { @@ -297,7 +300,7 @@ interface SecretMaterial { webhookPath: string; } -type PacReadOnlyAction = "status" | "history" | "debug-step"; +type PacReadOnlyAction = "status" | "history" | "diagnose-regression" | "debug-step"; interface PacReadOnlyCaptureObservation { readonly ok: boolean; @@ -363,6 +366,12 @@ export async function runPlatformInfraPipelinesAsCodeCommand(config: UniDeskConf const result = await history(config, options); return options.raw ? result : options.full ? compactHistoryJson(result, true) : options.json ? compactHistoryJson(result) : renderHistory(result); } + if (action === "diagnose-regression" || action === "regression") { + const options = parseDiagnoseRegressionOptions(args.slice(1)); + const result = await diagnoseRegression(config, options); + if (options.full || options.raw || options.json) return result; + return rendered(result, "platform-infra pipelines-as-code diagnose-regression", renderPacRegressionDiagnosis(result).trimEnd().split("\n")); + } if (action === "delivery-timing" || action === "timing") { const options = parseCommonOptions(args.slice(1)); const result = await deliveryTiming(config, options); @@ -493,12 +502,13 @@ function help(scope: string | null): Record { }; } return { - command: "platform-infra pipelines-as-code plan|status|history|delivery-timing|debug-step|source-artifact", + command: "platform-infra pipelines-as-code plan|status|history|diagnose-regression|delivery-timing|debug-step|source-artifact", configTruth: configLabel, usage: [ "bun scripts/cli.ts platform-infra pipelines-as-code plan --target JD01", "bun scripts/cli.ts platform-infra pipelines-as-code status --target JD01 [--json|--full|--raw]", "bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 [--consumer hwlab-jd01-v03] [--limit 10]", + "bun scripts/cli.ts platform-infra pipelines-as-code diagnose-regression --target NC01 --consumer agentrun-nc01-v02 [--limit 20] [--json|--full]", "bun scripts/cli.ts platform-infra pipelines-as-code delivery-timing --target NC01 --consumer selfmedia-nc01 [--json]", "bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 --id ", "bun scripts/cli.ts platform-infra pipelines-as-code debug-step --target JD01 [--consumer ] [--id ] [--json]", @@ -1675,7 +1685,12 @@ function printCloseoutProgress(target: PacTarget, consumer: PacConsumer, stage: async function history(config: UniDeskConfig, options: HistoryOptions): Promise> { const pac = readPacConfig({ consumerId: options.consumerId }); const target = resolveTarget(pac, options.targetId); - const selection = resolvePacHistoryConsumerSelection(validPacConsumers(pac), target.id, options.consumerId, options.detailId); + const selection = options.diagnosisConsumerIds === undefined + ? resolvePacHistoryConsumerSelection(validPacConsumers(pac), target.id, options.consumerId, options.detailId) + : { + selectedConsumerIds: [...options.diagnosisConsumerIds], + nextConsumerId: options.diagnosisConsumerIds[0] ?? null, + }; const selectedConsumers = selection.selectedConsumerIds.map((id) => resolveConsumer(pac, id)); const firstConsumer = selectedConsumers[0]; if (firstConsumer === undefined) throw new Error("no Pipelines-as-Code consumers are configured"); @@ -1684,7 +1699,8 @@ async function history(config: UniDeskConfig, options: HistoryOptions): Promise< const next = selection.nextConsumerId === null ? pacNodeReadOnlyNext(target.id) : pacReadOnlyNext(target.id, selection.nextConsumerId); - const observed = await capturePacReadOnly(config, pac, "history", target, firstConsumer, remoteScript("history", pac, target, firstRepository, firstConsumer, { ...options, confirm: false, dryRun: true, wait: false }, secrets, "", selectedConsumers), stringValue(record(next).history)); + const remoteAction = options.summaryOnly === true ? "diagnose-regression" : "history"; + const observed = await capturePacReadOnly(config, pac, remoteAction, target, firstConsumer, remoteScript(remoteAction, pac, target, firstRepository, firstConsumer, { ...options, confirm: false, dryRun: true, wait: false }, secrets, "", selectedConsumers), stringValue(record(next).history)); const { result, parsed, observation } = observed; const remote = parsed ?? compactCapture(result, { full: true }); const historyErrors = arrayRecords(record(remote).historyErrors); @@ -1733,6 +1749,53 @@ async function history(config: UniDeskConfig, options: HistoryOptions): Promise< }; } +async function diagnoseRegression(config: UniDeskConfig, options: HistoryOptions): Promise> { + if (options.consumerId === null) throw new Error("pipelines-as-code diagnose-regression requires --consumer "); + const pac = readPacConfig({ consumerId: options.consumerId }); + const target = resolveTarget(pac, options.targetId); + const consumer = resolveConsumer(pac, options.consumerId); + if (consumer.node.toLowerCase() !== target.id.toLowerCase()) { + throw new Error(`Pipelines-as-Code consumer ${consumer.id} belongs to ${consumer.node}, not target ${target.id}`); + } + const authority = resolveCicdDeliveryAuthority({ consumerId: consumer.id, node: consumer.node, lane: consumer.lane }); + if (authority.kind !== "pac-pr-merge") throw new Error(`delivery authority is ${authority.kind}: ${authority.reason}`); + const diagnosisConsumers = validPacConsumers(pac) + .filter((item) => item.node.toLowerCase() === target.id.toLowerCase() && item.repositoryRef === consumer.repositoryRef); + const historyResult = await history(config, { + ...options, + consumerId: null, + detailId: null, + summaryOnly: true, + diagnosisConsumerIds: diagnosisConsumers.map((item) => item.id), + }); + const diagnosis = diagnosePacRegression({ + targetId: target.id, + consumerId: consumer.id, + sourceRepository: authority.consumer.sourceRepository, + limitPerConsumer: options.limit, + rows: arrayRecords(historyResult.rows), + consumerBindings: validPacConsumers(pac) + .filter((item) => item.node.toLowerCase() === target.id.toLowerCase()) + .map((item) => ({ id: item.id, repositoryRef: item.repositoryRef })), + }); + const evidence = record(diagnosis.evidence); + const readOk = historyResult.ok === true; + return { + ok: readOk, + action: "platform-infra-pipelines-as-code-diagnose-regression", + ...diagnosis, + state: readOk ? diagnosis.state : "read-failed", + incident: readOk ? diagnosis.incident : null, + evidence: options.full || options.raw + ? evidence + : { matchingFailureCount: arrayRecords(evidence.matchingFailures).length, valuesPrinted: false }, + observation: historyResult.observation, + historyErrors: historyResult.historyErrors, + warnings: historyResult.warnings, + valuesPrinted: false, + }; +} + export function resolvePacHistoryConsumerSelection( consumers: readonly PacHistoryConsumerIdentity[], targetId: string, @@ -1966,7 +2029,7 @@ export class PacReleaseManifestRenderError extends Error { } } -function remoteScript(action: "apply" | "status" | "history" | "debug-step", pac: PacConfig, target: PacTarget, repository: PacRepository, consumer: PacConsumer, options: ApplyOptions | HistoryOptions, secrets: SecretMaterial, releaseManifest: string, historyConsumers: PacConsumer[] = [consumer]): string { +function remoteScript(action: "apply" | "status" | "history" | "diagnose-regression" | "debug-step", pac: PacConfig, target: PacTarget, repository: PacRepository, consumer: PacConsumer, options: ApplyOptions | HistoryOptions, secrets: SecretMaterial, releaseManifest: string, historyConsumers: PacConsumer[] = [consumer]): string { const webhookUrl = `${pac.gitea.internalBaseUrl.replace(/\/+$/u, "").replace(/gitea-http\.[^.]+\.svc\.cluster\.local:3000/u, `${pac.release.controllerServiceName}.${pac.release.namespace}.svc.cluster.local:${pac.release.controllerServicePort}`)}`; const admissionIdentity = pacAdmissionDesiredIdentity(target.id); const rbacIdentity = pacConsumerRbacDesiredIdentity(target.id); @@ -3268,9 +3331,9 @@ function parseCloseoutOptions(args: string[]): CloseoutOptions { return { ...parseCommonOptions(commonArgs), sourceCommit, wait, timeoutMs }; } -function parseHistoryOptions(args: string[]): HistoryOptions { +function parseHistoryOptions(args: string[], defaultLimit = 5): HistoryOptions { const commonArgs: string[] = []; - let limit = 5; + let limit = defaultLimit; let detailId: string | null = null; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; @@ -3298,6 +3361,12 @@ function parseHistoryOptions(args: string[]): HistoryOptions { return { ...parseCommonOptions(commonArgs), limit, detailId }; } +function parseDiagnoseRegressionOptions(args: string[]): HistoryOptions { + const options = parseHistoryOptions(args, 20); + if (options.detailId !== null) throw new Error("pipelines-as-code diagnose-regression does not accept --id"); + return options; +} + function parseCommonOptions(args: string[]): CommonOptions { let targetId: string | null = null; let consumerId: string | null = null;