diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 0e126584..963de900 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -19,6 +19,7 @@ UniDesk 通过 `platform-infra sub2api` 运维 YAML 选中的 Sub2API target； bun scripts/cli.ts platform-infra sub2api report bun scripts/cli.ts platform-infra sub2api status --target PK01 bun scripts/cli.ts platform-infra sub2api validate --target PK01 +bun scripts/cli.ts platform-infra sub2api rollout --target PK01 --dry-run bun scripts/cli.ts platform-infra sub2api ops diagnosis --target PK01 bun scripts/cli.ts platform-infra sub2api ops channels --target PK01 --window 7d bun scripts/cli.ts platform-infra sub2api plan --target PK01 diff --git a/.agents/skills/unidesk-sub2api/references/operations.md b/.agents/skills/unidesk-sub2api/references/operations.md index 161316c8..e0622c75 100644 --- a/.agents/skills/unidesk-sub2api/references/operations.md +++ b/.agents/skills/unidesk-sub2api/references/operations.md @@ -1,5 +1,14 @@ # UniDesk Sub2API +## 目录 + +- [先看报表](#先看报表) +- [先读边界](#先读边界) +- [部署与状态](#部署与状态) +- [PK01 host-Docker target](#pk01-host-docker-target) +- [D601 Egress Proxy](#d601-egress-proxy) +- [镜像升级](#镜像升级) + UniDesk 通过 `platform-infra sub2api` 运维 YAML 选中的 Sub2API target。当前 active target 以 `config/platform-infra/sub2api.yaml` 为准；PK01 可作为 host-Docker active target 并通过 PK01 Caddy 本地反代提供 `api.pikapython.com`，D518/D601 等 k3s target 仍可按 YAML 声明为 external-active 或 retired，G14 由同一 YAML/CLI 控制为 standby predeploy。日常操作统一使用 UniDesk CLI，不直接写 Kubernetes 资源或手工调用 Sub2API 管理 API。 **固定入口**: `cd /root/unidesk && bun scripts/cli.ts platform-infra sub2api ...` @@ -41,6 +50,9 @@ bun scripts/cli.ts platform-infra sub2api plan bun scripts/cli.ts platform-infra sub2api plan --target G14 bun scripts/cli.ts platform-infra sub2api image-prepull --target PK01 bun scripts/cli.ts platform-infra sub2api image-prepull --target PK01 --confirm +bun scripts/cli.ts platform-infra sub2api rollout --target PK01 --dry-run +bun scripts/cli.ts platform-infra sub2api rollout --target PK01 --confirm +bun scripts/cli.ts platform-infra sub2api smoke --target PK01 bun scripts/cli.ts platform-infra sub2api apply --dry-run bun scripts/cli.ts platform-infra sub2api apply --target G14 --dry-run bun scripts/cli.ts platform-infra sub2api apply --confirm @@ -69,7 +81,13 @@ PK01 没有 k3s control plane。`codex-pool sync --target PK01 --confirm` 和 `c PK01 host-Docker apply 仍必须由 `platform-infra sub2api apply --target PK01 --confirm` 受控执行。若 dry-run 或 apply 输出显示 `docker compose is absent; apply will use raw docker run fallback`，这表示 CLI 选择了 host-Docker fallback，不是裸手工 Docker 操作；只要 YAML image、env、ports、Caddy managed block 和 `status/validate` 最终对齐，可作为受控滚动升级证据。不要改用手工 `docker run`、手工 compose 文件或直接编辑 PK01 Caddyfile。 -正式镜像升级前必须用 `platform-infra sub2api image-prepull --target --confirm` 预拉 YAML 声明的 Sub2API runtime images；不带 `--confirm` 只做 inspect/dry-run。`--confirm` 默认返回 async job，显式 `--wait` 才同步等待。不得用临时 `trans docker pull ...` 作为长期入口。 +- 正式镜像升级优先使用 `platform-infra sub2api rollout`： + - 同一入口组合镜像 presence、plan、apply dry-run、预拉、apply、status、validate 和既有消费配置 smoke； + - `--confirm` 默认返回 async job，显式 `--wait` 只用于 job 内部或同步调试； + - 镜像预拉长连接预算读取 `defaults.rollout.imagePrepullTimeoutSeconds`； + - 禁止依赖默认 60 秒 SSH 上限反复续拉。 +- `image-prepull` 只作为单阶段诊断入口，不作为标准升级主路径。 +- 禁止用临时 `trans docker pull ...` 作为长期入口。 ## D601 Egress Proxy @@ -89,19 +107,43 @@ proxy secret/config 文件只允许放在受控 Secret/state 路径，输出只 ## 镜像升级 -1. 先定位目标 target 的 `id`、`publicBaseUrl` 和当前 `image` block。`config/platform-infra/sub2api.yaml` 同时有全局默认镜像和多个 target-scoped 镜像；只升级单个 target 时必须用 `id: ` 附近的上下文 patch，不能按第一个 `tag:` 命中。 -2. 只修改目标 target 的 `image.repository`、`image.tag` 或 `pullPolicy`；不要顺手改全局默认、PK01 host-Docker target、standby target 或其它共用 `api2.pikapython.com` 历史 target。 -3. 提交前用 `git diff -- config/platform-infra/sub2api.yaml` 和 `rg -n "id: |tag: |publicBaseUrl"` 确认只有目标 target 变化，且要保护的公开入口没有被误改。 -4. 执行 `sub2api plan --target ` 和 `sub2api apply --target --dry-run`，确认策略检查通过。 -5. 提交并推送 YAML source truth 后，执行 `sub2api image-prepull --target --confirm`，按返回 job 或 `--wait` 输出确认 YAML 声明的镜像已存在于目标 runtime。 -6. 镜像准备好后执行 `sub2api apply --target --confirm`，按返回的 `job status` 命令轮询完成。 -7. 执行 `sub2api status --target `，确认运行镜像等于 YAML 声明；滚动过程中短暂 `0/1` 先等待并复查，不要立即改账号池、Caddy 或 Secret。 -8. 执行 `sub2api validate --target ` 和目标 public `/health`；若需要确认未影响其它公开入口，再跑对应 target 的 `validate`。 -9. 镜像升级禁止配置对齐： +- 目标 patch： + - 只修改目标 target 的 `image.repository`、`image.tag` 或 `pullPolicy`； + - 使用 target 附近的上下文 patch； + - 不按第一个 `tag:` 命中，不改全局默认或其他 target。 +- 发布 dry-run： + - 只调用一次 `sub2api rollout --target --dry-run`； + - 默认摘要同时披露当前镜像、目标镜像、presence、plan 和 apply dry-run； + - 摘要完整时不再分别调用 `status`、`plan`、`image-prepull` 和 `apply --dry-run`。 +- PR 身份： + - 提交并推送 YAML source truth，创建 draft PR； + - 创建后立即回读准确 PR； + - 核对 PR number、head branch 和 head SHA 与当前分支完全一致； + - 禁止通过“最新 PR”、列表顺序或相邻编号推断操作对象。 +- 发布确认： + - 只调用一次 `sub2api rollout --target --confirm`； + - 按返回的唯一 `job status` 命令观察终态； + - job 内顺序完成预拉和 apply，并行完成 status、validate 与既有消费配置 smoke； + - 不在外部重复逐阶段轮询。 +- 报告与合并： + - 把 job 终态和分阶段耗时写入同一 PR 的 MDTODO 报告； + - 推送后将准确 PR 标记 ready，再通过 guarded merge 合入； + - 运行面成功但 PR 身份或 source truth 不一致时，先恢复准确分支，不重复 rollout。 +- 精确下钻： + - 只有 rollout 终态缺少某项证据时，才使用输出给出的下钻命令； + - 不固定重复执行 status、validate、public health 或 smoke。 +- 镜像升级禁止配置对齐： - 不得执行 `codex-pool plan`、`codex-pool sync` 或 `codex-pool validate`； - 不得对齐账号、group、统一 key、capacity、load factor、WebSocket、临时不可调度规则、代理绑定或手工账号。 -10. 镜像升级的真实模型请求证据： +- 镜像升级的真实模型请求证据： - 只能复用升级前已经存在的消费配置执行无写入 smoke； + - `rollout` 自动复用匹配目标 public URL 的 `~/.codex/config.toml` 和 `auth.json`； + - 配置缺失或目标不匹配时只报告 skipped； - 不得为完成 smoke 创建、恢复、同步或覆盖任何配置。 +- 正常热缓存升级的操作目标是 5 分钟内完成： + - `rollout` 输出 `prepull`、`apply`、`verify` 和 `total` 分阶段耗时； + - 超时先按失败 stage 下钻； + - 不回退到旧的十余条手工串联流程。 + 不要把镜像版本写进脚本常量、JSON 或 manifest 模板。 diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index 0b308a1d..7e252c2e 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -12,6 +12,10 @@ metadata: defaults: targetId: PK01 + rollout: + imagePrepullTimeoutSeconds: 240 + smokeTimeoutSeconds: 120 + smokeMaxOutputTokens: 16 cleanup: externalDbState: postgresStatefulSetName: sub2api-postgres diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.14_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.14_Task_Report.md new file mode 100644 index 00000000..69f01ef9 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.14_Task_Report.md @@ -0,0 +1,58 @@ +# R2.14 任务报告 + +## 复盘结论 + +用户观察到的总耗时约 44 分钟，真正的 PK01 apply 与健康验收不足 1 分钟。主要时间消耗来自流程和工具，而不是 Sub2API 升级本身。 + +- 前置准备约 18 分钟：主代理重复读取多份 skill/reference、检查 worktree、建立 MDTODO、分别执行 status、plan 和 apply dry-run。单一 tag 变更没有命中最短路径。 +- 镜像预拉与排障约 10 分钟：`image-prepull --confirm` 虽返回本地 async job，但 job 内仍使用默认 60 秒 SSH capture。冷镜像三次被 60 秒中断，输出只显示 failed；直到 Docker 分层缓存续传后第四次才在 7 秒内完成。 +- 实际发布不足 1 分钟：apply job 约 34 秒，status、validate、public health 和 Responses smoke 均在数秒内完成。 +- PR 与收口约 15 分钟：PR #2227 被另一个 CI/CD 任务按错误编号误合并，随后 #2229 回退；主代理必须从最新 master 重建 #2232、重复 review/merge、同步主 worktree并清理两个分支。 + +直接可核对的 GitHub 时间段是首个升级提交 17:53:58 UTC+2 到最终 PR #2232 合并 18:19:42 UTC+2，共 25 分 44 秒；其余约 18 分钟发生在首个提交前的上下文、worktree、MDTODO 和分散预检阶段。 + +## 工具改进 + +- 新增 `platform-infra sub2api rollout`： + - dry-run 并行获取当前镜像、目标镜像、镜像 presence、plan 和 apply dry-run； + - confirm 默认创建 async job，顺序执行预拉与 apply，并行执行 status、validate 和 smoke； + - 内置运行镜像与 YAML desired image 对账； + - 输出 prepull、apply、verify 和 total 分阶段耗时。 +- 修复 `image-prepull` 长任务预算： + - 使用 `defaults.rollout.imagePrepullTimeoutSeconds`； + - 当前 YAML 为 240 秒； + - 不再被默认 60 秒 SSH capture 误杀。 +- 新增只读 `platform-infra sub2api smoke`： + - 复用已存在且 public URL 匹配的 `~/.codex` 消费配置； + - 固定 `stream:false`、`store:false`； + - 只输出 request ID、HTTP、响应状态和耗时，不输出 key 或响应正文。 +- 新增 YAML 参数： + - `imagePrepullTimeoutSeconds: 240`； + - `smokeTimeoutSeconds: 120`； + - `smokeMaxOutputTokens: 16`。 + +## Skill 与流程改进 + +后续版本升级固定为以下交互： + +1. 只修改目标 target tag。 +2. 执行一次 `rollout --dry-run`。 +3. 推送 source truth，创建 draft PR，并按 PR number、head branch、head SHA 精确回读身份。 +4. 执行一次 `rollout --confirm`。 +5. 执行返回的唯一 `job status` 命令观察终态。 +6. 把终态与阶段耗时写入同一 PR，标记 ready 后 guarded merge。 + +禁止再通过“最新 PR”、列表顺序或相邻编号推断 PR；禁止在 rollout 已提供证据时固定重复 status、plan、image-prepull、apply dry-run、validate、health 和 smoke。 + +## 验证 + +- `bun --check`：`rollout.ts`、`options.ts` 通过。 +- `quick_validate.py .agents/skills/unidesk-sub2api`：通过。 +- `sub2api rollout --target PK01 --dry-run`：约 3.3 秒，current/desired 均为 `0.1.156`，四阶段 ok，mutation=false。 +- `sub2api smoke --target PK01`：约 6 秒，HTTP 200，response status completed，未打印 Secret 或正文。 +- `sub2api plan --target PK01`：policy ok。 +- 未执行 `rollout --confirm`，因此本次工具验证未重启或修改 PK01 运行面。 + +## 预期效果 + +热缓存升级预期在 5 分钟内完成，人工工具调用从十余次降为 3 次：dry-run、confirm、job status。冷镜像可在 YAML 声明的 240 秒预拉预算内完成；若仍超时，rollout 会明确失败 stage 和阶段耗时，不再盲目重复整套流程。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md index 265a3d8a..99470396 100644 --- a/docs/MDTODO/sub2api-upstream-reliability.md +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -219,6 +219,9 @@ ### R2.13 [completed] 将 PK01 Sub2API 从 v0.1.155 受控升级到 v0.1.156：只修改 PK01 target 的 owning YAML 镜像版本，完成 status、plan、apply dry-run、镜像预拉、受控 apply、状态、validate、公开健康和既有消费配置最小 Responses smoke 验收；保持账号池 runtime、Codex pool、代理、外部哨兵与其他服务原样，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.13_Task_Report.md)。 +### R2.14 [completed] + +复盘 PK01 Sub2API v0.1.156 升级耗时 44 分钟的根因，并将同类版本升级优化为预期 5 分钟内可完成的通用流程：修复镜像预拉假异步与 60 秒超时、减少重复 status/plan/job/validate 调用、提供单一语义化发布和验收入口、在 skill 固化 PR 身份校验与最短路径；不修改 Sub2API 源码、账号池 runtime 或外部哨兵，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.14_Task_Report.md)。 ## R3 [completed] 增加 Sub2API 客户余额精准批量充值 CLI：支持从原生最近24小时用量提取活跃用户并生成每人增加20美元的补偿计划，也支持显式用户ID或账号 selector；默认 dry-run 输出完整账号、用户ID、当前余额、增额和目标余额，写入必须显式 --confirm 并自动回读对账。本轮只实现并验证 dry-run，禁止实际充值，必须等待用户再次明确授权，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R3_Task_Report.md)。 diff --git a/scripts/src/platform-infra/config.ts b/scripts/src/platform-infra/config.ts index 4c6b98c9..d4c1e1a4 100644 --- a/scripts/src/platform-infra/config.ts +++ b/scripts/src/platform-infra/config.ts @@ -94,6 +94,13 @@ export function parseDefaults(root: Record): Sub2ApiDefaults { const defaults = objectField(root, "defaults", ""); const targetId = stringField(defaults, "targetId", "defaults"); if (!/^[A-Za-z0-9._-]+$/u.test(targetId)) throw new Error(`${configPath}.defaults.targetId must be a simple target id`); + const rollout = objectField(defaults, "rollout", "defaults"); + const imagePrepullTimeoutSeconds = integerField(rollout, "imagePrepullTimeoutSeconds", "defaults.rollout"); + const smokeTimeoutSeconds = integerField(rollout, "smokeTimeoutSeconds", "defaults.rollout"); + const smokeMaxOutputTokens = integerField(rollout, "smokeMaxOutputTokens", "defaults.rollout"); + if (imagePrepullTimeoutSeconds < 1 || imagePrepullTimeoutSeconds > 3600) throw new Error(`${configPath}.defaults.rollout.imagePrepullTimeoutSeconds must be between 1 and 3600`); + if (smokeTimeoutSeconds < 1 || smokeTimeoutSeconds > 600) throw new Error(`${configPath}.defaults.rollout.smokeTimeoutSeconds must be between 1 and 600`); + if (smokeMaxOutputTokens < 1 || smokeMaxOutputTokens > 1024) throw new Error(`${configPath}.defaults.rollout.smokeMaxOutputTokens must be between 1 and 1024`); const cleanup = objectField(defaults, "cleanup", "defaults"); const externalDbState = objectField(cleanup, "externalDbState", "defaults.cleanup"); const redisPersistentState = objectField(cleanup, "redisPersistentState", "defaults.cleanup"); @@ -101,6 +108,11 @@ export function parseDefaults(root: Record): Sub2ApiDefaults { const egressProxy = objectField(cleanup, "egressProxy", "defaults.cleanup"); const parsed: Sub2ApiDefaults = { targetId, + rollout: { + imagePrepullTimeoutSeconds, + smokeTimeoutSeconds, + smokeMaxOutputTokens, + }, cleanup: { externalDbState: { postgresStatefulSetName: stringField(externalDbState, "postgresStatefulSetName", "defaults.cleanup.externalDbState"), diff --git a/scripts/src/platform-infra/entry.ts b/scripts/src/platform-infra/entry.ts index 8497c9cc..3f378ca4 100644 --- a/scripts/src/platform-infra/entry.ts +++ b/scripts/src/platform-infra/entry.ts @@ -85,6 +85,11 @@ export interface Sub2ApiConfig { export interface Sub2ApiDefaults { targetId: string; + rollout: { + imagePrepullTimeoutSeconds: number; + smokeTimeoutSeconds: number; + smokeMaxOutputTokens: number; + }; cleanup: { externalDbState: { postgresStatefulSetName: string; @@ -414,6 +419,8 @@ export function platformInfraHelp(): unknown { output: "json", usage: [ "bun scripts/cli.ts platform-infra sub2api plan [--target G14|D601]", + "bun scripts/cli.ts platform-infra sub2api rollout [--target G14|D601|PK01] [--dry-run|--confirm] [--wait]", + "bun scripts/cli.ts platform-infra sub2api smoke [--target G14|D601|PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api image-prepull [--target G14|D601|PK01] [--dry-run|--confirm] [--wait]", "bun scripts/cli.ts platform-infra sub2api apply [--target G14|D601] --dry-run", "bun scripts/cli.ts platform-infra sub2api apply [--target G14|D601] --confirm", diff --git a/scripts/src/platform-infra/image-prepull.ts b/scripts/src/platform-infra/image-prepull.ts index 4b0013d4..78392324 100644 --- a/scripts/src/platform-infra/image-prepull.ts +++ b/scripts/src/platform-infra/image-prepull.ts @@ -163,7 +163,15 @@ export async function imagePrepull(config: UniDeskConfig, options: ApplyOptions) }, }; } - const result = await capture(config, target.route, ["sh"], imagePrepullScript(images, options.confirm)); + const result = await capture( + config, + target.route, + ["sh"], + imagePrepullScript(images, options.confirm), + options.confirm + ? { runtimeTimeoutMs: sub2api.defaults.rollout.imagePrepullTimeoutSeconds * 1000 } + : {}, + ); const parsed = parseJsonOutput(result.stdout); return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), @@ -171,6 +179,7 @@ export async function imagePrepull(config: UniDeskConfig, options: ApplyOptions) mode: options.confirm ? "confirmed" : "dry-run", target: { id: target.id, route: target.route, namespace: target.namespace }, images, + timeoutSeconds: options.confirm ? sub2api.defaults.rollout.imagePrepullTimeoutSeconds : null, remote: parsed ?? compactCapture(result, { full: true }), next: { apply: `bun scripts/cli.ts platform-infra sub2api apply --target ${target.id} --confirm`, diff --git a/scripts/src/platform-infra/options.ts b/scripts/src/platform-infra/options.ts index 7fe0492b..a845ecd0 100644 --- a/scripts/src/platform-infra/options.ts +++ b/scripts/src/platform-infra/options.ts @@ -19,6 +19,7 @@ import { defaultSub2ApiTargetId, readSub2ApiConfig, validateOptions } from "./co import { configPath, platformInfraHelp, serviceName } from "./entry"; import { imagePrepull } from "./image-prepull"; import { isHostDockerTarget, resolveTarget } from "./manifest"; +import { rollout, rolloutSmoke } from "./rollout"; import { validate } from "./policy"; export function sub2ApiHelpTargetSummary(): Record { @@ -105,6 +106,16 @@ export async function runPlatformInfraCommand(config: UniDeskConfig, args: strin const result = await apply(config, options); return options.full || options.raw ? result : renderSub2ApiApply(result); } + if (action === "rollout") { + const options = parseApplyOptions(args.slice(2)); + const result = await rollout(config, options); + return options.full || options.raw ? result : renderSub2ApiRollout(result); + } + if (action === "smoke") { + const options = parseDisclosureOptions(args.slice(2)); + const result = await rolloutSmoke(options.targetId); + return options.full || options.raw ? result : renderSub2ApiSmoke(result); + } if (action === "image-prepull") { const options = parseApplyOptions(args.slice(2)); const result = await imagePrepull(config, options); @@ -391,6 +402,108 @@ function renderSub2ApiImagePrepull(result: Record): RenderedCli return rendered(result, "platform-infra sub2api image-prepull", lines); } +function rolloutStageRows(stages: Record): string[][] { + return Object.entries(stages).map(([name, value]) => { + const stage = record(value); + if (name === "smoke") { + return [name, stage.ok === true ? "ok" : "failed", stringValue(stage.status), stringValue(stage.requestId, stringValue(stage.reason))]; + } + if (name === "prepull") { + const images = arrayRecords(record(stage.remote).images); + const presence = images.length === 0 + ? "-" + : images.map((item) => `${stringValue(item.image)}=${boolText(item.presentBefore || item.presentAfter)}`).join(","); + return [name, stage.ok === true ? "ok" : "failed", stringValue(stage.mode), presence]; + } + if (name === "image") { + return [name, stage.ok === true ? "ok" : "failed", "runtime-alignment", `${stringValue(stage.desiredImage)} <= ${arrayValues(stage.observedImages).map((item) => stringValue(item)).join(",")}`]; + } + return [name, stage.ok === true ? "ok" : "failed", stringValue(stage.mode, stringValue(stage.action)), stringValue(stage.failedStage)]; + }); +} + +function rolloutCurrentImage(stages: Record): string { + const summary = record(record(stages.current).summary); + const containers = arrayRecords(summary.containers); + const appContainer = containers.find((item) => stringValue(item.Names, stringValue(item.names)) === "sub2api-app"); + if (appContainer !== undefined) return stringValue(appContainer.Image, stringValue(appContainer.image)); + const deployments = arrayRecords(summary.deployments); + const appDeployment = deployments.find((item) => stringValue(item.name) === "sub2api"); + return appDeployment === undefined ? "-" : arrayValues(appDeployment.images).map((item) => stringValue(item)).join(","); +} + +function renderSub2ApiRollout(result: Record): RenderedCliResult { + const target = record(result.target); + const mode = stringValue(result.mode); + const targetId = stringValue(target.id); + const stages = record(result.stages); + const lines = [ + "PLATFORM-INFRA SUB2API ROLLOUT", + ...table(["TARGET", "ROUTE", "MODE", "STATUS", "CURRENT_IMAGE", "DESIRED_IMAGE"], [[ + targetId, + stringValue(target.route), + mode, + result.ok === true ? "ok" : "failed", + rolloutCurrentImage(stages), + stringValue(result.desiredImage), + ]]), + ]; + if (mode === "async-job") { + const job = record(result.job); + const next = record(result.next); + lines.push( + "", + "JOB", + ...table(["ID", "STATUS", "COMMAND"], [[stringValue(job.id), stringValue(job.status, "started"), stringValue(result.statusCommand)]]), + "", + "NEXT", + ` status: ${stringValue(next.status, stringValue(result.statusCommand))}`, + ); + return rendered(result, "platform-infra sub2api rollout", lines); + } + + const timings = record(result.timingsMs); + lines.push( + "", + "STAGES", + ...table(["STAGE", "STATUS", "MODE", "DETAIL"], rolloutStageRows(stages)), + "", + "TIMINGS_MS", + ...table(["PREPULL", "APPLY", "VERIFY", "TOTAL"], [[ + stringValue(timings.prepull), + stringValue(timings.apply), + stringValue(timings.verify), + stringValue(timings.total), + ]]), + "", + "NEXT", + result.ok === true + ? ` source-truth: verify the merged branch still declares ${stringValue(result.desiredImage)}` + : ` inspect: bun scripts/cli.ts platform-infra sub2api rollout --target ${targetId} ${mode === "confirmed" ? "--confirm --wait" : "--dry-run"} --full`, + "Disclosure: account runtime, Codex pool, proxy, and sentinel configuration are not synchronized by rollout.", + ); + return rendered(result, "platform-infra sub2api rollout", lines); +} + +function renderSub2ApiSmoke(result: Record): RenderedCliResult { + const target = record(result.target); + const lines = [ + "PLATFORM-INFRA SUB2API SMOKE", + ...table(["TARGET", "STATUS", "EXECUTED", "HTTP", "RESPONSE_STATUS", "ELAPSED_MS", "REQUEST_ID"], [[ + stringValue(target.id), + stringValue(result.status, result.ok === true ? "ok" : "failed"), + boolText(result.executed), + stringValue(result.httpStatus), + stringValue(result.responseStatus), + stringValue(result.elapsedMs), + stringValue(result.requestId, stringValue(result.reason)), + ]]), + "", + "Disclosure: the existing consumer key and response body are never printed; valuesPrinted=false.", + ]; + return rendered(result, "platform-infra sub2api smoke", lines); +} + function renderApplyRemoteSummary(remote: Record): string[] { const steps = record(remote.steps); const stepRows = Object.entries(steps).map(([name, value]) => { diff --git a/scripts/src/platform-infra/rollout.ts b/scripts/src/platform-infra/rollout.ts new file mode 100644 index 00000000..a66e8c39 --- /dev/null +++ b/scripts/src/platform-infra/rollout.ts @@ -0,0 +1,252 @@ +import { randomUUID } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +import type { UniDeskConfig } from "../config"; +import { startJob } from "../jobs"; + +import { apply, plan, status } from "./actions"; +import { readSub2ApiConfig } from "./config"; +import type { Sub2ApiTargetConfig } from "./entry"; +import { imagePrepull } from "./image-prepull"; +import { imageRef, resolveTarget } from "./manifest"; +import type { ApplyOptions } from "./options"; +import { validate } from "./policy"; + +type JsonRecord = Record; + +function record(value: unknown): JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as JsonRecord : {}; +} + +function normalizedUrl(value: string): string { + return value.replace(/\/+$/u, ""); +} + +function skippedSmoke(reason: string, detail?: string): JsonRecord { + return { + ok: true, + executed: false, + status: "skipped", + reason, + detail: detail ?? null, + valuesPrinted: false, + }; +} + +async function existingConsumerSmoke(target: Sub2ApiTargetConfig): Promise { + if (target.publicExposure === null || !target.publicExposure.enabled) return skippedSmoke("public-exposure-disabled"); + const configFile = join(homedir(), ".codex", "config.toml"); + const authFile = join(homedir(), ".codex", "auth.json"); + if (!existsSync(configFile) || !existsSync(authFile)) return skippedSmoke("existing-consumer-config-missing"); + + let consumer: JsonRecord; + let auth: JsonRecord; + try { + consumer = record(Bun.TOML.parse(readFileSync(configFile, "utf8"))); + auth = record(JSON.parse(readFileSync(authFile, "utf8")) as unknown); + } catch { + return { ok: false, executed: false, status: "failed", reason: "existing-consumer-config-invalid", valuesPrinted: false }; + } + + const providerName = typeof consumer.model_provider === "string" ? consumer.model_provider : null; + const providers = record(consumer.model_providers); + const provider = providerName === null ? {} : record(providers[providerName]); + const baseUrl = typeof provider.base_url === "string" ? provider.base_url : null; + const model = typeof consumer.model === "string" ? consumer.model : null; + const apiKey = typeof auth.OPENAI_API_KEY === "string" ? auth.OPENAI_API_KEY : null; + if (baseUrl === null || model === null || apiKey === null || apiKey.length === 0) { + return { ok: false, executed: false, status: "failed", reason: "existing-consumer-config-incomplete", valuesPrinted: false }; + } + if (normalizedUrl(baseUrl) !== normalizedUrl(target.publicExposure.publicBaseUrl)) { + return skippedSmoke("existing-consumer-target-mismatch", `${normalizedUrl(baseUrl)} != ${normalizedUrl(target.publicExposure.publicBaseUrl)}`); + } + + const sub2api = readSub2ApiConfig(); + const requestId = randomUUID(); + const startedAt = Date.now(); + try { + const response = await fetch(`${normalizedUrl(baseUrl)}/v1/responses`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + "X-Request-ID": requestId, + "OpenAI-Client-Request-ID": requestId, + }, + body: JSON.stringify({ + model, + input: "Reply OK.", + stream: false, + store: false, + max_output_tokens: sub2api.defaults.rollout.smokeMaxOutputTokens, + }), + signal: AbortSignal.timeout(sub2api.defaults.rollout.smokeTimeoutSeconds * 1000), + }); + const body = record(await response.json().catch(() => ({}))); + const responseStatus = typeof body.status === "string" ? body.status : null; + const responseId = typeof body.id === "string" ? body.id : null; + const error = record(body.error); + const errorType = typeof error.type === "string" ? error.type : null; + return { + ok: response.ok && responseStatus === "completed", + executed: true, + status: response.ok && responseStatus === "completed" ? "passed" : "failed", + requestId, + httpStatus: response.status, + responseId, + responseStatus, + errorType, + elapsedMs: Date.now() - startedAt, + valuesPrinted: false, + }; + } catch (error) { + return { + ok: false, + executed: true, + status: "failed", + requestId, + httpStatus: null, + responseId: null, + responseStatus: null, + errorType: error instanceof Error ? error.name : "request-failed", + elapsedMs: Date.now() - startedAt, + valuesPrinted: false, + }; + } +} + +export async function rolloutSmoke(targetId: string): Promise { + const sub2api = readSub2ApiConfig(); + const target = resolveTarget(sub2api, targetId); + return { + action: "platform-infra-sub2api-smoke", + target: { id: target.id, route: target.route, namespace: target.namespace }, + ...await existingConsumerSmoke(target), + }; +} + +function stageOk(value: JsonRecord): boolean { + return value.ok === true; +} + +function runtimeImages(statusResult: JsonRecord): string[] { + const summary = record(statusResult.summary); + const containerImages = Array.isArray(summary.containers) + ? summary.containers.map((item) => record(item)).map((item) => item.Image ?? item.image).filter((item): item is string => typeof item === "string") + : []; + const deploymentImages = Array.isArray(summary.deployments) + ? summary.deployments.map((item) => record(item)).flatMap((item) => Array.isArray(item.images) ? item.images.filter((image): image is string => typeof image === "string") : []) + : []; + return Array.from(new Set([...containerImages, ...deploymentImages])); +} + +export async function rollout(config: UniDeskConfig, options: ApplyOptions): Promise { + const sub2api = readSub2ApiConfig(); + const target = resolveTarget(sub2api, options.targetId); + const desiredImage = imageRef(sub2api, target); + if (options.confirm && !options.wait) { + const job = startJob( + `platform_infra_sub2api_rollout_${target.id.toLowerCase()}`, + ["bun", "scripts/cli.ts", "platform-infra", "sub2api", "rollout", "--target", target.id, "--confirm", "--wait"], + `Roll out and verify ${target.id} Sub2API image from YAML-controlled desired state`, + ); + const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`; + return { + ok: true, + action: "platform-infra-sub2api-rollout", + mode: "async-job", + target: { id: target.id, route: target.route, namespace: target.namespace }, + desiredImage, + job, + statusCommand, + next: { status: statusCommand }, + }; + } + + const startedAt = Date.now(); + if (!options.confirm) { + const desiredPlan = plan({ targetId: target.id }); + const [current, prepull, applyDryRun] = await Promise.all([ + status(config, { targetId: target.id, full: false, raw: false }), + imagePrepull(config, { ...options, dryRun: true, confirm: false, wait: true }), + apply(config, { ...options, dryRun: true, confirm: false, wait: true }), + ]); + return { + ok: stageOk(desiredPlan) && stageOk(current) && stageOk(prepull) && stageOk(applyDryRun), + action: "platform-infra-sub2api-rollout", + mode: "dry-run", + target: { id: target.id, route: target.route, namespace: target.namespace }, + desiredImage, + stages: { plan: desiredPlan, current, prepull, apply: applyDryRun }, + timingsMs: { total: Date.now() - startedAt }, + mutation: false, + }; + } + + const prepullStartedAt = Date.now(); + const prepull = await imagePrepull(config, { ...options, dryRun: false, confirm: true, wait: true }); + const prepullMs = Date.now() - prepullStartedAt; + if (!stageOk(prepull)) { + return { + ok: false, + action: "platform-infra-sub2api-rollout", + mode: "confirmed", + target: { id: target.id, route: target.route, namespace: target.namespace }, + desiredImage, + failedStage: "prepull", + stages: { prepull }, + timingsMs: { prepull: prepullMs, total: Date.now() - startedAt }, + mutation: true, + }; + } + + const applyStartedAt = Date.now(); + const applied = await apply(config, { ...options, dryRun: false, confirm: true, wait: true }); + const applyMs = Date.now() - applyStartedAt; + if (!stageOk(applied)) { + return { + ok: false, + action: "platform-infra-sub2api-rollout", + mode: "confirmed", + target: { id: target.id, route: target.route, namespace: target.namespace }, + desiredImage, + failedStage: "apply", + stages: { prepull, apply: applied }, + timingsMs: { prepull: prepullMs, apply: applyMs, total: Date.now() - startedAt }, + mutation: true, + }; + } + + const verifyStartedAt = Date.now(); + const [current, validation, smoke] = await Promise.all([ + status(config, { targetId: target.id, full: false, raw: false }), + validate(config, { targetId: target.id, full: false, raw: false }), + rolloutSmoke(target.id), + ]); + const verifyMs = Date.now() - verifyStartedAt; + const observedImages = runtimeImages(current); + const imageAlignment = { + ok: observedImages.includes(desiredImage), + desiredImage, + observedImages, + }; + const verificationOk = stageOk(current) && stageOk(validation) && stageOk(smoke) && imageAlignment.ok; + return { + ok: verificationOk, + action: "platform-infra-sub2api-rollout", + mode: "confirmed", + target: { id: target.id, route: target.route, namespace: target.namespace }, + desiredImage, + failedStage: verificationOk ? null : "verify", + stages: { prepull, apply: applied, current, image: imageAlignment, validate: validation, smoke }, + timingsMs: { + prepull: prepullMs, + apply: applyMs, + verify: verifyMs, + total: Date.now() - startedAt, + }, + mutation: true, + }; +}