diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index f85727fd..3064e2cf 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -5,6 +5,10 @@ - 统一消费 key 写入 YAML 声明的 `targets[PK01].hostDocker.envPath`； - PK01 路径不创建 k8s Secret、不部署 sentinel 资源，也不触发 `sub2api apply`、Docker compose、Caddy reload 或容器重启； - `runtime list|get` 查询 pool group 内全部账号或指定账号的实际状态；`get` 同时披露脱敏 `modelMapping` 和完整临时不可调度规则； + - 新增手工账号的模板接入使用最短路径： + - 先执行 `runtime list --recent `，按原生 `created_at` 倒序查看新近账号、完整名称、ID、调度状态和现有模板； + - `created_at` 缺失的账号只计入 `unknownCreatedAt`，不猜测为新账号； + - 选定后仍以精确 `--account ` 依次执行 dry-run 和 confirm，禁止把“最新账号”隐式变成写入 selector； - 当前正式只读入口为 `runtime get --account` 和 `runtime errors --account --full`；前者读取账号配置与调度状态，后者读取原生 Ops 已观测模型证据，不把历史请求模型误报为实时上游能力； - `runtime infrastructure --account` 只读关联账号 proxy 绑定与基础设施状态： - 同时披露账号 proxy 记录和容器代理环境，不能因容器没有 `HTTP_PROXY` 就把账号误判为直连； @@ -89,6 +93,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool sync --target D601 --confirm bun scripts/cli.ts platform-infra sub2api codex-pool validate --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool runtime list --target PK01 +bun scripts/cli.ts platform-infra sub2api codex-pool runtime list --target PK01 --recent 24h bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --target PK01 --account bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --target PK01 --account --json bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --account --full diff --git a/.agents/skills/unidesk-sub2api/references/operations.md b/.agents/skills/unidesk-sub2api/references/operations.md index 202daade..f73938c2 100644 --- a/.agents/skills/unidesk-sub2api/references/operations.md +++ b/.agents/skills/unidesk-sub2api/references/operations.md @@ -127,6 +127,7 @@ proxy secret/config 文件只允许放在受控 Secret/state 路径，输出只 - 发布确认： - 只调用一次 `sub2api rollout --target --confirm`； - 按返回的唯一 `job status` 命令等待终态； + - `job status` 默认投影 `prepull`、`apply`、`verify` 当前阶段、终态、验证子项和阶段耗时；只有原始 stderr/stdout 排障才使用 `--full`； - job 内顺序完成预拉和 apply，并行完成 status、validate 与既有消费配置 smoke； - 不在外部重复逐阶段轮询。 - 运行面验收： diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R12_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R12_Task_Report.md new file mode 100644 index 00000000..13c8f73e --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R12_Task_Report.md @@ -0,0 +1,26 @@ +# R12 任务报告 + +## 结果 + +完成 UniDesk #2361 的两项通用 CLI 改进。 + +- `runtime list --recent ` 使用 Sub2API 原生账号 `created_at` 过滤并按创建时间倒序展示；表格同时展示账号完整身份、创建时间、调度状态、priority、规则数和匹配模板。创建时间缺失的账号不猜测，单列 `unknownCreatedAt`。 +- `--recent` 只允许只读 list；apply/delete 继续要求精确 `--account` 或 `--accounts`，并明确拒绝把 recent 作为写入 selector。 +- Sub2API rollout worker 在 `prepull`、`apply`、`verify` 开始和终态输出结构化进度事件；默认 `job status` 新增 `sub2api-rollout` 投影，展示当前阶段、阶段状态、验证子项和耗时。 +- 对旧 rollout job 没有进度事件的情况，job status 从既有终态表格恢复阶段与耗时，因此历史任务也无需 `--full` 才能完成常规验收。 + +## 调用缩减 + +- 新账号套模板由“全量 list + 人工猜测 + get + dry-run + confirm”缩短为“recent list + 精确 dry-run + confirm”。 +- 版本滚动由“job status + terminal --full”缩短为默认 `job status` 一次读取终态；运行中同一入口持续显示当前阶段。 + +## 验证 + +- `bun scripts/cli.ts check --syntax-only`：11/11 通过。 +- `runtime list --target PK01 --recent 24h`：只读 smoke 通过，返回 2/12 个新近账号；#39 排第一且显示已匹配模板，`unknownCreatedAt=1`，时区转换为 UTC 正确。 +- `runtime apply --account 39 --template codex-upstream-failover --recent 24h`：在本地参数解析阶段拒绝，未访问写入端点。 +- 历史真实 rollout job 默认 status：识别 `sub2api-rollout`，展示 prepull/apply/verify 成功及 74473/19753/2627/96854 ms 耗时，无需 `--full`。 +- 本地 rollout 事件 fixture：6 个事件被识别，三阶段和每阶段 8000 ms 均正常展示；fixture 不调用 Sub2API。 +- 受控 scripts typecheck 的全仓基线仍失败；修改路径 `platform-infra-sub2api-codex` 与 `platform-infra/rollout.ts` 匹配新增诊断均为 0。`jobs.ts` 匹配的 9 条为既有诊断，新增代码没有新增匹配项。 + +未执行 rollout、runtime confirm、账号修改或其他生产运行面变更。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md index e9beec5c..166e7564 100644 --- a/docs/MDTODO/sub2api-upstream-reliability.md +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -319,3 +319,7 @@ ## R11 [completed] 为 PK01 新增的 runtime-manual 上游账号 `https://www.xiaobaishu.org pro 0.02`（#39）精确应用 YAML 通用 `codex-upstream-failover` 临时不可调度模板：先读取 runtime 证明账号 active、schedulable 且尚无规则，再单账号 dry-run 和 confirm，自动回读 9 条规则与状态码一致；不修改 lyon9801、其他账号、账号字段、Sub2API 源码、版本、代理、Secret、分组或外部哨兵，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R11_Task_Report.md)。 + +## R12 [completed] + +完成 [UniDesk #2361](https://github.com/pikasTech/unidesk/issues/2361)：增强 Sub2API `runtime list --recent `，使用原生账号创建时间过滤并倒序展示新近账号，写入仍要求精准 selector；同时让 Sub2API rollout worker 发出阶段进度，使默认 `job status` 直接展示 prepull、apply、verify、终态和耗时，减少人工全表判断与 `--full` 下钻；只做本地 fixture/smoke，不修改生产运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R12_Task_Report.md)。 diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index 5bcd04e3..f07cca8d 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -29,7 +29,7 @@ export interface JobRecord { } export interface JobProgressSummary { - kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "generic"; + kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "sub2api-rollout" | "generic"; stage: string | null; stageStatus: string | null; sourceCommit: string | null; @@ -246,6 +246,7 @@ export function renderJobStatusSummary(job: ReturnType): Ren ), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.slice(0, 6).map((item) => `- ${jobStatusCell(item, 220)}`)].join("\n"), + ...renderJobTimings(progress.timings), "", jobStatusTable( ["TAIL", "BYTES", "TRUNCATED", "PATH"], @@ -328,9 +329,14 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou || job.name === "agentrun_v01_git_mirror_flush" || /^hwlab_nodes_v[0-9]{2}_git-mirror_(sync|flush)$/u.test(job.name); const ciInstallWorkflow = job.name === "ci_install"; + const sub2ApiRolloutWorkflow = /^platform_infra_sub2api_rollout_[a-z0-9_-]+$/u.test(job.name); if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs); const progressTailBytes = Math.max(4096, Math.floor(maxBytes)); const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes); + if (sub2ApiRolloutWorkflow) { + const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes); + return summarizeSub2ApiRolloutJobProgress(job, stdoutTail, stderrTail, nowMs); + } const agentRunYamlLaneProgressObserved = agentRunYamlLaneTriggerWorkflow || hasAgentRunYamlLaneProgressEvents(stderrTail); if (!knownWorkflow && !defaultRuntimeLanePrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneProgressObserved && !gitMirrorWorkflow) return genericJobProgress(job, stderrTail); const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes); @@ -407,6 +413,84 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou }; } +function summarizeSub2ApiRolloutJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs: number): JobProgressSummary { + const events = parseJsonLineEvents(stderrTail, "platform-infra.sub2api.rollout.progress"); + const lastEvent = events.at(-1) ?? {}; + const terminal = parseSub2ApiRolloutTerminal(stdoutTail); + const stage = stringField(lastEvent.stage) ?? terminal.stage; + const stageStatus = stringField(lastEvent.status) ?? terminal.stageStatus; + const lastEventAt = stringField(lastEvent.at); + const elapsedSeconds = jobElapsedSeconds(job, nowMs); + const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs); + const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); + const eventTimings = Object.fromEntries(events.flatMap((event) => { + const eventStage = stringField(event.stage); + const elapsedMs = numberField(event.elapsedMs); + return eventStage === null || elapsedMs === null ? [] : [[eventStage, elapsedMs]]; + })); + const timings = Object.keys(eventTimings).length > 0 ? eventTimings : terminal.timings; + const statuses = new Map(); + for (const event of events) { + const eventStage = stringField(event.stage); + const eventStatus = stringField(event.status); + if (eventStage !== null && eventStatus !== null) statuses.set(eventStage, eventStatus); + } + for (const [name, status] of Object.entries(terminal.statuses)) if (!statuses.has(name)) statuses.set(name, status); + const checks = recordField(lastEvent.checks); + const checkSummary = Object.entries(checks).map(([name, value]) => `${name}=${value === true ? "ok" : "failed"}`).join(","); + const statusSummary = ["prepull", "apply", "verify"].flatMap((name) => statuses.has(name) ? [`${name}=${statuses.get(name)}`] : []); + const warnings = jobProgressWarnings({ + job, + eventsObserved: events.length, + elapsedSeconds, + stage, + stageStatus, + stageElapsedSeconds, + lastEventAgeSeconds, + }); + return { + kind: "sub2api-rollout", + stage, + stageStatus, + sourceCommit: null, + pipelineRun: null, + pipelineCreated: null, + elapsedSeconds, + stageElapsedSeconds, + lastEventAt, + lastEventAgeSeconds, + eventsObserved: events.length, + slow: warnings.length > 0, + warnings, + diagnostics: Object.keys(checks).length === 0 ? null : { checks }, + timings, + summary: [job.status, ...statusSummary, checkSummary].filter(Boolean).join(" "), + nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, + }; +} + +function parseSub2ApiRolloutTerminal(stdoutTail: string): { stage: string | null; stageStatus: string | null; statuses: Record; timings: Record } { + const statuses: Record = {}; + for (const match of stdoutTail.matchAll(/^(prepull|apply|current|image|validate|smoke)\s+(ok|failed)\s+/gmu)) statuses[match[1]!] = match[2]!; + if (["current", "image", "validate", "smoke"].some((name) => statuses[name] !== undefined)) { + statuses.verify = ["current", "image", "validate", "smoke"].every((name) => statuses[name] === "ok") ? "succeeded" : "failed"; + } + const timingMatch = stdoutTail.match(/PREPULL\s+APPLY\s+VERIFY\s+TOTAL\s*\n(\d+|-)\s+(\d+|-)\s+(\d+|-)\s+(\d+|-)/mu); + const timings: Record = {}; + for (const [index, name] of ["prepull", "apply", "verify", "total"].entries()) { + const parsed = Number(timingMatch?.[index + 1]); + if (Number.isFinite(parsed)) timings[name] = parsed; + } + const stage = statuses.verify !== undefined ? "verify" : statuses.apply !== undefined ? "apply" : statuses.prepull !== undefined ? "prepull" : null; + return { stage, stageStatus: stage === null ? null : statuses[stage] ?? null, statuses, timings }; +} + +function renderJobTimings(timings: Record): string[] { + const rows = Object.entries(timings); + if (rows.length === 0) return []; + return ["", jobStatusTable(["TIMING", "ELAPSED_MS"], rows)]; +} + function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary { const action = job.name.endsWith("_flush") ? "flush" : job.name.endsWith("_sync") ? "sync" : "unknown"; const elapsedSeconds = jobElapsedSeconds(job, nowMs); diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-options.ts b/scripts/src/platform-infra-sub2api-codex/runtime-options.ts index 98985006..340ef412 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime-options.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime-options.ts @@ -21,6 +21,7 @@ export interface RuntimeOptions { full: boolean; raw: boolean; json: boolean; + recent: string | null; since: string; tail: number; targetId: string; @@ -29,7 +30,7 @@ export interface RuntimeOptions { export function parseRuntimeOptions(args: string[]): RuntimeOptions { const [actionRaw = "list", ...rest] = args; if (!(["list", "get", "errors", "infrastructure", "apply", "delete"] as string[]).includes(actionRaw)) { - throw new Error("runtime usage: list|get|errors|infrastructure|apply|delete [--account |--accounts |--select-configured-policy --account-type apikey [--exclude-accounts ]] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable|priority|account-name] [--priority <0-1000>|--name ] [--confirm] [--target ] [--json|--full|--raw]"); + throw new Error("runtime usage: list|get|errors|infrastructure|apply|delete [--account |--accounts |--select-configured-policy --account-type apikey [--exclude-accounts ]] [--recent 24h] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable|priority|account-name] [--priority <0-1000>|--name ] [--confirm] [--target ] [--json|--full|--raw]"); } let account: string | null = null; let accounts: string[] = []; @@ -48,6 +49,7 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { let full = false; let raw = false; let json = false; + let recent: string | null = null; let since = "24h"; let tail = 50_000; let targetId = defaultCodexPoolRuntimeTargetId(); @@ -90,6 +92,8 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { else if (arg.startsWith("--name=")) name = arg.slice("--name=".length).trim(); else if (arg === "--target") targetId = readValue("--target"); else if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length).trim(); + else if (arg === "--recent") recent = parseDuration(readValue("--recent"), "--recent"); + else if (arg.startsWith("--recent=")) recent = parseDuration(arg.slice("--recent=".length), "--recent"); else if (arg === "--since") since = readValue("--since"); else if (arg.startsWith("--since=")) since = arg.slice("--since=".length).trim(); else if (arg === "--tail") tail = parseRuntimeTail(readValue("--tail")); @@ -111,6 +115,7 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { if (group !== null && allGroups) throw new Error("use only one of --group or --all-groups"); if ((allGroups || pageToken !== null) && action !== "errors") throw new Error(`runtime ${action} does not accept all-groups pagination options`); if ((group !== null || platform !== null) && action !== "errors" && action !== "infrastructure") throw new Error(`runtime ${action} does not accept group scope options`); + if (recent !== null && action !== "list") throw new Error(`runtime ${action} does not accept --recent`); if (pageToken !== null && !allGroups) throw new Error("--page-token requires --all-groups"); if (account !== null && allGroups) throw new Error("--account cannot be combined with --all-groups; use --group first"); if (selectConfiguredPolicy && !((action === "apply" || action === "delete") && kind === "temp-unschedulable")) { @@ -136,7 +141,13 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); if (["list", "get", "errors", "infrastructure"].includes(action) && confirm) throw new Error(`runtime ${action} does not accept --confirm`); if ([full, raw, json].filter(Boolean).length > 1) throw new Error("use only one of --json, --full, or --raw"); - return { action, account, accounts, selectConfiguredPolicy, accountType, excludeAccounts, group, allGroups, platform, pageToken, template, kind, priority, name, confirm, full, raw, json, since, tail, targetId }; + return { action, account, accounts, selectConfiguredPolicy, accountType, excludeAccounts, group, allGroups, platform, pageToken, template, kind, priority, name, confirm, full, raw, json, recent, since, tail, targetId }; +} + +function parseDuration(value: string, option: string): string { + const normalized = value.trim(); + if (!/^[1-9]\d*[mhd]$/u.test(normalized)) throw new Error(`${option} must use m, h, or d`); + return normalized; } function parseAccountSelectors(value: string, option = "--accounts"): string[] { diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-head.ts b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-head.ts index 0663f7dc..19736a71 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-head.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-head.ts @@ -331,6 +331,8 @@ def account_summary(detail, include_rules=False, token=None): "currentConcurrency": detail.get("current_concurrency"), "loadFactor": detail.get("load_factor"), "priority": detail.get("priority"), + "createdAt": detail.get("created_at"), + "updatedAt": detail.get("updated_at"), "prioritySemantics": {"order": "lower-is-higher", "highest": 0}, "tempUnschedulable": {key: policy[key] for key in ("enabled", "ruleCount", "statusCodes")}, "matchingTemplate": matching_template(credentials), diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-tail.ts b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-tail.ts index 506e919f..84a774ba 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-tail.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-tail.ts @@ -1258,7 +1258,32 @@ def runtime_result(): } if action == "list": details = [account_detail(token, item["id"]) for item in accounts] - return {**base, "ok": True, "operation": "list", "mutation": False, "accountCount": len(details), "accounts": [account_summary(item) for item in sorted(details, key=lambda value: value.get("name") or "")]} + recent = PAYLOAD.get("recent") + recent_filter = None + if recent: + cutoff = parse_native_time(since_start_time(recent)) + known = [(item, parse_native_time(item.get("created_at"))) for item in details] + unknown_count = sum(1 for _, created_at in known if created_at is None) + details = [item for item, created_at in known if created_at is not None and cutoff is not None and created_at >= cutoff] + details = sorted(details, key=lambda value: parse_native_time(value.get("created_at")) or datetime.min.replace(tzinfo=timezone.utc), reverse=True) + recent_filter = { + "window": recent, + "cutoff": cutoff.isoformat().replace("+00:00", "Z") if cutoff is not None else None, + "matchedCount": len(details), + "unknownCreatedAtCount": unknown_count, + } + else: + details = sorted(details, key=lambda value: value.get("name") or "") + return { + **base, + "ok": True, + "operation": "list", + "mutation": False, + "accountCount": len(details), + "totalAccountCount": len(accounts), + "recentFilter": recent_filter, + "accounts": [account_summary(item) for item in details], + } if action == "errors": errors = observed_runtime_errors(token, accounts, group["id"], effective_platform) return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-render.ts b/scripts/src/platform-infra-sub2api-codex/runtime-render.ts index 19ce3e2c..7306d4c1 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime-render.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime-render.ts @@ -527,12 +527,16 @@ function renderRuntimeAccountList(lines: string[], runtime: Record { const temp = runtimeRecord(account.tempUnschedulable) ?? {}; return [ - runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeTimestamp(account.createdAt), runtimeText(account.management), runtimeText(account.status), runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate), ]; @@ -541,6 +545,13 @@ function renderRuntimeAccountList(lines: string[], runtime: Record"); } +function runtimeTimestamp(value: unknown): string { + if (typeof value !== "string" || value.trim().length === 0) return "unknown"; + const parsed = Date.parse(value); + if (!Number.isFinite(parsed)) return value; + return new Date(parsed).toISOString().replace("T", " ").replace(/\.\d{3}Z$/u, "Z"); +} + function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { const account = runtimeRecord(runtime.account) ?? {}; const temp = runtimeRecord(account.tempUnschedulable) ?? {}; diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index d44c1cba..3851c981 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -35,6 +35,7 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P name: options.name, confirm: options.confirm, full: options.full, + recent: options.recent, since: options.since, tail: options.tail, groupName: pool.groupName, @@ -94,6 +95,7 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P priority: options.priority, name: options.name, confirm: options.confirm, + recent: options.recent, since: options.action === "errors" || options.action === "infrastructure" ? options.since : undefined, tail: options.action === "errors" || options.action === "infrastructure" ? options.tail : undefined, }, diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index aa016c7b..05e93efa 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -402,7 +402,7 @@ export function codexPoolHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601", "bun scripts/cli.ts platform-infra sub2api codex-pool sync [--target D601] --confirm [--prune-removed]", "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--target D601] [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--json|--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--recent 24h] [--target PK01] [--json|--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--json|--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--group |--all-groups] [--platform ] [--page-token ] [--account ] [--since 24h] [--tail 50000] [--target PK01] [--json|--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool faults [--level P0|P1|P2] [--group ] [--account ] [--model ] [--stream sync|stream] [--endpoint ] [--request-id ] [--page-token ] [--target PK01] [--json]", diff --git a/scripts/src/platform-infra/rollout.ts b/scripts/src/platform-infra/rollout.ts index a66e8c39..a89c95bf 100644 --- a/scripts/src/platform-infra/rollout.ts +++ b/scripts/src/platform-infra/rollout.ts @@ -142,6 +142,17 @@ function runtimeImages(statusResult: JsonRecord): string[] { return Array.from(new Set([...containerImages, ...deploymentImages])); } +function emitRolloutProgress(targetId: string, stage: "prepull" | "apply" | "verify", status: "started" | "succeeded" | "failed", detail: JsonRecord = {}): void { + process.stderr.write(`${JSON.stringify({ + event: "platform-infra.sub2api.rollout.progress", + at: new Date().toISOString(), + targetId, + stage, + status, + ...detail, + })}\n`); +} + export async function rollout(config: UniDeskConfig, options: ApplyOptions): Promise { const sub2api = readSub2ApiConfig(); const target = resolveTarget(sub2api, options.targetId); @@ -186,8 +197,16 @@ export async function rollout(config: UniDeskConfig, options: ApplyOptions): Pro } const prepullStartedAt = Date.now(); - const prepull = await imagePrepull(config, { ...options, dryRun: false, confirm: true, wait: true }); + emitRolloutProgress(target.id, "prepull", "started"); + let prepull: JsonRecord; + try { + prepull = await imagePrepull(config, { ...options, dryRun: false, confirm: true, wait: true }); + } catch (error) { + emitRolloutProgress(target.id, "prepull", "failed", { elapsedMs: Date.now() - prepullStartedAt, errorType: error instanceof Error ? error.name : "unknown" }); + throw error; + } const prepullMs = Date.now() - prepullStartedAt; + emitRolloutProgress(target.id, "prepull", stageOk(prepull) ? "succeeded" : "failed", { elapsedMs: prepullMs }); if (!stageOk(prepull)) { return { ok: false, @@ -203,8 +222,16 @@ export async function rollout(config: UniDeskConfig, options: ApplyOptions): Pro } const applyStartedAt = Date.now(); - const applied = await apply(config, { ...options, dryRun: false, confirm: true, wait: true }); + emitRolloutProgress(target.id, "apply", "started"); + let applied: JsonRecord; + try { + applied = await apply(config, { ...options, dryRun: false, confirm: true, wait: true }); + } catch (error) { + emitRolloutProgress(target.id, "apply", "failed", { elapsedMs: Date.now() - applyStartedAt, errorType: error instanceof Error ? error.name : "unknown" }); + throw error; + } const applyMs = Date.now() - applyStartedAt; + emitRolloutProgress(target.id, "apply", stageOk(applied) ? "succeeded" : "failed", { elapsedMs: applyMs }); if (!stageOk(applied)) { return { ok: false, @@ -220,11 +247,20 @@ export async function rollout(config: UniDeskConfig, options: ApplyOptions): Pro } const verifyStartedAt = Date.now(); - const [current, validation, smoke] = await Promise.all([ - status(config, { targetId: target.id, full: false, raw: false }), - validate(config, { targetId: target.id, full: false, raw: false }), - rolloutSmoke(target.id), - ]); + emitRolloutProgress(target.id, "verify", "started"); + let current: JsonRecord; + let validation: JsonRecord; + let smoke: JsonRecord; + try { + [current, validation, smoke] = await Promise.all([ + status(config, { targetId: target.id, full: false, raw: false }), + validate(config, { targetId: target.id, full: false, raw: false }), + rolloutSmoke(target.id), + ]); + } catch (error) { + emitRolloutProgress(target.id, "verify", "failed", { elapsedMs: Date.now() - verifyStartedAt, errorType: error instanceof Error ? error.name : "unknown" }); + throw error; + } const verifyMs = Date.now() - verifyStartedAt; const observedImages = runtimeImages(current); const imageAlignment = { @@ -233,6 +269,15 @@ export async function rollout(config: UniDeskConfig, options: ApplyOptions): Pro observedImages, }; const verificationOk = stageOk(current) && stageOk(validation) && stageOk(smoke) && imageAlignment.ok; + emitRolloutProgress(target.id, "verify", verificationOk ? "succeeded" : "failed", { + elapsedMs: verifyMs, + checks: { + status: stageOk(current), + image: imageAlignment.ok, + validate: stageOk(validation), + smoke: stageOk(smoke), + }, + }); return { ok: verificationOk, action: "platform-infra-sub2api-rollout",