import type { RenderedCliResult } from "../output"; import { renderedCliResult, renderTable } from "./render"; import type { RuntimeOptions } from "./runtime-options"; export function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown { if (options.action !== "errors" || options.full || value === null || typeof value !== "object" || Array.isArray(value)) return value; const runtime = value as Record; const allGroups = runtimeRecord(runtime.allGroups); if (allGroups !== null) { const nextPageToken = typeof allGroups.nextPageToken === "string" ? allGroups.nextPageToken : null; const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; return { ok: runtime.ok, operation: runtime.operation, mutation: runtime.mutation, endpoint: runtime.endpoint, allGroups, disclosure: { group: `${baseCommand} --group ${platformFlag}`, nextPage: nextPageToken === null ? null : `${baseCommand} --all-groups${platformFlag} --page-token ${nextPageToken}`, note: "The overview uses stable group-id pagination. Use group drilldown for account root causes and request indexes.", }, valuesPrinted: false, }; } const errors = runtimeRecord(runtime.errors); if (errors === null) return value; const effectiveness = runtimeRecord(errors.policyEffectiveness) ?? {}; const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; const availabilityGroup = runtimeRecord(availability.group) ?? {}; const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; const concurrencyGroup = runtimeRecord(concurrency.group) ?? {}; const sink = runtimeRecord(nativeOps.systemLogSink) ?? {}; const sourceStatus = runtimeRecord(effectiveness.sourceStatus) ?? {}; const drilldown = runtimeRecord(effectiveness.requestDrilldown) ?? {}; const requestIndexes = Object.fromEntries(Object.entries(drilldown).flatMap(([key, item]) => { if (!Array.isArray(item)) return []; const ids = item.filter((entry): entry is string => typeof entry === "string"); if (ids.length === 0) return []; return [[key, { count: ids.length, sampleRequestId: ids[0], moreAvailable: ids.length > 1, }]]; })); const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; const groupFlag = options.group === null ? "" : ` --group ${options.group}`; const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; if (options.json && options.account === null) { return { ok: runtime.ok, operation: runtime.operation, mutation: runtime.mutation, group: runtime.group, endpoint: runtime.endpoint, errors: { window: errors.window, nativeOps: { source: nativeOps.source, overview: nativeOps.overview, accountQuality: compactAccountQuality(nativeOps.accountQuality, false), accountAvailability: { status: availability.status, totalAccounts: availabilityGroup.total_accounts, availableCount: availabilityGroup.available_count, rateLimitCount: availabilityGroup.rate_limit_count, errorCount: availabilityGroup.error_count, unavailableAccounts: Array.isArray(availability.unavailableAccounts) ? availability.unavailableAccounts.map((item) => { const account = runtimeRecord(item) ?? {}; return { accountId: account.accountId, accountName: account.accountName, status: account.status, reason: account.error, }; }) : [], }, concurrency: { status: concurrency.status, currentInUse: concurrencyGroup.current_in_use, maxCapacity: concurrencyGroup.max_capacity, loadPercentage: concurrencyGroup.load_percentage, waitingInQueue: concurrencyGroup.waiting_in_queue, }, }, }, disclosure: { account: `${baseCommand}${groupFlag}${platformFlag} --account `, trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", note: "This group overview keeps every account name and quality summary. Use account drilldown for request indexes and complete failure evidence.", }, valuesPrinted: false, }; } return { ok: runtime.ok, operation: runtime.operation, mutation: runtime.mutation, group: runtime.group, endpoint: runtime.endpoint, errors: { window: errors.window, nativeOps: { source: nativeOps.source, overview: nativeOps.overview, responseHeaderTimeout: options.account === null || !options.json ? nativeOps.responseHeaderTimeout : undefined, requestTiming: options.account === null || !options.json ? compactRequestTiming(nativeOps.requestTiming) : undefined, customerErrors: compactCustomerErrors(nativeOps.customerErrors, options.account !== null || !options.json), accountQuality: compactAccountQuality(nativeOps.accountQuality, options.account !== null || !options.json), accountAvailability: { status: availability.status, totalAccounts: availabilityGroup.total_accounts, availableCount: availabilityGroup.available_count, rateLimitCount: availabilityGroup.rate_limit_count, errorCount: availabilityGroup.error_count, unavailableAccounts: Array.isArray(availability.unavailableAccounts) ? availability.unavailableAccounts.map((item) => { const account = runtimeRecord(item) ?? {}; return { accountId: account.accountId, accountName: account.accountName, status: account.status, reason: account.error, }; }) : [], }, concurrency: { status: concurrency.status, currentInUse: concurrencyGroup.current_in_use, maxCapacity: concurrencyGroup.max_capacity, loadPercentage: concurrencyGroup.load_percentage, waitingInQueue: concurrencyGroup.waiting_in_queue, }, systemLogSink: { status: sink.status, queueDepth: sink.queueDepth, droppedCount: sink.droppedCount, writeFailedCount: sink.writeFailedCount, }, }, rootCauseAnalysis: errors.rootCauseAnalysis, policyEffectiveness: { source: effectiveness.source, scope: effectiveness.scope, attribution: effectiveness.attribution, assessment: effectiveness.assessment, ruleMatches: effectiveness.ruleMatches, failover: effectiveness.failover, forwardFailure: effectiveness.forwardFailure, selectFailedEvents: effectiveness.selectFailedEvents, requestIndexes, sourceStatus: { status: sourceStatus.status, fallbackUsed: sourceStatus.fallbackUsed, fallbackReason: sourceStatus.fallbackReason, }, }, }, disclosure: { complete: `${baseCommand}${groupFlag}${platformFlag}${options.account === null ? "" : " --account "} --full`, account: `${baseCommand}${groupFlag}${platformFlag} --account `, trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", note: "Use requestIndexes for trace drilldown and --full for the complete finite index.", }, valuesPrinted: false, }; } function compactRequestTiming(value: unknown): unknown { const timing = runtimeRecord(value); if (timing === null) return value; return { status: timing.status, reason: timing.reason, source: timing.source, ttft: timing.ttft, duration: timing.duration, observedFloorSeconds: timing.observedFloorSeconds, attribution: timing.attribution, }; } function compactCustomerErrors(value: unknown, detailed: boolean): unknown { const profile = runtimeRecord(value); if (profile === null) return value; return { status: profile.status, reason: profile.reason, source: profile.source, total: profile.total, modelBuckets: runtimeRecords(profile.modelBuckets).slice(0, 5), accountBuckets: detailed ? runtimeRecords(profile.accountBuckets).slice(0, 5) : undefined, statusBuckets: runtimeRecords(profile.statusBuckets).slice(0, 5), streamBuckets: runtimeRecords(profile.streamBuckets).slice(0, 5), messageBuckets: runtimeRecords(profile.messageBuckets).slice(0, detailed ? 10 : 5), phaseBuckets: detailed ? runtimeRecords(profile.phaseBuckets).slice(0, 5) : undefined, endpointBuckets: detailed ? runtimeRecords(profile.endpointBuckets).slice(0, 5) : undefined, pathBuckets: detailed ? runtimeRecords(profile.pathBuckets).slice(0, 5) : undefined, samples: detailed ? runtimeRecords(profile.samples).slice(0, 15) : undefined, }; } function compactAccountQuality(value: unknown, detailed: boolean): unknown { const quality = runtimeRecord(value); if (quality === null) return value; return { source: quality.source, scope: quality.scope, window: quality.window, scoreRule: (() => { const rule = runtimeRecord(quality.scoreRule) ?? {}; return detailed ? { reliabilityWeight: rule.reliabilityWeight, latencyWeight: rule.latencyWeight, availabilityWeight: rule.availabilityWeight, failureDefinition: rule.failureDefinition, missingMetric: rule.missingMetric, } : { reliabilityWeight: rule.reliabilityWeight, latencyWeight: rule.latencyWeight, availabilityWeight: rule.availabilityWeight, }; })(), accounts: runtimeRecords(quality.accounts).map((account) => detailed ? { accountId: account.accountId, accountName: account.accountName, status: account.status, priority: account.priority, priorityOrder: account.priorityOrder, currentlyAvailable: account.currentlyAvailable, score: account.score, grade: account.grade, assessment: account.assessment, confidence: account.confidence, observedAttempts: account.observedAttempts, successRequests: account.successRequests, failureRequests: account.failureRequests, failureRate: account.failureRate, firstTokenSamples: account.firstTokenSamples, firstTokenCoverage: account.firstTokenCoverage, ttftP95Ms: account.ttftP95Ms, customerErrorRequests: account.customerErrorRequests, scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests, excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests, excludedReasonBuckets: account.excludedReasonBuckets, usage: account.usage, failoverRequests: account.failoverRequests, failoverRecovered: account.failoverRecovered, failoverFailed: account.failoverFailed, sameAccountRetryEvents: account.sameAccountRetryEvents, tempUnschedulableEvents: account.tempUnschedulableEvents, forwardFailedRequests: account.forwardFailedRequests, reasons: account.reasons, } : { accountId: account.accountId, accountName: account.accountName, priority: account.priority, priorityOrder: account.priorityOrder, currentlyAvailable: account.currentlyAvailable, score: account.score, grade: account.grade, confidence: account.confidence, observedAttempts: account.observedAttempts, failureRate: account.failureRate, ttftP95Ms: account.ttftP95Ms, customerErrorRequests: account.customerErrorRequests, scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests, excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests, excludedReasonBuckets: account.excludedReasonBuckets, usage: account.usage, failoverRequests: account.failoverRequests, failoverRecovered: account.failoverRecovered, failoverFailed: account.failoverFailed, }), attribution: detailed ? quality.attribution : undefined, }; } function runtimeRecord(value: unknown): Record | null { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } function runtimeRecords(value: unknown): Record[] { return Array.isArray(value) ? value.flatMap((item) => { const record = runtimeRecord(item); return record === null ? [] : [record]; }) : []; } function runtimeText(value: unknown): string { if (value === null || value === undefined || value === "") return "-"; if (typeof value === "boolean") return value ? "true" : "false"; if (Array.isArray(value)) return value.map(runtimeText).join(","); return String(value).replace(/\s+/gu, " ").trim(); } function runtimeShort(value: unknown, max = 42): string { const text = runtimeText(value); return text.length <= max ? text : `${text.slice(0, max - 1)}…`; } function runtimeRate(value: unknown): string { return typeof value === "number" && Number.isFinite(value) ? `${(value * 100).toFixed(2)}%` : "-"; } function runtimeMillions(value: unknown): string { return typeof value === "number" && Number.isFinite(value) ? `${(value / 1_000_000).toFixed(3)}M` : "-"; } function runtimeDecimal(value: unknown, digits = 4): string { return typeof value === "number" && Number.isFinite(value) ? value.toFixed(digits) : "-"; } export function renderRuntimeResult(response: Record, options: RuntimeOptions): RenderedCliResult { const runtime = runtimeRecord(response.runtime) ?? {}; const ok = response.ok === true; const lines: string[] = []; if (runtime.ok !== true && typeof runtime.error === "string") { lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=false`, `ERROR: ${runtime.error}`); lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} --json`); return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n")); } if (options.action === "errors") renderRuntimeErrors(lines, runtime, options); else if (options.action === "infrastructure") renderRuntimeInfrastructure(lines, runtime); else if (options.action === "list") renderRuntimeAccountList(lines, runtime); else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime); else renderRuntimeMutation(lines, runtime, options); lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} --json`); return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n")); } function renderRuntimeErrors(lines: string[], runtime: Record, options: RuntimeOptions): void { const allGroups = runtimeRecord(runtime.allGroups); if (allGroups !== null) { const totals = runtimeRecord(allGroups.pageTotals) ?? {}; const timeout = runtimeRecord(allGroups.responseHeaderTimeout) ?? {}; const timing = runtimeRecord(allGroups.requestTiming) ?? {}; const groups = runtimeRecords(allGroups.groups); lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`); lines.push(renderTable([ ["GROUPS", "RETURNED", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UPSTREAM_ERR", "UP_RATE", "ATTENTION"], [ runtimeText(allGroups.totalGroups), runtimeText(allGroups.returnedGroups), runtimeText(totals.requestCount), runtimeText(totals.errorCount), runtimeRate(totals.errorRate), runtimeText(totals.upstreamErrorCount), runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention), ], ])); lines.push(renderTable([ ["OPENAI_HEADER_TIMEOUT", "SOURCE", "GENERIC_TIMEOUT", "GENERIC_APPLIES_TO_OPENAI", "PAGE_MAX_TTFT_P99_MS", "OBSERVED_FLOOR_SECONDS"], [ timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), `${runtimeText(timeout.genericResponseHeaderTimeoutSeconds)}s`, runtimeText(timeout.genericAppliesToOpenAI), runtimeText(timing.maxTtftP99Ms), runtimeText(timing.observedFloorSeconds), ], ])); if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); lines.push("", "GROUPS"); lines.push(renderTable([ ["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "TTFT_P99_MS", "DUR_P99_MS", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"], ...groups.map((group) => [ runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status), runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount), runtimeRate(group.upstreamErrorRate), runtimeText(group.ttftP99Ms), runtimeText(group.durationP99Ms), runtimeText(group.totalAccounts), runtimeText(group.availableCount), `${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no", ]), ])); const disclosure = runtimeRecord(runtime.disclosure) ?? {}; lines.push("", `Detail: ${runtimeText(disclosure.group)}`); if (typeof disclosure.nextPage === "string") lines.push(`Next: ${disclosure.nextPage}`); return; } const group = runtimeRecord(runtime.group) ?? {}; const errors = runtimeRecord(runtime.errors) ?? {}; const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; const overview = runtimeRecord(nativeOps.overview) ?? {}; const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; const timeout = runtimeRecord(nativeOps.responseHeaderTimeout) ?? {}; const timing = runtimeRecord(nativeOps.requestTiming) ?? {}; const ttft = runtimeRecord(timing.ttft) ?? {}; const duration = runtimeRecord(timing.duration) ?? {}; const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {}; const accountQuality = runtimeRecord(nativeOps.accountQuality) ?? {}; lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); lines.push(renderTable([ ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], [ runtimeShort(group.name, 30), runtimeText(group.id), runtimeText(group.platform), runtimeText(overview.requestCount), runtimeText(overview.errorCount), runtimeRate(overview.errorRate), runtimeRate(overview.upstreamErrorRate), runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore), ], ])); lines.push(renderTable([ ["OPENAI_HEADER_TIMEOUT", "SOURCE", "TTFT_P50_MS", "TTFT_P95_MS", "TTFT_P99_MS", "TTFT_MAX_MS", "DURATION_P99_MS", "DURATION_MAX_MS"], [ timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), runtimeText(ttft.p50_ms), runtimeText(ttft.p95_ms), runtimeText(ttft.p99_ms), runtimeText(ttft.max_ms), runtimeText(duration.p99_ms), runtimeText(duration.max_ms), ], ])); const candidates = runtimeRecords(timing.candidates); if (candidates.length > 0) lines.push("", "TIMEOUT CANDIDATES", renderTable([ ["SECONDS", "OBSERVED_FLOOR", "TTFT_P99_HEADROOM_MS", "TTFT_P99_BELOW", "ALL_DUR>=", "ERROR_DUR>=", "ASSESSMENT"], ...candidates.map((candidate) => [ runtimeText(candidate.seconds), runtimeText(candidate.observedFloor), runtimeText(candidate.ttftP99HeadroomMs), runtimeText(candidate.ttftP99Below), runtimeText(candidate.totalDurationAtLeastCount), runtimeText(candidate.errorDurationAtLeastCount), runtimeText(candidate.assessment), ]), ])); if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); const modelTiming = runtimeRecords(timing.modelTiming); if (modelTiming.length > 0) lines.push("", "MODEL TIMING", renderTable([ ["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"], ...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]), ])); const qualityAccounts = runtimeRecords(accountQuality.accounts); if (qualityAccounts.length > 0) { lines.push("", "ACCOUNT QUALITY", renderTable([ ["ACCOUNT", "ID", "PRIORITY", "SCORE", "GRADE", "QUALITY", "CONF", "ATTEMPTS", "SUCCESS", "FAIL", "UP_CUST", "EXCLUDED", "FAIL_RATE", "TTFT_P95_MS", "TTFT_N", "AVAILABLE", "REASONS"], ...qualityAccounts.map((item) => [ runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.priority), runtimeText(item.score), runtimeText(item.grade), runtimeText(item.assessment), runtimeText(item.confidence), runtimeText(item.observedAttempts), runtimeText(item.successRequests), runtimeText(item.failureRequests), runtimeText(item.scoreableUpstreamErrorRequests), runtimeText(item.excludedNonUpstreamErrorRequests), runtimeRate(item.failureRate), runtimeText(item.ttftP95Ms), runtimeText(item.firstTokenSamples), runtimeText(item.currentlyAvailable), runtimeShort(Array.isArray(item.reasons) ? item.reasons.join(",") : "", 58), ]), ])); lines.push("ACCOUNT USAGE AND COST", renderTable([ ["ACCOUNT", "ID", "REQUESTS", "TOKENS", "API_USD", "COST_RATE_CNY", "UPSTREAM_COST_CNY", "EXCLUDED_REASONS"], ...qualityAccounts.map((item) => { const usage = runtimeRecord(item.usage) ?? {}; const excluded = runtimeRecords(item.excludedReasonBuckets).map((entry) => `${runtimeText(entry.reason)}:${runtimeText(entry.count)}`).join(","); return [ runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(usage.requestCount), runtimeMillions(usage.tokenCount), runtimeDecimal(usage.apiAmountUsd, 4), runtimeDecimal(usage.costRateCnyPerApiUsd, 3), runtimeDecimal(usage.upstreamCostCny, 4), runtimeShort(excluded, 64), ]; }), ])); const controlRows = qualityAccounts.filter((item) => [ item.failoverRequests, item.sameAccountRetryEvents, item.tempUnschedulableEvents, item.forwardFailedRequests, item.customerErrorRequests, ].some((value) => typeof value === "number" && value > 0)); if (controlRows.length > 0) lines.push("ACCOUNT FAILURE CONTROL", renderTable([ ["ACCOUNT", "ID", "FAILOVER", "RECOVERED", "FAILED", "MISSING", "RETRY", "TEMP_UNSCHED", "FORWARD_FAIL", "CUSTOMER_ERR"], ...controlRows.map((item) => [ runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.failoverRequests), runtimeText(item.failoverRecovered), runtimeText(item.failoverFailed), runtimeText(item.failoverOutcomeMissing), runtimeText(item.sameAccountRetryEvents), runtimeText(item.tempUnschedulableEvents), runtimeText(item.forwardFailedRequests), runtimeText(item.customerErrorRequests), ]), ])); const scoreRule = runtimeRecord(accountQuality.scoreRule) ?? {}; lines.push(`SCORE RULE reliability=${runtimeText(scoreRule.reliabilityWeight)} latency=${runtimeText(scoreRule.latencyWeight)} availability=${runtimeText(scoreRule.availabilityWeight)} confidence=separate priority=lower-is-higher highest=0`); if (typeof accountQuality.attribution === "string") lines.push(`NOTE: ${accountQuality.attribution}`); } const errorModels = runtimeRecords(customerErrors.modelBuckets); const errorAccounts = runtimeRecords(customerErrors.accountBuckets); const errorDimensions = [ ...runtimeRecords(customerErrors.statusBuckets).map((item) => ["status", runtimeText(item.value), runtimeText(item.count)]), ...runtimeRecords(customerErrors.streamBuckets).map((item) => ["mode", runtimeText(item.value), runtimeText(item.count)]), ...runtimeRecords(customerErrors.phaseBuckets).map((item) => ["phase", runtimeText(item.value), runtimeText(item.count)]), ...runtimeRecords(customerErrors.endpointBuckets).map((item) => ["endpoint", runtimeText(item.value), runtimeText(item.count)]), ...runtimeRecords(customerErrors.pathBuckets).map((item) => ["path", runtimeText(item.value), runtimeText(item.count)]), ]; if (errorModels.length > 0) lines.push("", `CUSTOMER ERRORS source=${runtimeText(customerErrors.source)} total=${runtimeText(customerErrors.total)}`, renderTable([ ["MODEL", "COUNT"], ...errorModels.map((item) => [runtimeShort(item.value, 40), runtimeText(item.count)]), ])); if (errorAccounts.length > 0) lines.push(renderTable([ ["ACCOUNT_ID", "COUNT"], ...errorAccounts.map((item) => [runtimeText(item.value), runtimeText(item.count)]), ])); const errorMessages = runtimeRecords(customerErrors.messageBuckets); if (errorMessages.length > 0) lines.push("CUSTOMER ERROR MESSAGES", renderTable([ ["MESSAGE", "COUNT"], ...errorMessages.map((item) => [runtimeShort(item.value, 100), runtimeText(item.count)]), ])); if (errorDimensions.length > 0) lines.push(renderTable([["DIMENSION", "VALUE", "COUNT"], ...errorDimensions])); const errorSamples = runtimeRecords(customerErrors.samples); if (errorSamples.length > 0) lines.push("CUSTOMER ERROR INDEX", renderTable([ ["REQUEST_ID", "MODEL", "STATUS", "ACCOUNT", "MODE", "PATH", "MESSAGE"], ...errorSamples.map((item) => [runtimeText(item.requestId), runtimeShort(item.model, 26), runtimeText(item.statusCode), runtimeText(item.accountId), runtimeText(item.mode), runtimeShort(item.path, 34), runtimeShort(item.message, 48)]), ])); lines.push(renderTable([ ["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"], [ runtimeText(availability.totalAccounts), runtimeText(availability.availableCount), runtimeText(availability.rateLimitCount), runtimeText(availability.errorCount), runtimeText(concurrency.currentInUse), runtimeText(concurrency.maxCapacity), runtimeText(concurrency.waitingInQueue), ], ])); const unavailable = runtimeRecords(availability.unavailableAccounts); if (unavailable.length > 0) { lines.push("", "UNAVAILABLE ACCOUNTS", renderTable([ ["ACCOUNT", "ID", "STATUS", "REASON"], ...unavailable.map((account) => [runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.status), runtimeShort(account.reason, 70)]), ])); } const root = runtimeRecord(errors.rootCauseAnalysis) ?? {}; const causes = runtimeRecords(root.causeBuckets); if (causes.length > 0) { lines.push("", `ROOT CAUSES scope=${runtimeText(root.scope)}`, renderTable([ ["CAUSE", "COUNT"], ...causes.map((cause) => [runtimeText(cause.cause), runtimeText(cause.count)]), ])); } const policy = runtimeRecord(errors.policyEffectiveness) ?? {}; const rules = runtimeRecord(policy.ruleMatches) ?? {}; const failover = runtimeRecord(policy.failover) ?? {}; const forward = runtimeRecord(policy.forwardFailure) ?? {}; lines.push("", `POLICY EFFECTIVENESS scope=${runtimeText(policy.scope)}`); lines.push(renderTable([ ["TEMP_UNSCHED", "FAILOVER", "SUCCESS", "FAILED", "MISSING", "FORWARD_200", "FORWARD_ERR", "SELECT_FAIL"], [ runtimeText(rules.tempUnschedulableEvents), runtimeText(failover.requests), runtimeText(failover.succeeded), runtimeText(failover.failed), runtimeText(failover.outcomeMissing), runtimeText(forward.withHttpSuccess), runtimeText(forward.withHttpError), runtimeText(policy.selectFailedEvents), ], ])); const indexes = runtimeRecord(policy.requestIndexes) ?? {}; const indexRows = Object.entries(indexes).flatMap(([kind, value]) => { const item = runtimeRecord(value); return item === null ? [] : [[kind, runtimeText(item.count), runtimeText(item.sampleRequestId), item.moreAvailable === true ? "yes" : "no"]]; }); if (indexRows.length > 0) lines.push("", "REQUEST INDEX", renderTable([["TYPE", "COUNT", "SAMPLE_REQUEST_ID", "MORE"], ...indexRows])); if (typeof root.attribution === "string") lines.push("", `NOTE: ${root.attribution}`); if (typeof policy.attribution === "string") lines.push(`NOTE: ${policy.attribution}`); const disclosure = runtimeRecord(runtime.disclosure) ?? {}; lines.push("", `Account: ${runtimeText(disclosure.account)}`, `Trace: ${runtimeText(disclosure.trace)}`, `Full: ${runtimeText(disclosure.complete)}`); } function renderRuntimeAccountList(lines: string[], runtime: Record): void { const group = runtimeRecord(runtime.group) ?? {}; const accounts = runtimeRecords(runtime.accounts); lines.push(`SUB2API RUNTIME ACCOUNTS ok=${runtime.ok === true ? "true" : "false"} group=${runtimeText(group.name)} count=${accounts.length}`); lines.push(renderTable([ ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "CONCURRENCY", "LOAD", "PRIORITY", "TEMP_RULES", "TEMPLATE"], ...accounts.map((account) => { const temp = runtimeRecord(account.tempUnschedulable) ?? {}; return [ runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate), ]; }), ])); lines.push("", "Detail: bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account "); } function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { const account = runtimeRecord(runtime.account) ?? {}; const temp = runtimeRecord(account.tempUnschedulable) ?? {}; const priority = runtimeRecord(account.prioritySemantics) ?? {}; const limits = runtimeRecord(account.upstreamLimits) ?? {}; const probe = runtimeRecord(limits.nativeProbe) ?? {}; const globalSettings = runtimeRecord(limits.globalSettings) ?? {}; const windows = runtimeRecords(limits.windows); const protections = runtimeRecords(limits.protections); lines.push(`SUB2API RUNTIME ACCOUNT ok=${runtime.ok === true ? "true" : "false"}`); lines.push(renderTable([ ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "PROXY", "CONCURRENCY", "LOAD", "PRIORITY", "TEMPLATE"], [ runtimeShort(account.accountName, 42), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), runtimeText(account.schedulable), runtimeText(account.proxyId), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(account.matchingTemplate), ], ])); lines.push(renderTable([ ["PRIORITY_ORDER", "HIGHEST_PRIORITY"], [runtimeText(priority.order), runtimeText(priority.highest)], ])); lines.push(renderTable([ ["TEMP_ENABLED", "RULES", "STATUS_CODES", "POOL_MODE"], [runtimeText(temp.enabled), runtimeText(temp.ruleCount), runtimeText(temp.statusCodes), runtimeText(account.poolModeEnabled)], ])); lines.push("", "UPSTREAM LIMITS"); lines.push(renderTable([ ["WINDOW", "USED", "RESET_AFTER", "RESET_AT", "DATA_SOURCE", "PROTECT_AT", "PROTECT_SOURCE", "TRIGGERED"], ...windows.map((window) => { const protection = protections.find((item) => item.window === window.window) ?? {}; return [ runtimeText(window.window), runtimePercent(window.usedPercent), runtimeSeconds(window.resetAfterSeconds), runtimeText(window.resetAt), runtimeText(window.source), runtimePercent(protection.thresholdPercent), runtimeText(protection.source), runtimeText(protection.triggered), ]; }), ])); lines.push(`NATIVE_QUOTA status=${runtimeText(probe.status)} allowed=${runtimeText(probe.allowed)} limitReached=${runtimeText(probe.limitReached)} fetchedAt=${runtimeText(probe.fetchedAt)}`); if (typeof probe.reason === "string") lines.push(`NATIVE_QUOTA_REASON: ${probe.reason}`); lines.push(`GLOBAL_AUTO_PAUSE status=${runtimeText(globalSettings.status)}`); if (typeof globalSettings.reason === "string") lines.push(`GLOBAL_AUTO_PAUSE_REASON: ${globalSettings.reason}`); if (typeof limits.attribution === "string") lines.push(`NOTE: ${limits.attribution}`); } function runtimePercent(value: unknown): string { return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : "-"; } function runtimeSeconds(value: unknown): string { return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)}s` : "-"; } function renderRuntimeInfrastructure(lines: string[], runtime: Record): void { const account = runtimeRecord(runtime.account) ?? {}; const infra = runtimeRecord(runtime.infrastructure) ?? {}; const proxy = runtimeRecord(infra.accountProxy) ?? {}; const app = runtimeRecord(infra.appRuntime) ?? {}; const route = runtimeRecord(infra.defaultRoute) ?? {}; const dns = runtimeRecord(infra.dns) ?? {}; const sockets = runtimeRecord(infra.proxySockets) ?? {}; const listener = runtimeRecord(infra.proxyEndpointOwner) ?? {}; const serviceLogs = runtimeRecord(infra.proxyServiceEvidence) ?? {}; const logs = runtimeRecord(infra.recentNetworkEvidence) ?? {}; lines.push(`SUB2API RUNTIME INFRASTRUCTURE ok=${runtime.ok === true ? "true" : "false"} since=${runtimeText(logs.window)}`); if (runtime.ok !== true && typeof runtime.error === "string") lines.push(`ERROR: ${runtime.error}`); lines.push(renderTable([ ["ACCOUNT", "ID", "TYPE", "STATUS", "SCHED", "PROXY_ID", "PROXY_NAME", "PROTOCOL", "PROXY_ENDPOINT", "PROXY_STATUS"], [ runtimeText(account.accountName), runtimeText(account.accountId), runtimeText(account.type), runtimeText(account.status), runtimeText(account.schedulable), runtimeText(proxy.id), runtimeText(proxy.name), runtimeText(proxy.protocol), `${runtimeText(proxy.host)}:${runtimeText(proxy.port)}`, runtimeText(proxy.status), ], ])); lines.push("", "APP RUNTIME", renderTable([ ["MODE", "CONTAINER", "STATE", "RESTARTS", "OOM", "NETWORK_MODE", "STARTED_AT"], [ runtimeText(app.runtimeMode), runtimeText(app.name), runtimeText(app.state), runtimeText(app.restartCount), runtimeText(app.oomKilled), runtimeText(app.networkMode), runtimeText(app.startedAt), ], ])); const envRows = runtimeRecords(infra.proxyEnvironment); if (envRows.length > 0) lines.push("PROXY ENVIRONMENT", renderTable([ ["VARIABLE", "PRESENT", "SCHEME", "HOST", "PORT", "CREDENTIALS_REDACTED"], ...envRows.map((item) => [runtimeText(item.name), runtimeText(item.present), runtimeText(item.scheme), runtimeText(item.host), runtimeText(item.port), runtimeText(item.credentialsRedacted)]), ])); lines.push("NETWORK", renderTable([ ["DEFAULT_IFACE", "DEFAULT_GATEWAY", "DNS_NAMESERVERS", "DNS_SEARCH", "PROXY_RESOLVED_IPS", "SOCKETS", "ESTABLISHED"], [ runtimeText(route.interface), runtimeText(route.gateway), runtimeText(dns.nameservers), runtimeText(dns.search), runtimeText(sockets.resolvedIps), runtimeText(sockets.total), runtimeText(sockets.established), ], ])); const listeners = runtimeRecords(listener.listeners); if (listeners.length > 0) lines.push("PROXY ENDPOINT OWNER", renderTable([ ["LISTEN", "PROCESS", "PID", "UNIT", "ACTIVE", "RESTARTS", "TCP_PEERS", "UDP_PEERS"], ...listeners.map((item) => [runtimeText(item.localEndpoint), runtimeText(item.processName), runtimeText(item.pid), runtimeText(item.systemdUnit), runtimeText(item.activeState), runtimeText(item.serviceRestarts), runtimeText(item.establishedPeers), runtimeText(item.udpPeers)]), ])); const serviceBuckets = runtimeRecords(serviceLogs.buckets); if (serviceBuckets.length > 0) lines.push("PROXY SERVICE EVIDENCE", renderTable([ ["CAUSE", "COUNT"], ...serviceBuckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), ])); const serviceSamples = runtimeRecords(serviceLogs.samples); if (serviceSamples.length > 0) lines.push("PROXY SERVICE INDEX", renderTable([ ["AT", "CAUSE", "MESSAGE"], ...serviceSamples.map((item) => [runtimeText(item.at), runtimeText(item.cause), runtimeShort(item.message, 100)]), ])); const interfaces = runtimeRecords(infra.interfaces); if (interfaces.length > 0) lines.push("INTERFACES", renderTable([ ["NAME", "RX_BYTES", "TX_BYTES", "RX_ERRORS", "TX_ERRORS", "RX_DROPPED", "TX_DROPPED"], ...interfaces.map((item) => [runtimeText(item.name), runtimeText(item.rxBytes), runtimeText(item.txBytes), runtimeText(item.rxErrors), runtimeText(item.txErrors), runtimeText(item.rxDropped), runtimeText(item.txDropped)]), ])); const buckets = runtimeRecords(logs.buckets); lines.push("", `RECENT NETWORK EVIDENCE matched=${runtimeText(logs.matchedEvents)} source=${runtimeText(logs.source)}`); if (buckets.length > 0) lines.push(renderTable([ ["CAUSE", "COUNT"], ...buckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), ])); const samples = runtimeRecords(logs.samples); if (samples.length > 0) lines.push("EVIDENCE INDEX", renderTable([ ["AT", "REQUEST_ID", "CAUSE", "STATUS", "MESSAGE"], ...samples.map((item) => [runtimeText(item.at), runtimeText(item.requestId), runtimeText(item.cause), runtimeText(item.statusCode), runtimeShort(item.message, 90)]), ])); if (typeof infra.attribution === "string") lines.push(`NOTE: ${infra.attribution}`); } function renderRuntimeMutation(lines: string[], runtime: Record, options: RuntimeOptions): void { const plans = runtimeRecords(runtime.plans); if (plans.length > 0) { const summary = runtimeRecord(runtime.summary) ?? {}; lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); lines.push(renderTable([ ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES", "RESULT"], ...plans.map((plan) => { const account = runtimeRecord(plan.account) ?? {}; const before = runtimeRecord(plan.before) ?? {}; const after = runtimeRecord(plan.after) ?? {}; return [ runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), runtimeText(plan.result), ]; }), ])); lines.push(`SUMMARY selected=${runtimeText(summary.selected)} changed=${runtimeText(summary.changed)} succeeded=${runtimeText(summary.succeeded)} failed=${runtimeText(summary.failed)}`); return; } const plan = runtimeRecord(runtime.plan) ?? {}; const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {}; const before = runtimeRecord(plan.before) ?? {}; const after = runtimeRecord(plan.after) ?? {}; lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); lines.push(renderTable([ ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES"], [ runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), ], ])); }