Codex
704 lines
40 KiB
TypeScript
704 lines
40 KiB
TypeScript
import type { RenderedCliResult } from "../output";
|
|
import { renderedCliResult, renderTable } from "./render";
|
|
import type { RuntimeOptions } from "./runtime-options";
|
|
|
|
export function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown {
|
|
if (options.action !== "errors" || options.full || value === null || typeof value !== "object" || Array.isArray(value)) return value;
|
|
const runtime = value as Record<string, unknown>;
|
|
const allGroups = runtimeRecord(runtime.allGroups);
|
|
if (allGroups !== null) {
|
|
const nextPageToken = typeof allGroups.nextPageToken === "string" ? allGroups.nextPageToken : null;
|
|
const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`;
|
|
const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`;
|
|
return {
|
|
ok: runtime.ok,
|
|
operation: runtime.operation,
|
|
mutation: runtime.mutation,
|
|
endpoint: runtime.endpoint,
|
|
allGroups,
|
|
disclosure: {
|
|
group: `${baseCommand} --group <group-name-or-id>${platformFlag}`,
|
|
nextPage: nextPageToken === null ? null : `${baseCommand} --all-groups${platformFlag} --page-token ${nextPageToken}`,
|
|
note: "The overview uses stable group-id pagination. Use group drilldown for account root causes and request indexes.",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const errors = runtimeRecord(runtime.errors);
|
|
if (errors === null) return value;
|
|
const effectiveness = runtimeRecord(errors.policyEffectiveness) ?? {};
|
|
const nativeOps = runtimeRecord(errors.nativeOps) ?? {};
|
|
const availability = runtimeRecord(nativeOps.accountAvailability) ?? {};
|
|
const availabilityGroup = runtimeRecord(availability.group) ?? {};
|
|
const concurrency = runtimeRecord(nativeOps.concurrency) ?? {};
|
|
const concurrencyGroup = runtimeRecord(concurrency.group) ?? {};
|
|
const sink = runtimeRecord(nativeOps.systemLogSink) ?? {};
|
|
const sourceStatus = runtimeRecord(effectiveness.sourceStatus) ?? {};
|
|
const drilldown = runtimeRecord(effectiveness.requestDrilldown) ?? {};
|
|
const requestIndexes = Object.fromEntries(Object.entries(drilldown).flatMap(([key, item]) => {
|
|
if (!Array.isArray(item)) return [];
|
|
const ids = item.filter((entry): entry is string => typeof entry === "string");
|
|
if (ids.length === 0) return [];
|
|
return [[key, {
|
|
count: ids.length,
|
|
sampleRequestId: ids[0],
|
|
moreAvailable: ids.length > 1,
|
|
}]];
|
|
}));
|
|
const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`;
|
|
const groupFlag = options.group === null ? "" : ` --group ${options.group}`;
|
|
const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`;
|
|
if (options.json && options.account === null) {
|
|
return {
|
|
ok: runtime.ok,
|
|
operation: runtime.operation,
|
|
mutation: runtime.mutation,
|
|
group: runtime.group,
|
|
endpoint: runtime.endpoint,
|
|
errors: {
|
|
window: errors.window,
|
|
nativeOps: {
|
|
source: nativeOps.source,
|
|
overview: nativeOps.overview,
|
|
accountQuality: compactAccountQuality(nativeOps.accountQuality, false),
|
|
accountAvailability: {
|
|
status: availability.status,
|
|
totalAccounts: availabilityGroup.total_accounts,
|
|
availableCount: availabilityGroup.available_count,
|
|
rateLimitCount: availabilityGroup.rate_limit_count,
|
|
errorCount: availabilityGroup.error_count,
|
|
unavailableAccounts: Array.isArray(availability.unavailableAccounts)
|
|
? availability.unavailableAccounts.map((item) => {
|
|
const account = runtimeRecord(item) ?? {};
|
|
return {
|
|
accountId: account.accountId,
|
|
accountName: account.accountName,
|
|
status: account.status,
|
|
reason: account.error,
|
|
};
|
|
})
|
|
: [],
|
|
},
|
|
concurrency: {
|
|
status: concurrency.status,
|
|
currentInUse: concurrencyGroup.current_in_use,
|
|
maxCapacity: concurrencyGroup.max_capacity,
|
|
loadPercentage: concurrencyGroup.load_percentage,
|
|
waitingInQueue: concurrencyGroup.waiting_in_queue,
|
|
},
|
|
},
|
|
},
|
|
disclosure: {
|
|
account: `${baseCommand}${groupFlag}${platformFlag} --account <name-or-id>`,
|
|
trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target <target-id> --request-id <request-id>",
|
|
note: "This group overview keeps every account name and quality summary. Use account drilldown for request indexes and complete failure evidence.",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
return {
|
|
ok: runtime.ok,
|
|
operation: runtime.operation,
|
|
mutation: runtime.mutation,
|
|
group: runtime.group,
|
|
endpoint: runtime.endpoint,
|
|
errors: {
|
|
window: errors.window,
|
|
nativeOps: {
|
|
source: nativeOps.source,
|
|
overview: nativeOps.overview,
|
|
responseHeaderTimeout: options.account === null || !options.json ? nativeOps.responseHeaderTimeout : undefined,
|
|
requestTiming: options.account === null || !options.json ? compactRequestTiming(nativeOps.requestTiming) : undefined,
|
|
customerErrors: compactCustomerErrors(nativeOps.customerErrors, options.account !== null || !options.json),
|
|
accountQuality: compactAccountQuality(nativeOps.accountQuality, options.account !== null || !options.json),
|
|
accountAvailability: {
|
|
status: availability.status,
|
|
totalAccounts: availabilityGroup.total_accounts,
|
|
availableCount: availabilityGroup.available_count,
|
|
rateLimitCount: availabilityGroup.rate_limit_count,
|
|
errorCount: availabilityGroup.error_count,
|
|
unavailableAccounts: Array.isArray(availability.unavailableAccounts)
|
|
? availability.unavailableAccounts.map((item) => {
|
|
const account = runtimeRecord(item) ?? {};
|
|
return {
|
|
accountId: account.accountId,
|
|
accountName: account.accountName,
|
|
status: account.status,
|
|
reason: account.error,
|
|
};
|
|
})
|
|
: [],
|
|
},
|
|
concurrency: {
|
|
status: concurrency.status,
|
|
currentInUse: concurrencyGroup.current_in_use,
|
|
maxCapacity: concurrencyGroup.max_capacity,
|
|
loadPercentage: concurrencyGroup.load_percentage,
|
|
waitingInQueue: concurrencyGroup.waiting_in_queue,
|
|
},
|
|
systemLogSink: {
|
|
status: sink.status,
|
|
queueDepth: sink.queueDepth,
|
|
droppedCount: sink.droppedCount,
|
|
writeFailedCount: sink.writeFailedCount,
|
|
},
|
|
},
|
|
rootCauseAnalysis: errors.rootCauseAnalysis,
|
|
policyEffectiveness: {
|
|
source: effectiveness.source,
|
|
scope: effectiveness.scope,
|
|
attribution: effectiveness.attribution,
|
|
assessment: effectiveness.assessment,
|
|
ruleMatches: effectiveness.ruleMatches,
|
|
failover: effectiveness.failover,
|
|
forwardFailure: effectiveness.forwardFailure,
|
|
selectFailedEvents: effectiveness.selectFailedEvents,
|
|
requestIndexes,
|
|
sourceStatus: {
|
|
status: sourceStatus.status,
|
|
fallbackUsed: sourceStatus.fallbackUsed,
|
|
fallbackReason: sourceStatus.fallbackReason,
|
|
},
|
|
},
|
|
},
|
|
disclosure: {
|
|
complete: `${baseCommand}${groupFlag}${platformFlag}${options.account === null ? "" : " --account <same-account-selector>"} --full`,
|
|
account: `${baseCommand}${groupFlag}${platformFlag} --account <name-or-id>`,
|
|
trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target <target-id> --request-id <request-id>",
|
|
note: "Use requestIndexes for trace drilldown and --full for the complete finite index.",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
function compactRequestTiming(value: unknown): unknown {
|
|
const timing = runtimeRecord(value);
|
|
if (timing === null) return value;
|
|
return {
|
|
status: timing.status,
|
|
reason: timing.reason,
|
|
source: timing.source,
|
|
ttft: timing.ttft,
|
|
duration: timing.duration,
|
|
observedFloorSeconds: timing.observedFloorSeconds,
|
|
attribution: timing.attribution,
|
|
};
|
|
}
|
|
function compactCustomerErrors(value: unknown, detailed: boolean): unknown {
|
|
const profile = runtimeRecord(value);
|
|
if (profile === null) return value;
|
|
return {
|
|
status: profile.status,
|
|
reason: profile.reason,
|
|
source: profile.source,
|
|
total: profile.total,
|
|
modelBuckets: runtimeRecords(profile.modelBuckets).slice(0, 5),
|
|
accountBuckets: detailed ? runtimeRecords(profile.accountBuckets).slice(0, 5) : undefined,
|
|
statusBuckets: runtimeRecords(profile.statusBuckets).slice(0, 5),
|
|
streamBuckets: runtimeRecords(profile.streamBuckets).slice(0, 5),
|
|
messageBuckets: runtimeRecords(profile.messageBuckets).slice(0, detailed ? 10 : 5),
|
|
phaseBuckets: detailed ? runtimeRecords(profile.phaseBuckets).slice(0, 5) : undefined,
|
|
endpointBuckets: detailed ? runtimeRecords(profile.endpointBuckets).slice(0, 5) : undefined,
|
|
pathBuckets: detailed ? runtimeRecords(profile.pathBuckets).slice(0, 5) : undefined,
|
|
samples: detailed ? runtimeRecords(profile.samples).slice(0, 15) : undefined,
|
|
};
|
|
}
|
|
function compactAccountQuality(value: unknown, detailed: boolean): unknown {
|
|
const quality = runtimeRecord(value);
|
|
if (quality === null) return value;
|
|
return {
|
|
source: quality.source,
|
|
scope: quality.scope,
|
|
window: quality.window,
|
|
scoreRule: (() => {
|
|
const rule = runtimeRecord(quality.scoreRule) ?? {};
|
|
return detailed ? {
|
|
reliabilityWeight: rule.reliabilityWeight,
|
|
latencyWeight: rule.latencyWeight,
|
|
availabilityWeight: rule.availabilityWeight,
|
|
failureDefinition: rule.failureDefinition,
|
|
missingMetric: rule.missingMetric,
|
|
} : {
|
|
reliabilityWeight: rule.reliabilityWeight,
|
|
latencyWeight: rule.latencyWeight,
|
|
availabilityWeight: rule.availabilityWeight,
|
|
};
|
|
})(),
|
|
accounts: runtimeRecords(quality.accounts).map((account) => detailed ? {
|
|
accountId: account.accountId,
|
|
accountName: account.accountName,
|
|
status: account.status,
|
|
priority: account.priority,
|
|
priorityOrder: account.priorityOrder,
|
|
currentlyAvailable: account.currentlyAvailable,
|
|
score: account.score,
|
|
grade: account.grade,
|
|
assessment: account.assessment,
|
|
confidence: account.confidence,
|
|
observedAttempts: account.observedAttempts,
|
|
successRequests: account.successRequests,
|
|
failureRequests: account.failureRequests,
|
|
failureRate: account.failureRate,
|
|
firstTokenSamples: account.firstTokenSamples,
|
|
firstTokenCoverage: account.firstTokenCoverage,
|
|
ttftP95Ms: account.ttftP95Ms,
|
|
customerErrorRequests: account.customerErrorRequests,
|
|
scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests,
|
|
excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests,
|
|
excludedReasonBuckets: account.excludedReasonBuckets,
|
|
usage: account.usage,
|
|
failoverRequests: account.failoverRequests,
|
|
failoverRecovered: account.failoverRecovered,
|
|
failoverFailed: account.failoverFailed,
|
|
sameAccountRetryEvents: account.sameAccountRetryEvents,
|
|
tempUnschedulableEvents: account.tempUnschedulableEvents,
|
|
forwardFailedRequests: account.forwardFailedRequests,
|
|
reasons: account.reasons,
|
|
} : {
|
|
accountId: account.accountId,
|
|
accountName: account.accountName,
|
|
priority: account.priority,
|
|
priorityOrder: account.priorityOrder,
|
|
currentlyAvailable: account.currentlyAvailable,
|
|
score: account.score,
|
|
grade: account.grade,
|
|
confidence: account.confidence,
|
|
observedAttempts: account.observedAttempts,
|
|
failureRate: account.failureRate,
|
|
ttftP95Ms: account.ttftP95Ms,
|
|
customerErrorRequests: account.customerErrorRequests,
|
|
scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests,
|
|
excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests,
|
|
excludedReasonBuckets: account.excludedReasonBuckets,
|
|
usage: account.usage,
|
|
failoverRequests: account.failoverRequests,
|
|
failoverRecovered: account.failoverRecovered,
|
|
failoverFailed: account.failoverFailed,
|
|
}),
|
|
attribution: detailed ? quality.attribution : undefined,
|
|
};
|
|
}
|
|
function runtimeRecord(value: unknown): Record<string, unknown> | null {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
|
}
|
|
|
|
function runtimeRecords(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.flatMap((item) => {
|
|
const record = runtimeRecord(item);
|
|
return record === null ? [] : [record];
|
|
}) : [];
|
|
}
|
|
function runtimeText(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
if (Array.isArray(value)) return value.map(runtimeText).join(",");
|
|
return String(value).replace(/\s+/gu, " ").trim();
|
|
}
|
|
|
|
function runtimeShort(value: unknown, max = 42): string {
|
|
const text = runtimeText(value);
|
|
return text.length <= max ? text : `${text.slice(0, max - 1)}…`;
|
|
}
|
|
|
|
function runtimeRate(value: unknown): string {
|
|
return typeof value === "number" && Number.isFinite(value) ? `${(value * 100).toFixed(2)}%` : "-";
|
|
}
|
|
|
|
function runtimeMillions(value: unknown): string {
|
|
return typeof value === "number" && Number.isFinite(value) ? `${(value / 1_000_000).toFixed(3)}M` : "-";
|
|
}
|
|
|
|
function runtimeDecimal(value: unknown, digits = 4): string {
|
|
return typeof value === "number" && Number.isFinite(value) ? value.toFixed(digits) : "-";
|
|
}
|
|
|
|
export function renderRuntimeResult(response: Record<string, unknown>, options: RuntimeOptions): RenderedCliResult {
|
|
const runtime = runtimeRecord(response.runtime) ?? {};
|
|
const ok = response.ok === true;
|
|
const lines: string[] = [];
|
|
if (runtime.ok !== true && typeof runtime.error === "string") {
|
|
lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=false`, `ERROR: ${runtime.error}`);
|
|
lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} <same-options> --json`);
|
|
return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n"));
|
|
}
|
|
if (options.action === "errors") renderRuntimeErrors(lines, runtime, options);
|
|
else if (options.action === "infrastructure") renderRuntimeInfrastructure(lines, runtime);
|
|
else if (options.action === "list") renderRuntimeAccountList(lines, runtime);
|
|
else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime);
|
|
else renderRuntimeMutation(lines, runtime, options);
|
|
lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} <same-options> --json`);
|
|
return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n"));
|
|
}
|
|
|
|
function renderRuntimeErrors(lines: string[], runtime: Record<string, unknown>, options: RuntimeOptions): void {
|
|
const allGroups = runtimeRecord(runtime.allGroups);
|
|
if (allGroups !== null) {
|
|
const totals = runtimeRecord(allGroups.pageTotals) ?? {};
|
|
const timeout = runtimeRecord(allGroups.responseHeaderTimeout) ?? {};
|
|
const timing = runtimeRecord(allGroups.requestTiming) ?? {};
|
|
const groups = runtimeRecords(allGroups.groups);
|
|
lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`);
|
|
lines.push(renderTable([
|
|
["GROUPS", "RETURNED", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UPSTREAM_ERR", "UP_RATE", "ATTENTION"],
|
|
[
|
|
runtimeText(allGroups.totalGroups), runtimeText(allGroups.returnedGroups), runtimeText(totals.requestCount),
|
|
runtimeText(totals.errorCount), runtimeRate(totals.errorRate), runtimeText(totals.upstreamErrorCount),
|
|
runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention),
|
|
],
|
|
]));
|
|
lines.push(renderTable([
|
|
["OPENAI_HEADER_TIMEOUT", "SOURCE", "GENERIC_TIMEOUT", "GENERIC_APPLIES_TO_OPENAI", "PAGE_MAX_TTFT_P99_MS", "OBSERVED_FLOOR_SECONDS"],
|
|
[
|
|
timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source),
|
|
`${runtimeText(timeout.genericResponseHeaderTimeoutSeconds)}s`, runtimeText(timeout.genericAppliesToOpenAI),
|
|
runtimeText(timing.maxTtftP99Ms), runtimeText(timing.observedFloorSeconds),
|
|
],
|
|
]));
|
|
if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`);
|
|
lines.push("", "GROUPS");
|
|
lines.push(renderTable([
|
|
["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "TTFT_P99_MS", "DUR_P99_MS", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"],
|
|
...groups.map((group) => [
|
|
runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status),
|
|
runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount),
|
|
runtimeRate(group.upstreamErrorRate), runtimeText(group.ttftP99Ms), runtimeText(group.durationP99Ms), runtimeText(group.totalAccounts), runtimeText(group.availableCount),
|
|
`${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no",
|
|
]),
|
|
]));
|
|
const disclosure = runtimeRecord(runtime.disclosure) ?? {};
|
|
lines.push("", `Detail: ${runtimeText(disclosure.group)}`);
|
|
if (typeof disclosure.nextPage === "string") lines.push(`Next: ${disclosure.nextPage}`);
|
|
return;
|
|
}
|
|
const group = runtimeRecord(runtime.group) ?? {};
|
|
const errors = runtimeRecord(runtime.errors) ?? {};
|
|
const nativeOps = runtimeRecord(errors.nativeOps) ?? {};
|
|
const overview = runtimeRecord(nativeOps.overview) ?? {};
|
|
const availability = runtimeRecord(nativeOps.accountAvailability) ?? {};
|
|
const concurrency = runtimeRecord(nativeOps.concurrency) ?? {};
|
|
const timeout = runtimeRecord(nativeOps.responseHeaderTimeout) ?? {};
|
|
const timing = runtimeRecord(nativeOps.requestTiming) ?? {};
|
|
const ttft = runtimeRecord(timing.ttft) ?? {};
|
|
const duration = runtimeRecord(timing.duration) ?? {};
|
|
const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {};
|
|
const accountQuality = runtimeRecord(nativeOps.accountQuality) ?? {};
|
|
lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`);
|
|
lines.push(renderTable([
|
|
["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"],
|
|
[
|
|
runtimeShort(group.name, 30), runtimeText(group.id), runtimeText(group.platform), runtimeText(overview.requestCount),
|
|
runtimeText(overview.errorCount), runtimeRate(overview.errorRate), runtimeRate(overview.upstreamErrorRate),
|
|
runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore),
|
|
],
|
|
]));
|
|
lines.push(renderTable([
|
|
["OPENAI_HEADER_TIMEOUT", "SOURCE", "TTFT_P50_MS", "TTFT_P95_MS", "TTFT_P99_MS", "TTFT_MAX_MS", "DURATION_P99_MS", "DURATION_MAX_MS"],
|
|
[
|
|
timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source),
|
|
runtimeText(ttft.p50_ms), runtimeText(ttft.p95_ms), runtimeText(ttft.p99_ms), runtimeText(ttft.max_ms),
|
|
runtimeText(duration.p99_ms), runtimeText(duration.max_ms),
|
|
],
|
|
]));
|
|
const candidates = runtimeRecords(timing.candidates);
|
|
if (candidates.length > 0) lines.push("", "TIMEOUT CANDIDATES", renderTable([
|
|
["SECONDS", "OBSERVED_FLOOR", "TTFT_P99_HEADROOM_MS", "TTFT_P99_BELOW", "ALL_DUR>=", "ERROR_DUR>=", "ASSESSMENT"],
|
|
...candidates.map((candidate) => [
|
|
runtimeText(candidate.seconds), runtimeText(candidate.observedFloor), runtimeText(candidate.ttftP99HeadroomMs), runtimeText(candidate.ttftP99Below),
|
|
runtimeText(candidate.totalDurationAtLeastCount), runtimeText(candidate.errorDurationAtLeastCount), runtimeText(candidate.assessment),
|
|
]),
|
|
]));
|
|
if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`);
|
|
const modelTiming = runtimeRecords(timing.modelTiming);
|
|
if (modelTiming.length > 0) lines.push("", "MODEL TIMING", renderTable([
|
|
["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"],
|
|
...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]),
|
|
]));
|
|
const qualityAccounts = runtimeRecords(accountQuality.accounts);
|
|
if (qualityAccounts.length > 0) {
|
|
lines.push("", "ACCOUNT QUALITY", renderTable([
|
|
["ACCOUNT", "ID", "PRIORITY", "SCORE", "GRADE", "QUALITY", "CONF", "ATTEMPTS", "SUCCESS", "FAIL", "UP_CUST", "EXCLUDED", "FAIL_RATE", "TTFT_P95_MS", "TTFT_N", "AVAILABLE", "REASONS"],
|
|
...qualityAccounts.map((item) => [
|
|
runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.priority), runtimeText(item.score), runtimeText(item.grade), runtimeText(item.assessment), runtimeText(item.confidence),
|
|
runtimeText(item.observedAttempts), runtimeText(item.successRequests), runtimeText(item.failureRequests), runtimeText(item.scoreableUpstreamErrorRequests),
|
|
runtimeText(item.excludedNonUpstreamErrorRequests), runtimeRate(item.failureRate),
|
|
runtimeText(item.ttftP95Ms), runtimeText(item.firstTokenSamples), runtimeText(item.currentlyAvailable),
|
|
runtimeShort(Array.isArray(item.reasons) ? item.reasons.join(",") : "", 58),
|
|
]),
|
|
]));
|
|
lines.push("ACCOUNT USAGE AND COST", renderTable([
|
|
["ACCOUNT", "ID", "REQUESTS", "TOKENS", "API_USD", "COST_RATE_CNY", "UPSTREAM_COST_CNY", "EXCLUDED_REASONS"],
|
|
...qualityAccounts.map((item) => {
|
|
const usage = runtimeRecord(item.usage) ?? {};
|
|
const excluded = runtimeRecords(item.excludedReasonBuckets).map((entry) => `${runtimeText(entry.reason)}:${runtimeText(entry.count)}`).join(",");
|
|
return [
|
|
runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(usage.requestCount), runtimeMillions(usage.tokenCount),
|
|
runtimeDecimal(usage.apiAmountUsd, 4), runtimeDecimal(usage.costRateCnyPerApiUsd, 3), runtimeDecimal(usage.upstreamCostCny, 4),
|
|
runtimeShort(excluded, 64),
|
|
];
|
|
}),
|
|
]));
|
|
const controlRows = qualityAccounts.filter((item) => [
|
|
item.failoverRequests, item.sameAccountRetryEvents, item.tempUnschedulableEvents, item.forwardFailedRequests, item.customerErrorRequests,
|
|
].some((value) => typeof value === "number" && value > 0));
|
|
if (controlRows.length > 0) lines.push("ACCOUNT FAILURE CONTROL", renderTable([
|
|
["ACCOUNT", "ID", "FAILOVER", "RECOVERED", "FAILED", "MISSING", "RETRY", "TEMP_UNSCHED", "FORWARD_FAIL", "CUSTOMER_ERR"],
|
|
...controlRows.map((item) => [
|
|
runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.failoverRequests), runtimeText(item.failoverRecovered),
|
|
runtimeText(item.failoverFailed), runtimeText(item.failoverOutcomeMissing), runtimeText(item.sameAccountRetryEvents),
|
|
runtimeText(item.tempUnschedulableEvents), runtimeText(item.forwardFailedRequests), runtimeText(item.customerErrorRequests),
|
|
]),
|
|
]));
|
|
const scoreRule = runtimeRecord(accountQuality.scoreRule) ?? {};
|
|
lines.push(`SCORE RULE reliability=${runtimeText(scoreRule.reliabilityWeight)} latency=${runtimeText(scoreRule.latencyWeight)} availability=${runtimeText(scoreRule.availabilityWeight)} confidence=separate priority=lower-is-higher highest=0`);
|
|
if (typeof accountQuality.attribution === "string") lines.push(`NOTE: ${accountQuality.attribution}`);
|
|
}
|
|
const errorModels = runtimeRecords(customerErrors.modelBuckets);
|
|
const errorAccounts = runtimeRecords(customerErrors.accountBuckets);
|
|
const errorDimensions = [
|
|
...runtimeRecords(customerErrors.statusBuckets).map((item) => ["status", runtimeText(item.value), runtimeText(item.count)]),
|
|
...runtimeRecords(customerErrors.streamBuckets).map((item) => ["mode", runtimeText(item.value), runtimeText(item.count)]),
|
|
...runtimeRecords(customerErrors.phaseBuckets).map((item) => ["phase", runtimeText(item.value), runtimeText(item.count)]),
|
|
...runtimeRecords(customerErrors.endpointBuckets).map((item) => ["endpoint", runtimeText(item.value), runtimeText(item.count)]),
|
|
...runtimeRecords(customerErrors.pathBuckets).map((item) => ["path", runtimeText(item.value), runtimeText(item.count)]),
|
|
];
|
|
if (errorModels.length > 0) lines.push("", `CUSTOMER ERRORS source=${runtimeText(customerErrors.source)} total=${runtimeText(customerErrors.total)}`, renderTable([
|
|
["MODEL", "COUNT"], ...errorModels.map((item) => [runtimeShort(item.value, 40), runtimeText(item.count)]),
|
|
]));
|
|
if (errorAccounts.length > 0) lines.push(renderTable([
|
|
["ACCOUNT_ID", "COUNT"], ...errorAccounts.map((item) => [runtimeText(item.value), runtimeText(item.count)]),
|
|
]));
|
|
const errorMessages = runtimeRecords(customerErrors.messageBuckets);
|
|
if (errorMessages.length > 0) lines.push("CUSTOMER ERROR MESSAGES", renderTable([
|
|
["MESSAGE", "COUNT"], ...errorMessages.map((item) => [runtimeShort(item.value, 100), runtimeText(item.count)]),
|
|
]));
|
|
if (errorDimensions.length > 0) lines.push(renderTable([["DIMENSION", "VALUE", "COUNT"], ...errorDimensions]));
|
|
const errorSamples = runtimeRecords(customerErrors.samples);
|
|
if (errorSamples.length > 0) lines.push("CUSTOMER ERROR INDEX", renderTable([
|
|
["REQUEST_ID", "MODEL", "STATUS", "ACCOUNT", "MODE", "PATH", "MESSAGE"],
|
|
...errorSamples.map((item) => [runtimeText(item.requestId), runtimeShort(item.model, 26), runtimeText(item.statusCode), runtimeText(item.accountId), runtimeText(item.mode), runtimeShort(item.path, 34), runtimeShort(item.message, 48)]),
|
|
]));
|
|
lines.push(renderTable([
|
|
["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"],
|
|
[
|
|
runtimeText(availability.totalAccounts), runtimeText(availability.availableCount), runtimeText(availability.rateLimitCount),
|
|
runtimeText(availability.errorCount), runtimeText(concurrency.currentInUse), runtimeText(concurrency.maxCapacity), runtimeText(concurrency.waitingInQueue),
|
|
],
|
|
]));
|
|
const unavailable = runtimeRecords(availability.unavailableAccounts);
|
|
if (unavailable.length > 0) {
|
|
lines.push("", "UNAVAILABLE ACCOUNTS", renderTable([
|
|
["ACCOUNT", "ID", "STATUS", "REASON"],
|
|
...unavailable.map((account) => [runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.status), runtimeShort(account.reason, 70)]),
|
|
]));
|
|
}
|
|
const root = runtimeRecord(errors.rootCauseAnalysis) ?? {};
|
|
const causes = runtimeRecords(root.causeBuckets);
|
|
if (causes.length > 0) {
|
|
lines.push("", `ROOT CAUSES scope=${runtimeText(root.scope)}`, renderTable([
|
|
["CAUSE", "COUNT"],
|
|
...causes.map((cause) => [runtimeText(cause.cause), runtimeText(cause.count)]),
|
|
]));
|
|
}
|
|
const policy = runtimeRecord(errors.policyEffectiveness) ?? {};
|
|
const rules = runtimeRecord(policy.ruleMatches) ?? {};
|
|
const failover = runtimeRecord(policy.failover) ?? {};
|
|
const forward = runtimeRecord(policy.forwardFailure) ?? {};
|
|
lines.push("", `POLICY EFFECTIVENESS scope=${runtimeText(policy.scope)}`);
|
|
lines.push(renderTable([
|
|
["TEMP_UNSCHED", "FAILOVER", "SUCCESS", "FAILED", "MISSING", "FORWARD_200", "FORWARD_ERR", "SELECT_FAIL"],
|
|
[
|
|
runtimeText(rules.tempUnschedulableEvents), runtimeText(failover.requests), runtimeText(failover.succeeded),
|
|
runtimeText(failover.failed), runtimeText(failover.outcomeMissing), runtimeText(forward.withHttpSuccess),
|
|
runtimeText(forward.withHttpError), runtimeText(policy.selectFailedEvents),
|
|
],
|
|
]));
|
|
const indexes = runtimeRecord(policy.requestIndexes) ?? {};
|
|
const indexRows = Object.entries(indexes).flatMap(([kind, value]) => {
|
|
const item = runtimeRecord(value);
|
|
return item === null ? [] : [[kind, runtimeText(item.count), runtimeText(item.sampleRequestId), item.moreAvailable === true ? "yes" : "no"]];
|
|
});
|
|
if (indexRows.length > 0) lines.push("", "REQUEST INDEX", renderTable([["TYPE", "COUNT", "SAMPLE_REQUEST_ID", "MORE"], ...indexRows]));
|
|
if (typeof root.attribution === "string") lines.push("", `NOTE: ${root.attribution}`);
|
|
if (typeof policy.attribution === "string") lines.push(`NOTE: ${policy.attribution}`);
|
|
const disclosure = runtimeRecord(runtime.disclosure) ?? {};
|
|
lines.push("", `Account: ${runtimeText(disclosure.account)}`, `Trace: ${runtimeText(disclosure.trace)}`, `Full: ${runtimeText(disclosure.complete)}`);
|
|
}
|
|
|
|
function renderRuntimeAccountList(lines: string[], runtime: Record<string, unknown>): void {
|
|
const group = runtimeRecord(runtime.group) ?? {};
|
|
const accounts = runtimeRecords(runtime.accounts);
|
|
lines.push(`SUB2API RUNTIME ACCOUNTS ok=${runtime.ok === true ? "true" : "false"} group=${runtimeText(group.name)} count=${accounts.length}`);
|
|
lines.push(renderTable([
|
|
["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "CONCURRENCY", "LOAD", "PRIORITY", "TEMP_RULES", "TEMPLATE"],
|
|
...accounts.map((account) => {
|
|
const temp = runtimeRecord(account.tempUnschedulable) ?? {};
|
|
return [
|
|
runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status),
|
|
runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`,
|
|
runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate),
|
|
];
|
|
}),
|
|
]));
|
|
lines.push("", "Detail: bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account <name-or-id>");
|
|
}
|
|
|
|
function renderRuntimeAccountDetail(lines: string[], runtime: Record<string, unknown>): void {
|
|
const account = runtimeRecord(runtime.account) ?? {};
|
|
const temp = runtimeRecord(account.tempUnschedulable) ?? {};
|
|
const priority = runtimeRecord(account.prioritySemantics) ?? {};
|
|
const limits = runtimeRecord(account.upstreamLimits) ?? {};
|
|
const probe = runtimeRecord(limits.nativeProbe) ?? {};
|
|
const globalSettings = runtimeRecord(limits.globalSettings) ?? {};
|
|
const windows = runtimeRecords(limits.windows);
|
|
const protections = runtimeRecords(limits.protections);
|
|
lines.push(`SUB2API RUNTIME ACCOUNT ok=${runtime.ok === true ? "true" : "false"}`);
|
|
lines.push(renderTable([
|
|
["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "PROXY", "CONCURRENCY", "LOAD", "PRIORITY", "TEMPLATE"],
|
|
[
|
|
runtimeShort(account.accountName, 42), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status),
|
|
runtimeText(account.schedulable), runtimeText(account.proxyId), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`,
|
|
runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(account.matchingTemplate),
|
|
],
|
|
]));
|
|
lines.push(renderTable([
|
|
["PRIORITY_ORDER", "HIGHEST_PRIORITY"],
|
|
[runtimeText(priority.order), runtimeText(priority.highest)],
|
|
]));
|
|
lines.push(renderTable([
|
|
["TEMP_ENABLED", "RULES", "STATUS_CODES", "POOL_MODE"],
|
|
[runtimeText(temp.enabled), runtimeText(temp.ruleCount), runtimeText(temp.statusCodes), runtimeText(account.poolModeEnabled)],
|
|
]));
|
|
lines.push("", "UPSTREAM LIMITS");
|
|
lines.push(renderTable([
|
|
["WINDOW", "USED", "RESET_AFTER", "RESET_AT", "DATA_SOURCE", "PROTECT_AT", "PROTECT_SOURCE", "TRIGGERED"],
|
|
...windows.map((window) => {
|
|
const protection = protections.find((item) => item.window === window.window) ?? {};
|
|
return [
|
|
runtimeText(window.window), runtimePercent(window.usedPercent), runtimeSeconds(window.resetAfterSeconds),
|
|
runtimeText(window.resetAt), runtimeText(window.source), runtimePercent(protection.thresholdPercent),
|
|
runtimeText(protection.source), runtimeText(protection.triggered),
|
|
];
|
|
}),
|
|
]));
|
|
lines.push(`NATIVE_QUOTA status=${runtimeText(probe.status)} allowed=${runtimeText(probe.allowed)} limitReached=${runtimeText(probe.limitReached)} fetchedAt=${runtimeText(probe.fetchedAt)}`);
|
|
if (typeof probe.reason === "string") lines.push(`NATIVE_QUOTA_REASON: ${probe.reason}`);
|
|
lines.push(`GLOBAL_AUTO_PAUSE status=${runtimeText(globalSettings.status)}`);
|
|
if (typeof globalSettings.reason === "string") lines.push(`GLOBAL_AUTO_PAUSE_REASON: ${globalSettings.reason}`);
|
|
if (typeof limits.attribution === "string") lines.push(`NOTE: ${limits.attribution}`);
|
|
}
|
|
|
|
function runtimePercent(value: unknown): string {
|
|
return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : "-";
|
|
}
|
|
|
|
function runtimeSeconds(value: unknown): string {
|
|
return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)}s` : "-";
|
|
}
|
|
|
|
function renderRuntimeInfrastructure(lines: string[], runtime: Record<string, unknown>): void {
|
|
const account = runtimeRecord(runtime.account) ?? {};
|
|
const infra = runtimeRecord(runtime.infrastructure) ?? {};
|
|
const proxy = runtimeRecord(infra.accountProxy) ?? {};
|
|
const app = runtimeRecord(infra.appRuntime) ?? {};
|
|
const route = runtimeRecord(infra.defaultRoute) ?? {};
|
|
const dns = runtimeRecord(infra.dns) ?? {};
|
|
const sockets = runtimeRecord(infra.proxySockets) ?? {};
|
|
const listener = runtimeRecord(infra.proxyEndpointOwner) ?? {};
|
|
const serviceLogs = runtimeRecord(infra.proxyServiceEvidence) ?? {};
|
|
const logs = runtimeRecord(infra.recentNetworkEvidence) ?? {};
|
|
lines.push(`SUB2API RUNTIME INFRASTRUCTURE ok=${runtime.ok === true ? "true" : "false"} since=${runtimeText(logs.window)}`);
|
|
if (runtime.ok !== true && typeof runtime.error === "string") lines.push(`ERROR: ${runtime.error}`);
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "ID", "TYPE", "STATUS", "SCHED", "PROXY_ID", "PROXY_NAME", "PROTOCOL", "PROXY_ENDPOINT", "PROXY_STATUS"],
|
|
[
|
|
runtimeText(account.accountName), runtimeText(account.accountId), runtimeText(account.type), runtimeText(account.status),
|
|
runtimeText(account.schedulable), runtimeText(proxy.id), runtimeText(proxy.name), runtimeText(proxy.protocol),
|
|
`${runtimeText(proxy.host)}:${runtimeText(proxy.port)}`, runtimeText(proxy.status),
|
|
],
|
|
]));
|
|
lines.push("", "APP RUNTIME", renderTable([
|
|
["MODE", "CONTAINER", "STATE", "RESTARTS", "OOM", "NETWORK_MODE", "STARTED_AT"],
|
|
[
|
|
runtimeText(app.runtimeMode), runtimeText(app.name), runtimeText(app.state), runtimeText(app.restartCount),
|
|
runtimeText(app.oomKilled), runtimeText(app.networkMode), runtimeText(app.startedAt),
|
|
],
|
|
]));
|
|
const envRows = runtimeRecords(infra.proxyEnvironment);
|
|
if (envRows.length > 0) lines.push("PROXY ENVIRONMENT", renderTable([
|
|
["VARIABLE", "PRESENT", "SCHEME", "HOST", "PORT", "CREDENTIALS_REDACTED"],
|
|
...envRows.map((item) => [runtimeText(item.name), runtimeText(item.present), runtimeText(item.scheme), runtimeText(item.host), runtimeText(item.port), runtimeText(item.credentialsRedacted)]),
|
|
]));
|
|
lines.push("NETWORK", renderTable([
|
|
["DEFAULT_IFACE", "DEFAULT_GATEWAY", "DNS_NAMESERVERS", "DNS_SEARCH", "PROXY_RESOLVED_IPS", "SOCKETS", "ESTABLISHED"],
|
|
[
|
|
runtimeText(route.interface), runtimeText(route.gateway), runtimeText(dns.nameservers), runtimeText(dns.search),
|
|
runtimeText(sockets.resolvedIps), runtimeText(sockets.total), runtimeText(sockets.established),
|
|
],
|
|
]));
|
|
const listeners = runtimeRecords(listener.listeners);
|
|
if (listeners.length > 0) lines.push("PROXY ENDPOINT OWNER", renderTable([
|
|
["LISTEN", "PROCESS", "PID", "UNIT", "ACTIVE", "RESTARTS", "TCP_PEERS", "UDP_PEERS"],
|
|
...listeners.map((item) => [runtimeText(item.localEndpoint), runtimeText(item.processName), runtimeText(item.pid), runtimeText(item.systemdUnit), runtimeText(item.activeState), runtimeText(item.serviceRestarts), runtimeText(item.establishedPeers), runtimeText(item.udpPeers)]),
|
|
]));
|
|
const serviceBuckets = runtimeRecords(serviceLogs.buckets);
|
|
if (serviceBuckets.length > 0) lines.push("PROXY SERVICE EVIDENCE", renderTable([
|
|
["CAUSE", "COUNT"],
|
|
...serviceBuckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]),
|
|
]));
|
|
const serviceSamples = runtimeRecords(serviceLogs.samples);
|
|
if (serviceSamples.length > 0) lines.push("PROXY SERVICE INDEX", renderTable([
|
|
["AT", "CAUSE", "MESSAGE"],
|
|
...serviceSamples.map((item) => [runtimeText(item.at), runtimeText(item.cause), runtimeShort(item.message, 100)]),
|
|
]));
|
|
const interfaces = runtimeRecords(infra.interfaces);
|
|
if (interfaces.length > 0) lines.push("INTERFACES", renderTable([
|
|
["NAME", "RX_BYTES", "TX_BYTES", "RX_ERRORS", "TX_ERRORS", "RX_DROPPED", "TX_DROPPED"],
|
|
...interfaces.map((item) => [runtimeText(item.name), runtimeText(item.rxBytes), runtimeText(item.txBytes), runtimeText(item.rxErrors), runtimeText(item.txErrors), runtimeText(item.rxDropped), runtimeText(item.txDropped)]),
|
|
]));
|
|
const buckets = runtimeRecords(logs.buckets);
|
|
lines.push("", `RECENT NETWORK EVIDENCE matched=${runtimeText(logs.matchedEvents)} source=${runtimeText(logs.source)}`);
|
|
if (buckets.length > 0) lines.push(renderTable([
|
|
["CAUSE", "COUNT"],
|
|
...buckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]),
|
|
]));
|
|
const samples = runtimeRecords(logs.samples);
|
|
if (samples.length > 0) lines.push("EVIDENCE INDEX", renderTable([
|
|
["AT", "REQUEST_ID", "CAUSE", "STATUS", "MESSAGE"],
|
|
...samples.map((item) => [runtimeText(item.at), runtimeText(item.requestId), runtimeText(item.cause), runtimeText(item.statusCode), runtimeShort(item.message, 90)]),
|
|
]));
|
|
if (typeof infra.attribution === "string") lines.push(`NOTE: ${infra.attribution}`);
|
|
}
|
|
|
|
function renderRuntimeMutation(lines: string[], runtime: Record<string, unknown>, options: RuntimeOptions): void {
|
|
const plans = runtimeRecords(runtime.plans);
|
|
if (plans.length > 0) {
|
|
const summary = runtimeRecord(runtime.summary) ?? {};
|
|
lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`);
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES", "RESULT"],
|
|
...plans.map((plan) => {
|
|
const account = runtimeRecord(plan.account) ?? {};
|
|
const before = runtimeRecord(plan.before) ?? {};
|
|
const after = runtimeRecord(plan.after) ?? {};
|
|
return [
|
|
runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template),
|
|
runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), runtimeText(plan.result),
|
|
];
|
|
}),
|
|
]));
|
|
lines.push(`SUMMARY selected=${runtimeText(summary.selected)} changed=${runtimeText(summary.changed)} succeeded=${runtimeText(summary.succeeded)} failed=${runtimeText(summary.failed)}`);
|
|
return;
|
|
}
|
|
const plan = runtimeRecord(runtime.plan) ?? {};
|
|
const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {};
|
|
const before = runtimeRecord(plan.before) ?? {};
|
|
const after = runtimeRecord(plan.after) ?? {};
|
|
lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`);
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES"],
|
|
[
|
|
runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template),
|
|
runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes),
|
|
],
|
|
]));
|
|
}
|