fix: aggregate Sub2API scores on NC01

2026-07-17 13:10:39 +02:00
parent 2f280e5199
commit f235135398
13 changed files with 776 additions and 108 deletions
@@ -98,6 +98,7 @@ runtime:
      databasePath: .state/sub2rank.sqlite
      scoreCachePath: .state/account-scores.json
      monitorWorkDir: /root/unidesk
+      temporalTaskQueue: apistate-native
    production:
      mode: http
      baseUrl: https://apistate.hwpod.com
@@ -171,6 +172,8 @@ runtime:
      databasePath: .state/sub2rank.sqlite
      scoreCachePath: .state/account-scores.json
      monitorWorkDir: /root/unidesk
+      temporalTaskQueue: apistate-native
+      scoreScheduleWorkflowId: apistate-native-score-refresh-schedule
      adminTokenEnv: APISTATE_API_KEY
      sub2apiAdminEmailEnv: SUB2API_ADMIN_EMAIL
      sub2apiAdminPasswordEnv: SUB2API_ADMIN_PASSWORD
@@ -189,6 +192,8 @@ runtime:
      databasePath: /var/lib/sub2rank/sub2rank.sqlite
      scoreCachePath: /var/lib/sub2rank/account-scores.json
      monitorWorkDir: /opt/unidesk
+      temporalTaskQueue: apistate
+      scoreScheduleWorkflowId: apistate-score-refresh-schedule
      adminTokenEnv: APISTATE_API_KEY
      sub2apiAdminEmailEnv: SUB2API_ADMIN_EMAIL
      sub2apiAdminPasswordEnv: SUB2API_ADMIN_PASSWORD
@@ -160,7 +160,7 @@ function isAppCommand(value: AppCommand | Record<string, unknown>): value is App

 async function embedded(parsed: Parsed, config: ReturnType<typeof loadConfig>, target: EmbeddedCliTarget): Promise<unknown> {
  if (parsed.command.join(" ") === "scores refresh" || parsed.command.join(" ") === "workflow status") {
-    const temporal = await TemporalGateway.connect(config);
+    const temporal = await TemporalGateway.connect(config, { taskQueue: target.temporalTaskQueue });
    try {
      if (parsed.command[0] === "scores") return await temporal.submit({ kind: "scores.refresh" });
      if (!parsed.id) throw new Error("workflow status requires --id");
@@ -174,7 +174,7 @@ async function embedded(parsed: Parsed, config: ReturnType<typeof loadConfig>, t
  const context = createEmbeddedContext(config, target);
  let temporal: TemporalGateway | null = null;
  try {
-    if (usesWorkflow(command)) temporal = await TemporalGateway.connect(config);
+    if (usesWorkflow(command)) temporal = await TemporalGateway.connect(config, { taskQueue: target.temporalTaskQueue });
    return await new ApplicationDispatcher({ lottery: context.service, scores: context.monitor }, temporal).dispatch(command);
  } finally {
    if (temporal) await temporal.close();
@@ -206,11 +206,15 @@ async function remote(parsed: Parsed, config: ReturnType<typeof loadConfig>, tar
  if (group === "credit" && action === "test") return await client.creditTest(parsed.confirm);
  if (group === "api" && action === "smoke") {
    const [status, scores, ranking, lottery] = await Promise.all([client.serviceStatus(), client.scores(), client.ranking(), client.lottery()]);
-    const refreshed = await client.refreshScores();
+    const refreshed = await client.workflowSubmit({ kind: "scores.refresh" });
    return {
      ok: status.ok === true && scores.ok === true && refreshed.ok === true && ranking.ok === true && lottery.ok === true,
      action: "apistate-api-smoke",
-      checks: { status: status.ok === true, scores: scores.ok === true, refresh: refreshed.ok === true && refreshed.snapshotOk === true, ranking: ranking.ok === true, lottery: lottery.ok === true },
+      checks: { status: status.ok === true, scores: scores.ok === true, refreshSubmitted: refreshed.ok === true, ranking: ranking.ok === true, lottery: lottery.ok === true },
+      workflowId: refreshed.workflowId,
+      runId: refreshed.runId,
+      state: refreshed.state,
+      next: "workflow status --id <workflowId>",
      valuesPrinted: false,
    };
  }
@@ -244,7 +248,10 @@ export async function runCli(args: string[]): Promise<void> {
    const config = loadConfig(parsed.configPath);
    if (parsed.command.join(" ") === "config validate") return emit({
      ok: true, configPath: config.configPath, kind: config.kind, service: config.metadata.name,
-      temporalNamespace: config.temporal.namespace, temporalTaskQueue: config.temporal.taskQueue,
+      temporalNamespace: config.temporal.namespace,
+      temporalTaskQueue: config.temporal.taskQueue,
+      cliTargets: Object.fromEntries(Object.entries(config.runtime.cliTargets).map(([id, target]) => [id, target.mode === "embedded" ? { mode: target.mode, temporalTaskQueue: target.temporalTaskQueue } : { mode: target.mode }])),
+      serverTargets: Object.fromEntries(Object.entries(config.runtime.serverTargets).map(([id, target]) => [id, { temporalTaskQueue: target.temporalTaskQueue, scoreScheduleWorkflowId: target.scoreScheduleWorkflowId }])),
      refreshIntervalMinutes: config.monitor.refreshIntervalMinutes, scoreWindow: config.monitor.scoreWindow,
      automaticCreditEnabled: config.lottery.automaticCredit.enabled, valuesPrinted: false,
    }, parsed.json);
@@ -0,0 +1,118 @@
+import { describe, expect, test } from "bun:test";
+import { aggregateNativeGroupScore, collectNativeScores } from "./account-score-native";
+import type { Sub2ApiClient } from "./sub2api-client";
+
+const availableOps = { status: "available" as const, data: {}, reason: null };
+
+describe("aggregateNativeGroupScore", () => {
+  test("scores upstream failures and excludes client input locally", () => {
+    const group = { id: 2, name: "pool", platform: "openai", status: "active" };
+    const account = { id: 15, name: "primary 0.02", platform: "openai", status: "active", schedulable: true, priority: 1 };
+    const usage = Array.from({ length: 20 }, (_, index) => ({
+      id: index + 1,
+      account_id: 15,
+      group_id: 2,
+      model: "gpt-test",
+      stream: true,
+      input_tokens: 100,
+      output_tokens: 20,
+      actual_cost: 0.1,
+      duration_ms: 20_000,
+      first_token_ms: 5_000 + index,
+      created_at: "2026-07-17T09:00:00Z",
+    }));
+    const result = aggregateNativeGroupScore({
+      group,
+      accounts: [account],
+      usage,
+      requestErrors: [
+        { id: 1, request_id: "req-fail", account_id: 15, status_code: 502, phase: "upstream", type: "upstream_error", message: "Upstream service temporarily unavailable" },
+        { id: 2, request_id: "req-input", account_id: 15, status_code: 400, phase: "client", type: "invalid_request", message: "Input must be a list" },
+      ],
+      systemLogs: [
+        { id: 10, created_at: "2026-07-17T09:01:00Z", message: "openai.upstream_failover_switching", request_id: "req-fail", account_id: 15, extra: { group_id: 2 } },
+        { id: 11, created_at: "2026-07-17T09:01:01Z", message: "http request completed", request_id: "req-fail", account_id: 15, extra: { group_id: 2, status_code: 200 } },
+      ],
+      overview: {
+        request_count_total: 22,
+        error_count_total: 1,
+        upstream_error_count_excl_429_529: 1,
+        upstream_429_count: 0,
+        upstream_529_count: 0,
+        error_rate: 1 / 22,
+        upstream_error_rate: 1 / 22,
+        ttft: { p99_ms: 5019 },
+      },
+      availability: availableOps,
+      concurrency: availableOps,
+    });
+    const row = result.accounts[0]!;
+    expect(row.successRequests).toBe(20);
+    expect(row.failureRequests).toBe(1);
+    expect(row.scoreableUpstreamErrorRequests).toBe(1);
+    expect(row.excludedNonUpstreamErrorRequests).toBe(1);
+    expect(row.failoverRequests).toBe(1);
+    expect(row.failoverRecovered).toBe(1);
+    expect((row.usage as Record<string, unknown>).tokenCount).toBe(2400);
+    expect((row.usage as Record<string, unknown>).upstreamCostCny).toBe(0.04);
+    expect(result.collection.mode).toBe("nc01-native-api-local-aggregation");
+  });
+
+  test("does not attribute another group policy event", () => {
+    const result = aggregateNativeGroupScore({
+      group: { id: 2, name: "pool", platform: "openai", status: "active" },
+      accounts: [{ id: 15, name: "primary 0.0", platform: "openai", status: "active", schedulable: true, priority: 1 }],
+      usage: [],
+      requestErrors: [],
+      systemLogs: [{ id: 1, created_at: "2026-07-17T09:00:00Z", message: "openai.forward_failed", request_id: "req-other", account_id: 15, extra: { group_id: 3 } }],
+      overview: {},
+      availability: {
+        status: "available",
+        data: {
+          group: { "2": { total_accounts: 1, available_count: 0, rate_limit_count: 0, error_count: 1 } },
+          account: { "15": { account_id: 15, group_id: 2, is_available: false } },
+        },
+        reason: null,
+      },
+      concurrency: { status: "available", data: { group: { "2": { current_in_use: 0, max_capacity: 10, waiting_in_queue: 0 } } }, reason: null },
+    });
+    expect(result.accounts[0]!.forwardFailedRequests).toBe(0);
+    expect(result.accounts[0]!.currentlyAvailable).toBe(false);
+    expect(result.group.unavailableAccountCount).toBe(1);
+    expect(result.group.maxCapacity).toBe(10);
+  });
+
+  test("collects groups sequentially and deduplicates marker results", async () => {
+    const calls: string[] = [];
+    const fake = {
+      async listGroups() {
+        calls.push("groups");
+        return [
+          { id: 2, name: "pool", platform: "openai", status: "active" },
+          { id: 3, name: "self", platform: "openai", status: "active" },
+        ];
+      },
+      async listGroupAccounts(groupId: number) { calls.push(`accounts:${groupId}`); return []; },
+      async getOpsOverview(groupId: number) { calls.push(`overview:${groupId}`); return {}; },
+      async getOpsAccountAvailability(groupId: number) { calls.push(`availability:${groupId}`); return {}; },
+      async getOpsConcurrency(groupId: number) { calls.push(`concurrency:${groupId}`); return {}; },
+      async listGroupUsage(groupId: number) { calls.push(`usage:${groupId}`); return []; },
+      async listRequestErrors(groupId: number) { calls.push(`errors:${groupId}`); return []; },
+    } as unknown as Sub2ApiClient;
+    const events = {
+      async collect() {
+        calls.push("events");
+        return {
+          events: [{ id: 1, created_at: "2026-07-17T09:00:00Z", message: "account_temp_unschedulable", account_id: 15 }],
+          evidence: { source: "test", eventCount: 1 },
+        };
+      },
+    };
+    const result = await collectNativeScores(fake, events, "8h", new Date("2026-07-17T10:00:00Z"));
+    expect(result.groups).toHaveLength(2);
+    expect(calls.filter((call) => call === "events")).toHaveLength(1);
+    expect(calls.indexOf("accounts:3")).toBeGreaterThan(calls.indexOf("errors:2"));
+    const collection = result.collection.groups as Array<Record<string, unknown>>;
+    expect(collection[0]!.policyEventRows).toBe(1);
+  });
+});
@@ -0,0 +1,358 @@
+import type {
+  Sub2ApiAccount,
+  Sub2ApiGroup,
+  Sub2ApiRequestError,
+  Sub2ApiSystemLog,
+  Sub2ApiUsageRow,
+} from "./sub2api-client";
+import { Sub2ApiClient } from "./sub2api-client";
+import type { RuntimePolicyEventSource } from "./runtime-policy-events";
+
+type Row = Record<string, unknown>;
+
+export interface NativeGroupScoreInput {
+  group: Sub2ApiGroup;
+  accounts: Sub2ApiAccount[];
+  usage: Sub2ApiUsageRow[];
+  requestErrors: Sub2ApiRequestError[];
+  systemLogs: Sub2ApiSystemLog[];
+  overview: Record<string, unknown>;
+  availability: NativeOpsResult;
+  concurrency: NativeOpsResult;
+}
+
+interface NativeOpsResult {
+  status: "available" | "unavailable";
+  data: Row;
+  reason: string | null;
+}
+
+const policyMarkers = {
+  temp: "account_temp_unschedulable",
+  failover: "openai.upstream_failover_switching",
+  forward: "openai.forward_failed",
+  retryOpenAI: "openai.pool_mode_same_account_retry",
+  retryGateway: "gateway.failover_same_account_retry",
+  completed: "http request completed",
+} as const;
+
+export const nativeScoreSystemLogMarkers = Object.values(policyMarkers);
+
+function windowStart(window: string, now: Date): Date {
+  const match = window.match(/^([1-9][0-9]*)(m|h|d)$/u);
+  if (!match) throw new Error(`unsupported score window: ${window}`);
+  const amount = Number(match[1]);
+  const unitMs = match[2] === "m" ? 60_000 : match[2] === "h" ? 3_600_000 : 86_400_000;
+  return new Date(now.getTime() - amount * unitMs);
+}
+
+export async function collectNativeScores(client: Sub2ApiClient, eventSource: RuntimePolicyEventSource, window: string, now = new Date()): Promise<{ groups: Row[]; accounts: Row[]; collection: Row }> {
+  const start = windowStart(window, now);
+  const groups = await client.listGroups();
+  const policyEvents = await eventSource.collect(window);
+  const groupRows: Row[] = [];
+  const accountRows: Row[] = [];
+  const collectionRows: Row[] = [];
+  for (const group of groups) {
+    // Keep database-heavy reads sequential on the two-core Sub2API host.
+    const accounts = await client.listGroupAccounts(group.id, group.platform);
+    const overview = await client.getOpsOverview(group.id, group.platform, start);
+    const availability = await optionalOps(() => client.getOpsAccountAvailability(group.id, group.platform));
+    const concurrency = await optionalOps(() => client.getOpsConcurrency(group.id, group.platform));
+    const usage = await client.listGroupUsage(group.id, start, now);
+    const requestErrors = await client.listRequestErrors(group.id, group.platform, start);
+    const result = aggregateNativeGroupScore({ group, accounts, usage, requestErrors, systemLogs: policyEvents.events, overview, availability, concurrency });
+    groupRows.push(result.group);
+    accountRows.push(...result.accounts.map((account) => ({ groupId: group.id, groupName: group.name, platform: group.platform, ...account })));
+    collectionRows.push({ groupId: group.id, groupName: group.name, ...result.collection });
+  }
+  return {
+    groups: groupRows,
+    accounts: accountRows,
+    collection: {
+      mode: "nc01-native-api-local-aggregation",
+      window,
+      startAt: start.toISOString(),
+      endAt: now.toISOString(),
+      groupCount: groups.length,
+      groups: collectionRows,
+      policyEvents: policyEvents.evidence,
+    },
+  };
+}
+
+async function optionalOps(operation: () => Promise<Row>): Promise<NativeOpsResult> {
+  try {
+    return { status: "available", data: await operation(), reason: null };
+  } catch (error) {
+    return { status: "unavailable", data: {}, reason: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+function numeric(value: unknown): number | null {
+  const parsed = Number(value);
+  return value !== null && value !== undefined && Number.isFinite(parsed) ? parsed : null;
+}
+
+function record(value: unknown): Row {
+  return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
+}
+
+function percentile(values: number[], fraction: number): number | null {
+  if (values.length === 0) return null;
+  const ordered = [...values].sort((left, right) => left - right);
+  const position = (ordered.length - 1) * fraction;
+  const lower = Math.floor(position);
+  const upper = Math.min(lower + 1, ordered.length - 1);
+  return Math.round(ordered[lower]! + (ordered[upper]! - ordered[lower]!) * (position - lower));
+}
+
+function costRate(name: string): number | null {
+  const match = name.match(/(\d+(?:\.\d+)?)$/u);
+  return match ? Number(match[1]) : null;
+}
+
+function extra(log: Sub2ApiSystemLog): Row {
+  return log.extra && typeof log.extra === "object" ? log.extra : {};
+}
+
+function accountId(log: Sub2ApiSystemLog): number | null {
+  return numeric(log.account_id) ?? numeric(extra(log).account_id);
+}
+
+function requestId(log: Sub2ApiSystemLog): string | null {
+  const value = log.request_id ?? extra(log).request_id;
+  return typeof value === "string" && value ? value : null;
+}
+
+function groupMatches(log: Sub2ApiSystemLog, groupId: number): boolean {
+  const value = numeric(extra(log).group_id);
+  return value === null || value === groupId;
+}
+
+function customerErrorAttribution(row: Sub2ApiRequestError): { scoreable: boolean; reason: string } {
+  const message = String(row.message ?? row.error_message ?? "").toLowerCase();
+  if (message.includes("context window") || message.includes("context_length_exceeded")) return { scoreable: false, reason: "context-window" };
+  if (message.includes("input must be a list")) return { scoreable: false, reason: "invalid-client-input" };
+  if (message.includes("not supported by any configured account") || message.includes("no available channel for model")) return { scoreable: false, reason: "model-route" };
+  const phase = String(row.phase ?? "").toLowerCase();
+  if (["internal", "client", "business"].includes(phase)) return { scoreable: false, reason: "non-upstream-phase" };
+  if (row.account_id === null || row.account_id === undefined) return { scoreable: false, reason: "no-account-attribution" };
+  const category = String(row.type ?? "").toLowerCase();
+  if (phase === "upstream" || category.includes("upstream")) return { scoreable: true, reason: "explicit-upstream" };
+  const stable = ["upstream service temporarily unavailable", "upstream request failed", "bad gateway", "gateway timeout", "error code: 502", "error code: 503", "error code: 504", "error code: 524"];
+  return stable.some((marker) => message.includes(marker))
+    ? { scoreable: true, reason: "stable-upstream-message" }
+    : { scoreable: false, reason: "unattributed-customer-error" };
+}
+
+function grade(score: number | null, comparable: boolean, attempts: number): string {
+  if (score === null || (!comparable && !(score < 60 && attempts >= 10))) return "insufficient";
+  return score >= 90 ? "A" : score >= 80 ? "B" : score >= 70 ? "C" : score >= 60 ? "D" : "E";
+}
+
+export function aggregateNativeGroupScore(input: NativeGroupScoreInput): { group: Row; accounts: Row[]; collection: Row } {
+  const usageByAccount = new Map<number, Sub2ApiUsageRow[]>();
+  for (const row of input.usage) {
+    if (row.account_id === null) continue;
+    usageByAccount.set(row.account_id, [...(usageByAccount.get(row.account_id) ?? []), row]);
+  }
+
+  const scoreableByAccount = new Map<number, Set<string>>();
+  const excludedByAccount = new Map<number, Map<string, number>>();
+  const customerErrorCount = new Map<number, Set<string>>();
+  for (const row of input.requestErrors) {
+    if (row.account_id === null || row.account_id === undefined || !row.request_id) continue;
+    const id = row.account_id;
+    customerErrorCount.set(id, new Set([...(customerErrorCount.get(id) ?? []), row.request_id]));
+    const attribution = customerErrorAttribution(row);
+    if (attribution.scoreable) scoreableByAccount.set(id, new Set([...(scoreableByAccount.get(id) ?? []), row.request_id]));
+    else {
+      const reasons = excludedByAccount.get(id) ?? new Map<string, number>();
+      reasons.set(attribution.reason, (reasons.get(attribution.reason) ?? 0) + 1);
+      excludedByAccount.set(id, reasons);
+    }
+  }
+
+  const finalStatus = new Map<string, number>();
+  const failover = new Map<number, Set<string>>();
+  const forward = new Map<number, Set<string>>();
+  const temp = new Map<number, number>();
+  const retries = new Map<number, number>();
+  const upstreamStatuses = new Map<number, Map<number, number>>();
+  for (const log of input.systemLogs) {
+    if (!groupMatches(log, input.group.id)) continue;
+    const id = accountId(log);
+    const request = requestId(log);
+    const message = log.message ?? "";
+    if (message.includes(policyMarkers.completed) && request) {
+      const status = numeric(extra(log).status_code);
+      if (status !== null) finalStatus.set(request, status);
+    }
+    if (id === null) continue;
+    if (message.includes(policyMarkers.temp)) temp.set(id, (temp.get(id) ?? 0) + 1);
+    if ((message.includes(policyMarkers.retryOpenAI) || message.includes(policyMarkers.retryGateway))) retries.set(id, (retries.get(id) ?? 0) + 1);
+    const eventGroupId = numeric(extra(log).group_id);
+    if (request && eventGroupId === input.group.id && message.includes(policyMarkers.failover)) {
+      failover.set(id, new Set([...(failover.get(id) ?? []), request]));
+      const status = numeric(extra(log).upstream_status);
+      if (status !== null) {
+        const buckets = upstreamStatuses.get(id) ?? new Map<number, number>();
+        buckets.set(status, (buckets.get(status) ?? 0) + 1);
+        upstreamStatuses.set(id, buckets);
+      }
+    }
+    if (request && eventGroupId === input.group.id && message.includes(policyMarkers.forward)) forward.set(id, new Set([...(forward.get(id) ?? []), request]));
+  }
+
+  const availabilityAccounts = Object.values(record(input.availability.data.account)).map(record);
+  const availabilityByAccount = new Map<number, Row>();
+  for (const value of availabilityAccounts) {
+    const id = numeric(value.account_id);
+    if (id !== null && numeric(value.group_id) === input.group.id) availabilityByAccount.set(id, value);
+  }
+  const availabilityGroups = record(input.availability.data.group);
+  const availabilityGroup = record(availabilityGroups[String(input.group.id)] ?? availabilityGroups[input.group.id]);
+  const concurrencyGroups = record(input.concurrency.data.group);
+  const concurrencyGroup = record(concurrencyGroups[String(input.group.id)] ?? concurrencyGroups[input.group.id]);
+
+  const accountRows = input.accounts.map((account): Row => {
+    const usages = usageByAccount.get(account.id) ?? [];
+    const streams = usages.filter((row) => row.stream);
+    const ttft = streams.flatMap((row) => row.first_token_ms === null ? [] : [row.first_token_ms]);
+    const durations = usages.flatMap((row) => row.duration_ms === null ? [] : [row.duration_ms]);
+    const scoreable = scoreableByAccount.get(account.id) ?? new Set<string>();
+    const failed = new Set([...(failover.get(account.id) ?? []), ...(forward.get(account.id) ?? []), ...scoreable]);
+    const successRequests = usages.length;
+    const failureRequests = failed.size;
+    const attempts = successRequests + failureRequests;
+    const failureRate = attempts > 0 ? Math.round(failureRequests / attempts * 1_000_000) / 1_000_000 : null;
+    const ttftP95Ms = percentile(ttft, 0.95);
+    const reliability = failureRate === null ? null : Math.round(60 * (1 - Math.min(Math.max(failureRate, 0), 0.2) / 0.2) * 100) / 100;
+    const latency = ttft.length < 5 || ttftP95Ms === null ? null : Math.round(25 * (1 - Math.min(Math.max(ttftP95Ms - 10_000, 0), 170_000) / 170_000) * 100) / 100;
+    const nativeAvailability = availabilityByAccount.get(account.id)?.is_available;
+    const currentlyAvailable = typeof nativeAvailability === "boolean"
+      ? nativeAvailability
+      : account.status === "active" && account.schedulable !== false;
+    const availability = currentlyAvailable ? 15 : account.status === "active" ? 8 : 0;
+    const availableWeight = (reliability === null ? 0 : 60) + (latency === null ? 0 : 25) + 15;
+    const score = attempts > 0 ? Math.round(((reliability ?? 0) + (latency ?? 0) + availability) / availableWeight * 1_000) / 10 : null;
+    const comparable = attempts >= 10 && ttft.length >= 5;
+    const accountGrade = grade(score, comparable, attempts);
+    const failoverIds = failover.get(account.id) ?? new Set<string>();
+    const failoverRecovered = [...failoverIds].filter((id) => (finalStatus.get(id) ?? 999) < 400).length;
+    const failoverFailed = [...failoverIds].filter((id) => (finalStatus.get(id) ?? 0) >= 400).length;
+    const models = new Map<string, number>();
+    for (const usage of usages) models.set(usage.model || "unknown", (models.get(usage.model || "unknown") ?? 0) + 1);
+    const amount = usages.reduce((sum, row) => sum + (numeric(row.actual_cost) ?? 0), 0);
+    const rate = costRate(account.name);
+    const reasons = [
+      ...(failureRate !== null && failureRate >= 0.1 ? ["failure-rate>=10%"] : failureRate !== null && failureRate >= 0.03 ? ["failure-rate>=3%"] : []),
+      ...(failoverIds.size > 0 ? ["upstream-failover-triggered"] : []),
+      ...(!currentlyAvailable ? ["currently-unavailable"] : []),
+      ...(ttft.length < 5 ? ["ttft-evidence-insufficient"] : []),
+      ...(attempts < 10 ? ["request-evidence-insufficient"] : []),
+    ];
+    return {
+      accountId: account.id,
+      accountName: account.name,
+      status: account.status,
+      schedulable: account.schedulable,
+      currentlyAvailable,
+      priority: account.priority,
+      priorityOrder: "lower-is-higher",
+      score,
+      grade: accountGrade,
+      assessment: ({ A: "preferred", B: "healthy", C: "watch", D: "degraded", E: "poor" } as Row)[accountGrade] ?? "insufficient-evidence",
+      confidence: attempts >= 50 && ttft.length >= 20 ? "high" : attempts >= 10 && ttft.length >= 5 ? "medium" : "low",
+      scoreComparable: comparable,
+      observedAttempts: attempts,
+      successRequests,
+      failureRequests,
+      failureRate,
+      streamSuccessRequests: streams.length,
+      firstTokenSamples: ttft.length,
+      firstTokenCoverage: streams.length > 0 ? Math.round(ttft.length / streams.length * 1_000_000) / 1_000_000 : null,
+      ttftP50Ms: percentile(ttft, 0.5),
+      ttftP95Ms,
+      ttftP99Ms: percentile(ttft, 0.99),
+      ttftMaxMs: ttft.length ? Math.max(...ttft) : null,
+      durationP95Ms: percentile(durations, 0.95),
+      modelBuckets: [...models].sort((left, right) => right[1] - left[1]).slice(0, 8).map(([model, count]) => ({ model, count })),
+      customerErrorRequests: customerErrorCount.get(account.id)?.size ?? 0,
+      scoreableUpstreamErrorRequests: scoreable.size,
+      excludedNonUpstreamErrorRequests: [...(excludedByAccount.get(account.id)?.values() ?? [])].reduce((sum, value) => sum + value, 0),
+      excludedReasonBuckets: [...(excludedByAccount.get(account.id) ?? [])].map(([reason, count]) => ({ reason, count })),
+      usage: {
+        requestCount: usages.length,
+        tokenCount: usages.reduce((sum, row) => sum + row.input_tokens + row.output_tokens, 0),
+        apiAmountUsd: Math.round(amount * 100_000_000) / 100_000_000,
+        costRateCnyPerApiUsd: rate,
+        upstreamCostCny: rate === null ? null : Math.round(amount * rate * 100_000_000) / 100_000_000,
+      },
+      failoverRequests: failoverIds.size,
+      failoverRecovered,
+      failoverFailed,
+      failoverOutcomeMissing: failoverIds.size - failoverRecovered - failoverFailed,
+      sameAccountRetryEvents: retries.get(account.id) ?? 0,
+      tempUnschedulableEvents: temp.get(account.id) ?? 0,
+      forwardFailedRequests: forward.get(account.id)?.size ?? 0,
+      upstreamStatusBuckets: [...(upstreamStatuses.get(account.id) ?? [])].sort((left, right) => left[0] - right[0]).map(([statusCode, count]) => ({ statusCode, count })),
+      scoreComponents: { reliability, latency, availability, availableWeight },
+      reasons,
+      usageStatus: "available",
+      usageReason: null,
+    };
+  });
+
+  const overview = input.overview;
+  const requestCount = numeric(overview.request_count_total) ?? 0;
+  const errorCount = numeric(overview.error_count_total) ?? 0;
+  const upstreamErrorCount = [overview.upstream_error_count_excl_429_529, overview.upstream_429_count, overview.upstream_529_count]
+    .reduce((sum: number, value) => sum + (numeric(value) ?? 0), 0);
+  const ttftOverview = overview.ttft && typeof overview.ttft === "object" ? overview.ttft as Row : {};
+  const durationOverview = overview.duration && typeof overview.duration === "object" ? overview.duration as Row : {};
+  const unavailableAccountCount = numeric(availabilityGroup.total_accounts) !== null && numeric(availabilityGroup.available_count) !== null
+    ? Math.max(0, Number(availabilityGroup.total_accounts) - Number(availabilityGroup.available_count))
+    : [...availabilityByAccount.values()].filter((value) => value.is_available === false).length;
+  return {
+    group: {
+      groupId: input.group.id,
+      groupName: input.group.name,
+      platform: input.group.platform,
+      status: input.group.status,
+      requestCount,
+      errorCount,
+      errorRate: numeric(overview.error_rate),
+      upstreamErrorCount,
+      upstreamErrorRate: numeric(overview.upstream_error_rate),
+      businessLimitedCount: numeric(overview.business_limited_count) ?? 0,
+      ttftP99Ms: numeric(ttftOverview.p99_ms),
+      durationP99Ms: numeric(durationOverview.p99_ms),
+      totalAccounts: numeric(availabilityGroup.total_accounts),
+      availableCount: numeric(availabilityGroup.available_count),
+      rateLimitCount: numeric(availabilityGroup.rate_limit_count),
+      errorAccountCount: numeric(availabilityGroup.error_count),
+      unavailableAccountCount,
+      currentInUse: numeric(concurrencyGroup.current_in_use),
+      maxCapacity: numeric(concurrencyGroup.max_capacity),
+      waitingInQueue: numeric(concurrencyGroup.waiting_in_queue),
+      needsAttention: errorCount > 0 || upstreamErrorCount > 0 || unavailableAccountCount > 0,
+      opsStatus: {
+        overview: "available",
+        accountAvailability: input.availability.status,
+        concurrency: input.concurrency.status,
+      },
+    },
+    accounts: accountRows,
+    collection: {
+      mode: "nc01-native-api-local-aggregation",
+      accountCount: input.accounts.length,
+      usageRows: input.usage.length,
+      requestErrorRows: input.requestErrors.length,
+      policyEventRows: input.systemLogs.length,
+      accountAvailabilityStatus: input.availability.status,
+      concurrencyStatus: input.concurrency.status,
+    },
+  };
+}
@@ -1,7 +1,10 @@
-import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
 import { mergeAccountScores } from "./account-score-aggregation";
+import { collectNativeScores } from "./account-score-native";
 import type { AppConfig } from "./config";
+import type { Sub2ApiClient } from "./sub2api-client";
+import type { RuntimePolicyEventSource } from "./runtime-policy-events";

 interface ScoreSnapshot {
  ok: boolean;
@@ -14,6 +17,7 @@ interface ScoreSnapshot {
  accounts: Array<Record<string, unknown>>;
  error: string | null;
  source: string;
+  collection?: Record<string, unknown>;
 }

 function record(value: unknown): Record<string, unknown> | null {
@@ -24,31 +28,17 @@ function records(value: unknown): Array<Record<string, unknown>> {
  return Array.isArray(value) ? value.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
 }

-function nested(root: unknown, ...keys: string[]): unknown {
-  let value = root;
-  for (const key of keys) value = record(value)?.[key];
-  return value;
-}
-
-async function pooled<T, R>(values: T[], concurrency: number, operation: (value: T) => Promise<R>): Promise<R[]> {
-  const results = new Array<R>(values.length);
-  let cursor = 0;
-  await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, async () => {
-    for (;;) {
-      const index = cursor++;
-      if (index >= values.length) return;
-      results[index] = await operation(values[index]!);
-    }
-  }));
-  return results;
-}
-
 export class AccountScoreService {
  private timer: ReturnType<typeof setInterval> | null = null;
  private inFlight: Promise<ScoreSnapshot> | null = null;
  private snapshot: ScoreSnapshot;

-  constructor(private readonly config: AppConfig, private readonly cachePath: string, private readonly cliWorkDir: string) {
+  constructor(
+    private readonly config: AppConfig,
+    private readonly cachePath: string,
+    private readonly sub2api: Sub2ApiClient,
+    private readonly policyEvents: RuntimePolicyEventSource,
+  ) {
    this.snapshot = this.readCache();
  }

@@ -86,24 +76,9 @@ export class AccountScoreService {
    const startedAt = new Date();
    this.snapshot = { ...this.snapshot, status: "refreshing", refreshStartedAt: startedAt.toISOString(), error: null };
    try {
-      const overview = await this.invoke(["--all-groups"]);
-      const groups = records(nested(overview, "data", "parsed", "allGroups", "groups"));
-      const details = await pooled(groups, 2, async (group) => {
-        const id = String(group.groupId ?? "");
-        if (!id) return { group, accounts: [] as Array<Record<string, unknown>> };
-        const payload = await this.invoke(["--group", id]);
-        return {
-          group,
-          accounts: records(nested(payload, "data", "parsed", "errors", "nativeOps", "accountQuality", "accounts")),
-        };
-      });
+      const collected = await collectNativeScores(this.sub2api, this.policyEvents, this.config.monitor.scoreWindow);
      const refreshedAt = new Date();
-      const accounts = mergeAccountScores(details.flatMap(({ group, accounts }) => accounts.map((account): Record<string, unknown> => ({
-        groupId: group.groupId ?? null,
-        groupName: group.groupName ?? null,
-        platform: group.platform ?? null,
-        ...account,
-      }))));
+      const accounts = mergeAccountScores(collected.accounts);
      this.snapshot = {
        ok: true,
        status: "ready",
@@ -111,10 +86,11 @@ export class AccountScoreService {
        refreshStartedAt: startedAt.toISOString(),
        nextRefreshAt: new Date(refreshedAt.getTime() + this.config.monitor.refreshIntervalMinutes * 60_000).toISOString(),
        window: this.config.monitor.scoreWindow,
-        groups,
+        groups: collected.groups,
        accounts,
        error: null,
-        source: "unidesk-sub2api-runtime-errors",
+        source: "sub2api-native-admin-api-local-aggregation",
+        collection: collected.collection,
      };
      this.writeCache(this.snapshot);
      return this.snapshot;
@@ -131,50 +107,6 @@ export class AccountScoreService {
    }
  }

-  private async invoke(scope: string[]): Promise<Record<string, unknown>> {
-    const args = [
-      resolve(this.cliWorkDir, this.config.monitor.cli.entrypoint),
-      "platform-infra", "sub2api", "codex-pool", "runtime", "errors",
-      "--target", this.config.monitor.target,
-      "--since", this.config.monitor.scoreWindow,
-      ...scope,
-      "--raw",
-    ];
-    const process = Bun.spawn([this.config.monitor.cli.executable, ...args], {
-      cwd: this.cliWorkDir,
-      stdout: "pipe",
-      stderr: "pipe",
-      env: { ...Bun.env, UNIDESK_MAIN_SERVER_IP: this.config.monitor.cli.mainServerHost },
-    });
-    let timedOut = false;
-    const timeout = setTimeout(() => {
-      timedOut = true;
-      process.kill();
-    }, this.config.monitor.cli.timeoutMs);
-    const [stdout, stderr, exitCode] = await Promise.all([
-      new Response(process.stdout).text(),
-      new Response(process.stderr).text(),
-      process.exited,
-    ]).finally(() => clearTimeout(timeout));
-    if (timedOut) throw new Error(`评分 CLI 在 ${this.config.monitor.cli.timeoutMs}ms 后超时`);
-    if (exitCode !== 0) throw new Error(`评分 CLI 退出码 ${exitCode}: ${stderr.trim().slice(-600)}`);
-    let payload = record(JSON.parse(stdout));
-    if (!payload || payload.ok !== true) throw new Error("评分 CLI 返回无效结果");
-    if (nested(payload, "data", "outputTruncated")) {
-      const dumpPath = nested(payload, "data", "dump", "path");
-      if (typeof dumpPath !== "string" || !dumpPath.startsWith("/tmp/unidesk-cli-output/")) {
-        throw new Error("评分 CLI 渐进披露结果缺少受保护 dump");
-      }
-      try {
-        payload = record(JSON.parse(readFileSync(dumpPath, "utf8")));
-      } finally {
-        try { unlinkSync(dumpPath); } catch { /* The CLI may clean up its own temporary output. */ }
-      }
-      if (!payload || payload.ok !== true) throw new Error("评分 CLI dump 返回无效结果");
-    }
-    return payload;
-  }
-
  private readCache(): ScoreSnapshot {
    if (existsSync(this.cachePath)) {
      try {
@@ -194,7 +126,7 @@ export class AccountScoreService {
      groups: [],
      accounts: [],
      error: null,
-      source: "unidesk-sub2api-runtime-errors",
+      source: "sub2api-native-admin-api-local-aggregation",
    };
  }

@@ -33,9 +33,6 @@ export class AdminHttpClient {
  creditTest(execute: boolean): Promise<Record<string, unknown>> { return this.request("/api/admin/credit-test", { method: "POST", body: JSON.stringify({ execute }) }); }
  serviceStatus(): Promise<Record<string, unknown>> { return this.request("/api/status"); }
  scores(): Promise<Record<string, unknown>> { return this.request("/api/scores"); }
-  refreshScores(): Promise<Record<string, unknown>> {
-    return this.request("/api/scores/refresh", { method: "POST", body: "{}" }, this.config.monitor.cli.timeoutMs + 5_000);
-  }
  ranking(): Promise<Record<string, unknown>> { return this.request("/api/ranking"); }
  lottery(): Promise<Record<string, unknown>> { return this.request("/api/lottery"); }
  workflowSubmit(command: Record<string, unknown>): Promise<Record<string, unknown>> {
@@ -13,7 +13,7 @@ const adminToken = process.env[target.adminTokenEnv];
 if (!adminToken) throw new Error(`server target requires env ${target.adminTokenEnv}`);

 const context = createServerContext(config, target);
-const temporal = await TemporalGateway.connect(config);
+const temporal = await TemporalGateway.connect(config, { taskQueue: target.temporalTaskQueue, scoreScheduleWorkflowId: target.scoreScheduleWorkflowId });
 const dispatcher = new ApplicationDispatcher({ lottery: context.service, scores: context.monitor }, temporal);
 const server = Bun.serve({
  hostname: target.listenHost,
@@ -27,7 +27,7 @@ console.log(JSON.stringify({
  runtime: runtimeId,
  listen: server.url.toString(),
  temporalNamespace: config.temporal.namespace,
-  temporalTaskQueue: config.temporal.taskQueue,
+  temporalTaskQueue: target.temporalTaskQueue,
  automaticCreditEnabled: config.lottery.automaticCredit.enabled,
  valuesPrinted: false,
 }));
@@ -6,6 +6,7 @@ import { LotteryStore } from "./store";
 import { Sub2ApiClient } from "./sub2api-client";
 import { AccountScoreService } from "./account-score-service";
 import type { WebAuthSecrets } from "./web-auth";
+import { UniDeskRuntimePolicyEventSource } from "./runtime-policy-events";

 export interface AppContext {
  service: LotteryService;
@@ -18,7 +19,7 @@ export interface AppContext {
 export function createEmbeddedContext(config: AppConfig, target: EmbeddedCliTarget): AppContext {
  const store = new LotteryStore(config, resolveDataPath(config, target.databasePath));
  const client = new Sub2ApiClient(config, readSub2ApiCredentials(config));
-  const monitor = new AccountScoreService(config, resolveDataPath(config, target.scoreCachePath), target.monitorWorkDir);
+  const monitor = new AccountScoreService(config, resolveDataPath(config, target.scoreCachePath), client, new UniDeskRuntimePolicyEventSource(config, target.monitorWorkDir));
  return {
    service: new LotteryService(config, store, client),
    store,
@@ -39,7 +40,7 @@ export function createServerContext(config: AppConfig, target: ServerTarget): Ap
  }
  const store = new LotteryStore(config, resolveDataPath(config, target.databasePath));
  const client = new Sub2ApiClient(config, { email, password });
-  const monitor = new AccountScoreService(config, resolveDataPath(config, target.scoreCachePath), target.monitorWorkDir);
+  const monitor = new AccountScoreService(config, resolveDataPath(config, target.scoreCachePath), client, new UniDeskRuntimePolicyEventSource(config, target.monitorWorkDir));
  return {
    service: new LotteryService(config, store, client),
    store,
@@ -81,6 +81,7 @@ export interface EmbeddedCliTarget {
  databasePath: string;
  scoreCachePath: string;
  monitorWorkDir: string;
+  temporalTaskQueue: string;
 }

 export interface HttpCliTarget {
@@ -101,6 +102,8 @@ export interface ServerTarget {
  databasePath: string;
  scoreCachePath: string;
  monitorWorkDir: string;
+  temporalTaskQueue: string;
+  scoreScheduleWorkflowId: string;
  adminTokenEnv: string;
  sub2apiAdminEmailEnv: string;
  sub2apiAdminPasswordEnv: string;
@@ -214,6 +217,7 @@ export function loadConfig(path: string): AppConfig {
      databasePath: stringValue(target, "databasePath", `runtime.cliTargets.${id}`),
      scoreCachePath: stringValue(target, "scoreCachePath", `runtime.cliTargets.${id}`),
      monitorWorkDir: stringValue(target, "monitorWorkDir", `runtime.cliTargets.${id}`),
+      temporalTaskQueue: stringValue(target, "temporalTaskQueue", `runtime.cliTargets.${id}`),
    };
    else if (mode === "http") cliTargets[id] = { mode, baseUrl: stringValue(target, "baseUrl", `runtime.cliTargets.${id}`), adminToken: secretRef(target.adminToken, `runtime.cliTargets.${id}.adminToken`) };
    else throw new Error(`runtime.cliTargets.${id}.mode must be embedded or http`);
@@ -233,6 +237,8 @@ export function loadConfig(path: string): AppConfig {
      databasePath: stringValue(target, "databasePath", `runtime.serverTargets.${id}`),
      scoreCachePath: stringValue(target, "scoreCachePath", `runtime.serverTargets.${id}`),
      monitorWorkDir: stringValue(target, "monitorWorkDir", `runtime.serverTargets.${id}`),
+      temporalTaskQueue: stringValue(target, "temporalTaskQueue", `runtime.serverTargets.${id}`),
+      scoreScheduleWorkflowId: stringValue(target, "scoreScheduleWorkflowId", `runtime.serverTargets.${id}`),
      adminTokenEnv: stringValue(target, "adminTokenEnv", `runtime.serverTargets.${id}`),
      sub2apiAdminEmailEnv: stringValue(target, "sub2apiAdminEmailEnv", `runtime.serverTargets.${id}`),
      sub2apiAdminPasswordEnv: stringValue(target, "sub2apiAdminPasswordEnv", `runtime.serverTargets.${id}`),
@@ -0,0 +1,111 @@
+import { existsSync, readFileSync, unlinkSync } from "node:fs";
+import { resolve } from "node:path";
+import type { AppConfig } from "./config";
+import type { Sub2ApiSystemLog } from "./sub2api-client";
+
+type Row = Record<string, unknown>;
+
+export interface RuntimePolicyEventBatch {
+  events: Sub2ApiSystemLog[];
+  evidence: Row;
+}
+
+export interface RuntimePolicyEventSource {
+  collect(window: string): Promise<RuntimePolicyEventBatch>;
+}
+
+function record(value: unknown): Row | null {
+  return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : null;
+}
+
+function numberValue(value: unknown): number | null {
+  const parsed = Number(value);
+  return value !== null && value !== undefined && Number.isFinite(parsed) ? parsed : null;
+}
+
+function text(value: unknown): string | null {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function readCliPayload(stdout: string): Row {
+  let payload = record(JSON.parse(stdout));
+  if (!payload) throw new Error("runtime events CLI returned an invalid payload");
+  const outerData = record(payload.data);
+  if (outerData?.outputTruncated === true) {
+    const dumpPath = text(record(outerData.dump)?.path);
+    if (!dumpPath || !dumpPath.startsWith("/tmp/unidesk-cli-output/") || !existsSync(dumpPath)) {
+      throw new Error("runtime events CLI did not expose its protected raw result");
+    }
+    try {
+      payload = record(JSON.parse(readFileSync(dumpPath, "utf8")));
+    } finally {
+      try { unlinkSync(dumpPath); } catch { /* The CLI may clean up its own result. */ }
+    }
+    if (!payload) throw new Error("runtime events CLI raw result is invalid");
+  }
+  const response = record(payload.data);
+  const parsed = record(response?.parsed);
+  if (payload.ok !== true || response?.ok !== true || parsed?.ok !== true) {
+    throw new Error(`runtime events CLI failed: ${text(parsed?.error) ?? "unknown error"}`);
+  }
+  return parsed;
+}
+
+export class UniDeskRuntimePolicyEventSource implements RuntimePolicyEventSource {
+  constructor(private readonly config: AppConfig, private readonly workDir: string) {}
+
+  async collect(window: string): Promise<RuntimePolicyEventBatch> {
+    const args = [
+      resolve(this.workDir, this.config.monitor.cli.entrypoint),
+      "platform-infra", "sub2api", "codex-pool", "runtime", "events",
+      "--target", this.config.monitor.target,
+      "--since", window,
+      "--raw",
+    ];
+    const child = Bun.spawn([this.config.monitor.cli.executable, ...args], {
+      cwd: this.workDir,
+      stdout: "pipe",
+      stderr: "pipe",
+      env: { ...Bun.env, UNIDESK_MAIN_SERVER_IP: this.config.monitor.cli.mainServerHost },
+    });
+    let timedOut = false;
+    const timeout = setTimeout(() => {
+      timedOut = true;
+      child.kill();
+    }, this.config.monitor.cli.timeoutMs);
+    const [stdout, stderr, exitCode] = await Promise.all([
+      new Response(child.stdout).text(),
+      new Response(child.stderr).text(),
+      child.exited,
+    ]).finally(() => clearTimeout(timeout));
+    if (timedOut) throw new Error(`runtime events CLI timed out after ${this.config.monitor.cli.timeoutMs}ms`);
+    if (exitCode !== 0) throw new Error(`runtime events CLI exited ${exitCode}: ${stderr.trim().slice(-600)}`);
+    const payload = readCliPayload(stdout);
+    const rows = Array.isArray(payload.events) ? payload.events.map(record).filter((item): item is Row => item !== null) : [];
+    const events = rows.map((row, index): Sub2ApiSystemLog => ({
+      id: index + 1,
+      created_at: text(row.createdAt) ?? "",
+      message: text(row.marker) ?? "",
+      request_id: text(row.requestId) ?? undefined,
+      account_id: numberValue(row.accountId),
+      extra: {
+        group_id: numberValue(row.groupId),
+        status_code: numberValue(row.statusCode),
+        upstream_status: numberValue(row.upstreamStatus),
+        path: text(row.path),
+      },
+    }));
+    return {
+      events,
+      evidence: {
+        source: payload.source,
+        window: payload.window,
+        tail: payload.tail,
+        scannedLineCount: payload.scannedLineCount,
+        policyEventCount: payload.policyEventCount,
+        completionEventCount: payload.completionEventCount,
+        eventCount: payload.eventCount,
+      },
+    };
+  }
+}
@@ -7,7 +7,7 @@ interface Envelope<T> {
  data: T;
 }

-interface Paginated<T> {
+export interface Paginated<T> {
  items: T[];
  total: number;
  page: number;
@@ -15,6 +15,56 @@ interface Paginated<T> {
  pages: number;
 }

+export interface Sub2ApiGroup {
+  id: number;
+  name: string;
+  platform: string;
+  status: string;
+}
+
+export interface Sub2ApiAccount {
+  id: number;
+  name: string;
+  platform: string;
+  status: string;
+  schedulable?: boolean;
+  priority?: number;
+}
+
+export interface Sub2ApiUsageRow {
+  id: number;
+  account_id: number | null;
+  group_id: number | null;
+  model: string;
+  stream: boolean;
+  input_tokens: number;
+  output_tokens: number;
+  actual_cost: number;
+  duration_ms: number | null;
+  first_token_ms: number | null;
+  created_at: string;
+}
+
+export interface Sub2ApiRequestError {
+  id: number;
+  request_id?: string;
+  account_id?: number | null;
+  status_code?: number;
+  phase?: string;
+  type?: string;
+  message?: string;
+  error_message?: string;
+}
+
+export interface Sub2ApiSystemLog {
+  id: number;
+  created_at: string;
+  message: string;
+  request_id?: string;
+  account_id?: number | null;
+  extra?: Record<string, unknown>;
+}
+
 export class Sub2ApiClient {
  private token: string | null = null;
  private tokenExpiresAt = 0;
@@ -85,6 +135,85 @@ export class Sub2ApiClient {
    return await this.request(`/admin/dashboard/users-ranking?${params}`);
  }

+  async listGroups(): Promise<Sub2ApiGroup[]> {
+    return await this.request<Sub2ApiGroup[]>("/admin/groups/all");
+  }
+
+  async listGroupAccounts(groupId: number, platform: string): Promise<Sub2ApiAccount[]> {
+    const params = new URLSearchParams({
+      page: "1",
+      page_size: "1000",
+      group: String(groupId),
+      platform,
+      sort_by: "id",
+      sort_order: "asc",
+    });
+    return (await this.request<Paginated<Sub2ApiAccount>>(`/admin/accounts?${params}`)).items;
+  }
+
+  async listGroupUsage(groupId: number, start: Date, end: Date): Promise<Sub2ApiUsageRow[]> {
+    return await this.paginate<Sub2ApiUsageRow>("/admin/usage", {
+      group_id: String(groupId),
+      start_date: start.toISOString().slice(0, 10),
+      end_date: end.toISOString().slice(0, 10),
+      timezone: "UTC",
+      sort_by: "created_at",
+      sort_order: "desc",
+    }, 1000, (row) => {
+      const createdAt = Date.parse(row.created_at);
+      return Number.isFinite(createdAt) && createdAt >= start.getTime() && createdAt <= end.getTime();
+    });
+  }
+
+  async listRequestErrors(groupId: number, platform: string, start: Date): Promise<Sub2ApiRequestError[]> {
+    return await this.paginate<Sub2ApiRequestError>("/admin/ops/request-errors", {
+      group_id: String(groupId),
+      platform,
+      start_time: start.toISOString(),
+      view: "all",
+    }, 500);
+  }
+
+  async listSystemLogs(platform: string, start: Date, marker: string): Promise<Sub2ApiSystemLog[]> {
+    return await this.paginate<Sub2ApiSystemLog>("/admin/ops/system-logs", {
+      platform,
+      start_time: start.toISOString(),
+      q: marker,
+    }, 200);
+  }
+
+  async getOpsOverview(groupId: number, platform: string, start: Date): Promise<Record<string, unknown>> {
+    const params = new URLSearchParams({
+      group_id: String(groupId),
+      platform,
+      start_time: start.toISOString(),
+      query_mode: "raw",
+    });
+    return await this.request(`/admin/ops/dashboard/overview?${params}`);
+  }
+
+  async getOpsAccountAvailability(groupId: number, platform: string): Promise<Record<string, unknown>> {
+    const params = new URLSearchParams({ group_id: String(groupId), platform });
+    return await this.request(`/admin/ops/account-availability?${params}`);
+  }
+
+  async getOpsConcurrency(groupId: number, platform: string): Promise<Record<string, unknown>> {
+    const params = new URLSearchParams({ group_id: String(groupId), platform });
+    return await this.request(`/admin/ops/concurrency?${params}`);
+  }
+
+  private async paginate<T>(path: string, query: Record<string, string>, pageSize: number, keep: (row: T) => boolean = () => true): Promise<T[]> {
+    const rows: T[] = [];
+    let page = 1;
+    for (;;) {
+      const params = new URLSearchParams({ ...query, page: String(page), page_size: String(pageSize) });
+      const data = await this.request<Paginated<T>>(`${path}?${params}`);
+      rows.push(...data.items.filter(keep));
+      if (page >= data.pages) return rows;
+      page += 1;
+    }
+  }
+
  async addBalance(userId: number, amountUsd: number, notes: string): Promise<Sub2ApiUser> {
    return await this.request(`/admin/users/${userId}/balance`, {
      method: "POST",
@@ -14,11 +14,15 @@ export class TemporalGateway {
    private readonly connection: Connection,
    private readonly client: Client,
    private readonly config: AppConfig,
+    private readonly runtime: { taskQueue: string; scoreScheduleWorkflowId: string },
  ) {}

-  static async connect(config: AppConfig): Promise<TemporalGateway> {
+  static async connect(config: AppConfig, runtime: { taskQueue: string; scoreScheduleWorkflowId?: string }): Promise<TemporalGateway> {
    const connection = await Connection.connect({ address: temporalAddress(config) });
-    return new TemporalGateway(connection, new Client({ connection, namespace: config.temporal.namespace }), config);
+    return new TemporalGateway(connection, new Client({ connection, namespace: config.temporal.namespace }), config, {
+      taskQueue: runtime.taskQueue,
+      scoreScheduleWorkflowId: runtime.scoreScheduleWorkflowId ?? config.temporal.scoreScheduleWorkflowId,
+    });
  }

  async execute(command: AppCommand): Promise<unknown> {
@@ -29,7 +33,7 @@ export class TemporalGateway {
  async submit(command: AppCommand): Promise<{ ok: true; workflowId: string; runId: string; state: "submitted" }> {
    const operation: OperationRequest = { operationId: randomUUID(), command };
    const handle = await this.client.workflow.start("operationWorkflow", {
-      taskQueue: this.config.temporal.taskQueue,
+      taskQueue: this.runtime.taskQueue,
      workflowId: `apistate-${command.kind.replaceAll(".", "-")}-${operation.operationId}`,
      workflowExecutionTimeout: this.config.temporal.workflowExecutionTimeout,
      args: [{
@@ -56,10 +60,10 @@ export class TemporalGateway {
  }

  async ensureScoreSchedule(): Promise<{ started: boolean; workflowId: string }> {
-    const workflowId = this.config.temporal.scoreScheduleWorkflowId;
+    const workflowId = this.runtime.scoreScheduleWorkflowId;
    try {
      await this.client.workflow.start("scoreRefreshScheduleWorkflow", {
-        taskQueue: this.config.temporal.taskQueue,
+        taskQueue: this.runtime.taskQueue,
        workflowId,
        args: [{
          intervalMs: this.config.monitor.refreshIntervalMinutes * 60_000,
@@ -16,11 +16,11 @@ const connection = await NativeConnection.connect({ address: temporalAddress(con
 const worker = await Worker.create({
  connection,
  namespace: config.temporal.namespace,
-  taskQueue: config.temporal.taskQueue,
+  taskQueue: target.temporalTaskQueue,
  workflowsPath: fileURLToPath(new URL("./workflows.ts", import.meta.url)),
  activities: createActivities({ lottery: context.service, scores: context.monitor }),
 });
-const temporal = await TemporalGateway.connect(config);
+const temporal = await TemporalGateway.connect(config, { taskQueue: target.temporalTaskQueue, scoreScheduleWorkflowId: target.scoreScheduleWorkflowId });
 const schedule = await temporal.ensureScoreSchedule();
 let state: "ready" | "stopping" = "ready";
 const health = Bun.serve({
@@ -31,7 +31,7 @@ const health = Bun.serve({
    component: "apistate-worker",
    state,
    namespace: config.temporal.namespace,
-    taskQueue: config.temporal.taskQueue,
+    taskQueue: target.temporalTaskQueue,
    schedule,
  }, { status: state === "ready" ? 200 : 503 }),
 });
@@ -42,7 +42,7 @@ console.log(JSON.stringify({
  runtime: runtimeId,
  health: health.url.toString(),
  temporalNamespace: config.temporal.namespace,
-  temporalTaskQueue: config.temporal.taskQueue,
+  temporalTaskQueue: target.temporalTaskQueue,
  schedule,
  valuesPrinted: false,
 }));