// 重构前 index.ts 只读参考：commit 6a04144d3f5103014f75b637d7e6bc2f45bf007f，blob 56e590c1a6b5ca7ad128bf2c992f60e46c355a58；可用 `git show 6a04144d3f5103014f75b637d7e6bc2f45bf007f:src/components/microservices/code-queue/src/index.ts` 查看。

import type { ActiveRun } from "./code-agent/common";
import type { CodeQueueExecutionDiagnostics, JsonValue, QueueTask, SchedulerActiveRunHeartbeat } from "./types";

export const schedulerHeartbeatStaleMs = 5 * 60 * 1000;
export const schedulerHeartbeatStaleMsMax = 24 * 60 * 60_000;

interface ExecutionDiagnosticsInput {
  now?: string;
  heartbeatStaleMs?: number;
  controlPlane: string;
  executionStateSource: "scheduler-execution-plane" | "postgres-control-plane";
  tasks: QueueTask[];
  activeRuns?: Iterable<ActiveRun>;
  activeRunSlotCount?: number;
  activeQueueIds?: string[];
  processingQueueIds?: string[];
  orphanedActiveTaskIds?: string[];
  oaPublisher?: JsonValue | null;
}

interface StaleRecoveryInput {
  now?: string;
  heartbeatStaleMs?: number;
  task: QueueTask;
  localActive: boolean;
}

function timestampMs(value: string | null | undefined): number | null {
  if (typeof value !== "string" || value.length === 0) return null;
  const time = Date.parse(value);
  return Number.isFinite(time) ? time : null;
}

function maxTimestamp(values: Array<string | null | undefined>): string | null {
  let best: string | null = null;
  let bestMs = -Infinity;
  for (const value of values) {
    const ms = timestampMs(value);
    if (ms === null || ms < bestMs) continue;
    best = value ?? null;
    bestMs = ms;
  }
  return best;
}

function activeTask(task: QueueTask): boolean {
  return task.status === "running" || task.status === "judging";
}

function taskHeartbeat(task: QueueTask): SchedulerActiveRunHeartbeat | null {
  const heartbeat = task.schedulerHeartbeat;
  if (heartbeat === undefined || heartbeat === null || heartbeat.taskId !== task.id) return null;
  return heartbeat;
}

function heartbeatFresh(heartbeat: SchedulerActiveRunHeartbeat | null, nowMs: number, staleMs: number): boolean {
  const atMs = timestampMs(heartbeat?.lastLocalHeartbeatAt);
  return atMs !== null && nowMs - atMs < staleMs;
}

export function normalizeSchedulerHeartbeatStaleMs(
  value: number | null | undefined,
  options: { min?: number; max?: number } = {},
): number {
  const minimum = Math.max(1, Math.floor(options.min ?? schedulerHeartbeatStaleMs));
  const maximum = Math.max(minimum, Math.floor(options.max ?? Number.MAX_SAFE_INTEGER));
  const parsed = value ?? schedulerHeartbeatStaleMs;
  if (!Number.isFinite(parsed)) return minimum;
  const floored = Math.floor(parsed);
  if (floored < minimum) return minimum;
  return Math.min(maximum, floored);
}

function outputMaxSeq(task: QueueTask): number {
  const values = [
    Number(task.outputMaxSeq ?? 0),
    ...((Array.isArray(task.output) ? task.output : []).map((item) => Number(item.seq ?? 0))),
    ...((Array.isArray(task.attempts) ? task.attempts : []).map((item) => Number(item.outputEndSeq ?? 0))),
  ].filter((value) => Number.isFinite(value) && value >= 0);
  return values.length > 0 ? Math.max(...values.map((value) => Math.floor(value))) : 0;
}

export function staleRecoveryCandidate(input: StaleRecoveryInput): { allowed: boolean; reason: string; heartbeatFresh: boolean; heartbeatAgeMs: number | null } {
  if (!activeTask(input.task)) return { allowed: false, reason: "not-db-active", heartbeatFresh: false, heartbeatAgeMs: null };
  if (input.localActive) return { allowed: false, reason: "scheduler-local-active", heartbeatFresh: true, heartbeatAgeMs: 0 };
  const nowMs = timestampMs(input.now) ?? Date.now();
  const heartbeat = taskHeartbeat(input.task);
  if (heartbeat === null) return { allowed: false, reason: "owner-heartbeat-missing", heartbeatFresh: false, heartbeatAgeMs: null };
  const heartbeatMs = timestampMs(heartbeat.lastLocalHeartbeatAt);
  if (heartbeatMs === null) return { allowed: false, reason: "owner-heartbeat-missing", heartbeatFresh: false, heartbeatAgeMs: null };
  const heartbeatAgeMs = Math.max(0, nowMs - heartbeatMs);
  const staleMs = normalizeSchedulerHeartbeatStaleMs(input.heartbeatStaleMs);
  const fresh = heartbeatAgeMs < staleMs;
  if (fresh) return { allowed: false, reason: "owner-heartbeat-fresh", heartbeatFresh: true, heartbeatAgeMs };
  return { allowed: true, reason: "owner-heartbeat-expired", heartbeatFresh: false, heartbeatAgeMs };
}

export function taskHasTraceGapButFreshHeartbeat(task: QueueTask, now: string, heartbeatStaleMs = schedulerHeartbeatStaleMs): boolean {
  if (!activeTask(task)) return false;
  const heartbeat = taskHeartbeat(task);
  const staleMs = normalizeSchedulerHeartbeatStaleMs(heartbeatStaleMs);
  if (!heartbeatFresh(heartbeat, timestampMs(now) ?? Date.now(), staleMs)) return false;
  const traceAt = timestampMs(heartbeat?.lastPersistedTraceAt);
  if (traceAt === null) return false;
  return (timestampMs(now) ?? Date.now()) - traceAt >= staleMs;
}

export function buildSchedulerHeartbeat(task: QueueTask, run: ActiveRun | null, options: {
  now: string;
  owner: string;
  schedulerInstance: string;
  agentPort: string;
  lastObservedAgentEventAt?: string | null;
  lastPersistedTraceAt?: string | null;
}): SchedulerActiveRunHeartbeat {
  const previous = taskHeartbeat(task);
  return {
    taskId: task.id,
    queueId: task.queueId,
    attempt: Number.isInteger(task.currentAttempt) && task.currentAttempt > 0 ? task.currentAttempt : 0,
    activeTurnId: run?.turnId ?? task.activeTurnId ?? null,
    codexThreadId: run?.threadId ?? task.codexThreadId ?? null,
    owner: options.owner,
    schedulerInstance: options.schedulerInstance,
    executionPlane: "scheduler-execution-plane",
    agentPort: run?.port ?? options.agentPort,
    status: task.status,
    lastLocalHeartbeatAt: options.now,
    lastObservedAgentEventAt: options.lastObservedAgentEventAt ?? previous?.lastObservedAgentEventAt ?? null,
    lastPersistedTraceAt: options.lastPersistedTraceAt ?? previous?.lastPersistedTraceAt ?? null,
    outputMaxSeq: outputMaxSeq(task),
    source: "scheduler",
  };
}

export function buildExecutionDiagnostics(input: ExecutionDiagnosticsInput): CodeQueueExecutionDiagnostics {
  const now = input.now ?? new Date().toISOString();
  const nowMs = timestampMs(now) ?? Date.now();
  const staleMs = normalizeSchedulerHeartbeatStaleMs(input.heartbeatStaleMs);
  const databaseActiveTaskIds = input.tasks.filter(activeTask).map((task) => task.id).sort();
  const activeRunRows = Array.from(input.activeRuns ?? []);
  const schedulerActiveTaskIds = Array.from(new Set(activeRunRows.map((run) => run.taskId).filter(Boolean))).sort();
  const schedulerActiveTaskSet = new Set(schedulerActiveTaskIds);
  const activeHeartbeats = input.tasks
    .map((task) => taskHeartbeat(task))
    .filter((heartbeat): heartbeat is SchedulerActiveRunHeartbeat => heartbeat !== null);
  const heartbeatFreshTaskIds = activeHeartbeats
    .filter((heartbeat) => heartbeatFresh(heartbeat, nowMs, staleMs))
    .map((heartbeat) => heartbeat.taskId)
    .sort();
  const heartbeatFreshSet = new Set(heartbeatFreshTaskIds);
  const heartbeatExpiredTaskIds = activeHeartbeats
    .filter((heartbeat) => {
      const heartbeatTask = input.tasks.find((task) => task.id === heartbeat.taskId);
      return heartbeatTask !== undefined && activeTask(heartbeatTask) && !heartbeatFreshSet.has(heartbeat.taskId);
    })
    .map((heartbeat) => heartbeat.taskId)
    .sort();
  const activeHeartbeatTaskIds = activeHeartbeats.map((heartbeat) => heartbeat.taskId).sort();
  const activeHeartbeatSet = new Set(activeHeartbeatTaskIds);
  const heartbeatMissingTaskIds = databaseActiveTaskIds.filter((taskId) => !activeHeartbeatSet.has(taskId));
  const staleRecoveryCandidateTaskIds = input.tasks
    .filter((task) => staleRecoveryCandidate({ task, now, heartbeatStaleMs: staleMs, localActive: schedulerActiveTaskSet.has(task.id) }).allowed)
    .map((task) => task.id)
    .sort();
  const traceGapTaskIds = activeHeartbeats
    .filter((heartbeat) => {
      const traceAt = timestampMs(heartbeat.lastPersistedTraceAt);
      return traceAt !== null && nowMs - traceAt >= staleMs;
    })
    .map((heartbeat) => heartbeat.taskId)
    .sort();
  const traceGapNotStaleTaskIds = traceGapTaskIds.filter((taskId) => heartbeatFreshSet.has(taskId));
  const schedulerLocalMissingIds = databaseActiveTaskIds.filter((taskId) => !schedulerActiveTaskSet.has(taskId));
  const heartbeatRiskTaskIds = Array.from(new Set([
    ...heartbeatExpiredTaskIds,
    ...heartbeatMissingTaskIds,
    ...staleRecoveryCandidateTaskIds,
  ])).sort();
  const splitBrain = input.executionStateSource === "postgres-control-plane"
    && databaseActiveTaskIds.length > 0
    && schedulerActiveTaskIds.length === 0
    && heartbeatFreshTaskIds.length > 0;
  const splitBrainLive = splitBrain && heartbeatRiskTaskIds.length === 0;
  const staleActive = staleRecoveryCandidateTaskIds.length > 0;
  const degraded = splitBrain
    || staleActive
    || heartbeatMissingTaskIds.length > 0
    || heartbeatExpiredTaskIds.length > 0
    || schedulerLocalMissingIds.length > 0
    || traceGapTaskIds.length > 0
    || (typeof input.oaPublisher === "object" && input.oaPublisher !== null && !Array.isArray(input.oaPublisher) && (Number((input.oaPublisher as Record<string, JsonValue>).pending ?? 0) > 0 || (input.oaPublisher as Record<string, JsonValue>).lastError !== null));
  const state = splitBrain ? "split-brain" : staleActive ? "stale-active" : degraded ? "degraded" : "healthy";
  const effectiveLiveness = heartbeatRiskTaskIds.length > 0
    ? "at-risk"
    : splitBrainLive
      ? "live"
      : degraded
        ? "degraded"
        : "healthy";
  const recommendedAction = heartbeatRiskTaskIds.length > 0
    ? "investigate-heartbeat-risk"
    : splitBrainLive
      ? "continue-supervision"
      : degraded
        ? "observe-degraded"
        : "none";
  const livenessSummary = heartbeatRiskTaskIds.length > 0
    ? "Heartbeat is expired, missing, or stale-recovery eligible for at least one database-active task; investigate before assuming the task is still live."
    : splitBrainLive
      ? "PostgreSQL and the local control-plane view are split, but scheduler-owned heartbeat is fresh; treat the task as live and continue supervision."
      : degraded
        ? "Execution diagnostics are degraded; inspect the listed reasons together with heartbeat and trace progress."
        : "Execution diagnostics are healthy.";
  const lastSchedulerHeartbeatAt = maxTimestamp(activeHeartbeats.map((heartbeat) => heartbeat.lastLocalHeartbeatAt));
  const lastObservedAgentEventAt = maxTimestamp(activeHeartbeats.map((heartbeat) => heartbeat.lastObservedAgentEventAt));
  const lastPersistedTraceAt = maxTimestamp(activeHeartbeats.map((heartbeat) => heartbeat.lastPersistedTraceAt));
  const reasons: string[] = [];
  if (splitBrain) reasons.push("postgres control-plane has database-active tasks while its local active slots are empty, but scheduler heartbeat is fresh");
  if (staleActive) reasons.push("owner heartbeat is expired and scheduler has no local active run for at least one database-active task");
  if (heartbeatMissingTaskIds.length > 0) reasons.push("database-active tasks are missing scheduler-owned heartbeat");
  if (traceGapNotStaleTaskIds.length > 0) reasons.push("trace progress is stale while scheduler heartbeat is fresh; this is a trace gap, not stale active");
  const guidance = [
    "Do not use master activeRunSlotCount=0 alone to declare a task stuck.",
    "Check PostgreSQL task status, scheduler active runs/slots, scheduler heartbeat, and trace/OA publisher progress together.",
    "Trace gap with a fresh scheduler heartbeat must not trigger stale retry.",
    "Stale recovery is allowed only when the scheduler has no local active run and the owner heartbeat is expired.",
  ];
  return {
    state,
    health: state,
    degraded,
    splitBrain,
    splitBrainLive,
    effectiveLiveness,
    recommendedAction,
    livenessSummary,
    executionStateSource: input.executionStateSource,
    controlPlane: input.controlPlane,
    databaseActiveTaskIds,
    databaseActiveTaskCount: databaseActiveTaskIds.length,
    schedulerActiveTaskIds,
    schedulerActiveTaskCount: schedulerActiveTaskIds.length,
    schedulerActiveRunSlotCount: input.activeRunSlotCount ?? schedulerActiveTaskIds.length,
    schedulerActiveQueueIds: (input.activeQueueIds ?? activeRunRows.map((run) => run.queueId)).slice().sort(),
    schedulerProcessingQueueIds: (input.processingQueueIds ?? []).slice().sort(),
    schedulerOrphanedActiveTaskIds: (input.orphanedActiveTaskIds ?? schedulerLocalMissingIds).slice().sort(),
    schedulerOrphanedActiveTaskCount: (input.orphanedActiveTaskIds ?? schedulerLocalMissingIds).length,
    activeHeartbeatTaskIds,
    activeHeartbeatCount: activeHeartbeatTaskIds.length,
    heartbeatFreshTaskIds,
    heartbeatExpiredTaskIds,
    heartbeatMissingTaskIds,
    staleRecoveryCandidateTaskIds,
    heartbeatRiskTaskIds,
    traceGapTaskIds,
    traceGapNotStaleTaskIds,
    schedulerHeartbeatStaleMs: staleMs,
    now,
    lastSchedulerHeartbeatAt,
    lastObservedAgentEventAt,
    lastPersistedTraceAt,
    oaPublisher: input.oaPublisher ?? null,
    reasons,
    guidance,
  };
}
