fix: 周期回收全部失活 runner

src/mgr/runner-retention.ts

@@ -256,11 +256,10 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore
     const observedAt = nowIso();
     const before = await runnerSnapshot(input.store, input.options);
     const resources = [...before.jobs, ...before.pods];
-    const stalePendingCandidates = resources
-      .filter((item) => !item.protectedActive && item.candidateKind === "stale-pending")
-      .sort(compareCandidates);
+    const inactiveCandidates = resources.filter((item) => !item.protectedActive).sort(compareCandidates);
+    const stalePendingCandidates = inactiveCandidates.filter((item) => item.candidateKind === "stale-pending");
     const protectedActiveRunners = resources.filter((item) => item.protectedActive);
-    const selected = stalePendingCandidates.slice(0, limit);
+    const selected = inactiveCandidates.slice(0, limit);
     const outcomes: JsonRecord[] = [];
     let terminalizedRunCount = 0;
     let terminalizedCommandCount = 0;
@@ -297,6 +296,7 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore
       }
     }
     const candidateSummaries = boundedResourceSummaries(stalePendingCandidates);
+    const inactiveCandidateSummaries = boundedResourceSummaries(inactiveCandidates);
     const protectedSummaries = boundedResourceSummaries(protectedActiveRunners);
     const selectedSummaries = boundedResourceSummaries(selected);
     const boundedOutcomes = boundedJsonRecords(outcomes);
@@ -315,6 +315,8 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore
       nonTerminalRunnerPodCountBefore: before.nonTerminalRunnerPodCount,
       orphanNonTerminalRunnerPodCountBefore: before.orphanNonTerminalRunnerPodCount,
       inactiveCandidateCount: resources.filter((item) => !item.protectedActive).length,
+      inactiveCandidates: inactiveCandidateSummaries.items,
+      inactiveCandidateOmittedCount: inactiveCandidateSummaries.omittedCount,
       stalePendingCandidateCount: stalePendingCandidates.length,
       stalePendingCandidates: candidateSummaries.items,
       stalePendingCandidateOmittedCount: candidateSummaries.omittedCount,
@@ -337,7 +339,7 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore
       postObservationState: postObservationError ? "failed" : "completed",
       postObservationError,
       activeRunRisk: protectedActiveRunners.length > 0,
-      reason: postObservationError ? "stale-pending-post-observation-failed" : selected.length === 0 ? "no-stale-pending-candidate" : failedRunnerCount > 0 ? "stale-pending-reconcile-partial" : "stale-pending-reconciled",
+      reason: postObservationError ? "inactive-runner-post-observation-failed" : selected.length === 0 ? "no-inactive-runner-candidate" : failedRunnerCount > 0 ? "inactive-runner-reconcile-partial" : "inactive-runner-reconciled",
       cleanupPolicy: cleanupPolicySummary(),
       valuesPrinted: false,
     } satisfies RunnerStalePendingReconcileSummary;

src/selftest/cases/22-runner-retention-stale-pending.ts

@@ -138,6 +138,7 @@ const selfTest: SelfTestCase = async (context) => {
     await assertMemoryDbFenceRejectsConcurrentClaim();
     await assertRunnerCreateFenceSerializesNamespace();
     await assertPeriodicReconcilerCleansAtCapacity({ fixturePath, statePath, kubectlCommand });
+    await assertPeriodicReconcilerCleansTerminalRunners({ fixturePath, statePath, kubectlCommand });
     await assertPeriodicReconcilerProtectsUnsafeRunners({ fixturePath, statePath, kubectlCommand });
     await assertPeriodicReconcilerCasConflictFailsClosed({ fixturePath, statePath, kubectlCommand });
     await assertConcurrentPeriodicReconcileIsSerialized({ fixturePath, statePath, kubectlCommand });
@@ -185,6 +186,7 @@ const selfTest: SelfTestCase = async (context) => {
       "memory-db-fence-rejects-concurrent-claim",
       "namespace-runner-create-fence-prevents-capacity-overshoot",
       "periodic-reconciler-cleans-stale-pending-at-capacity",
+      "periodic-reconciler-cleans-terminal-inactive-runners",
       "periodic-reconciler-protects-unsafe-runners",
       "periodic-reconciler-cas-conflict-fails-closed",
       "periodic-reconciler-namespace-fence-prevents-duplicate-terminalization",
@@ -677,7 +679,7 @@ async function assertPeriodicReconcilerCleansAtCapacity(input: { fixturePath: st
   assert.equal(retention.terminalizedCommandCount, 3);
   assert.equal(retention.deletedRunnerJobCount, 3);
   assert.equal(retention.failedRunnerCount, 0);
-  assert.equal(retention.reason, "stale-pending-reconciled");
+  assert.equal(retention.reason, "inactive-runner-reconciled");
   assert.equal(store.terminalizationCount, 3);
   for (const fact of facts.slice(0, 3)) {
     assert.equal(store.getRun(fact.run.id).status, "failed");
@@ -693,6 +695,37 @@ async function assertPeriodicReconcilerCleansAtCapacity(input: { fixturePath: st
   );
 }
 
+async function assertPeriodicReconcilerCleansTerminalRunners(input: { fixturePath: string; statePath: string; kubectlCommand: string }): Promise<void> {
+  const oldAt = new Date(Date.now() - 60 * 60_000).toISOString();
+  const facts = [360, 361].map((index) => {
+    const fact = pendingFact(index, oldAt, "claimed");
+    fact.run.status = "completed";
+    fact.run.terminalStatus = "completed";
+    fact.command.state = "completed";
+    return fact;
+  });
+  await writeFixture(input, {
+    jobs: facts.map((fact, index) => runnerJob(fact, index + 360)),
+    pods: facts.map((fact, index) => runnerPod(fact, index + 360, "running")),
+  });
+  const store = new RetentionFactStore(facts);
+  const summary = await reconcileRunnerJobsOnce({
+    store,
+    namespace: "agentrun-nc01-v02",
+    kubectlCommand: input.kubectlCommand,
+    retention: retentionOptions(input.kubectlCommand, 20),
+    limit: 20,
+  });
+  const retention = summary.stalePendingRetention as JsonRecord;
+  assert.equal(retention.inactiveCandidateCount, 2);
+  assert.equal(retention.stalePendingCandidateCount, 0);
+  assert.equal(retention.selectedRunnerCount, 2);
+  assert.equal(retention.deletedRunnerJobCount, 2);
+  assert.equal(retention.reason, "inactive-runner-reconciled");
+  assert.equal(store.getRun(facts[0]!.run.id).claimedBy, null);
+  assert.equal(store.getRun(facts[1]!.run.id).claimedBy, null);
+}
+
 async function assertPeriodicReconcilerProtectsUnsafeRunners(input: { fixturePath: string; statePath: string; kubectlCommand: string }): Promise<void> {
   const oldAt = new Date(Date.now() - 60 * 60_000).toISOString();
   const freshAt = new Date(Date.now() - 60_000).toISOString();