From 7e831dca103133e98224fe1778a6e12f3a5689ac Mon Sep 17 00:00:00 2001 From: root Date: Sun, 12 Jul 2026 18:35:30 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=91=A8=E6=9C=9F=E5=9B=9E=E6=94=B6?= =?UTF-8?q?=E5=85=A8=E9=83=A8=E5=A4=B1=E6=B4=BB=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mgr/runner-retention.ts | 12 ++++--- .../22-runner-retention-stale-pending.ts | 35 ++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/mgr/runner-retention.ts b/src/mgr/runner-retention.ts index 7b51016..4ac3a33 100644 --- a/src/mgr/runner-retention.ts +++ b/src/mgr/runner-retention.ts @@ -256,11 +256,10 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore const observedAt = nowIso(); const before = await runnerSnapshot(input.store, input.options); const resources = [...before.jobs, ...before.pods]; - const stalePendingCandidates = resources - .filter((item) => !item.protectedActive && item.candidateKind === "stale-pending") - .sort(compareCandidates); + const inactiveCandidates = resources.filter((item) => !item.protectedActive).sort(compareCandidates); + const stalePendingCandidates = inactiveCandidates.filter((item) => item.candidateKind === "stale-pending"); const protectedActiveRunners = resources.filter((item) => item.protectedActive); - const selected = stalePendingCandidates.slice(0, limit); + const selected = inactiveCandidates.slice(0, limit); const outcomes: JsonRecord[] = []; let terminalizedRunCount = 0; let terminalizedCommandCount = 0; @@ -297,6 +296,7 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore } } const candidateSummaries = boundedResourceSummaries(stalePendingCandidates); + const inactiveCandidateSummaries = boundedResourceSummaries(inactiveCandidates); const protectedSummaries = boundedResourceSummaries(protectedActiveRunners); const selectedSummaries = boundedResourceSummaries(selected); const boundedOutcomes = boundedJsonRecords(outcomes); @@ -315,6 +315,8 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore nonTerminalRunnerPodCountBefore: before.nonTerminalRunnerPodCount, orphanNonTerminalRunnerPodCountBefore: before.orphanNonTerminalRunnerPodCount, inactiveCandidateCount: resources.filter((item) => !item.protectedActive).length, + inactiveCandidates: inactiveCandidateSummaries.items, + inactiveCandidateOmittedCount: inactiveCandidateSummaries.omittedCount, stalePendingCandidateCount: stalePendingCandidates.length, stalePendingCandidates: candidateSummaries.items, stalePendingCandidateOmittedCount: candidateSummaries.omittedCount, @@ -337,7 +339,7 @@ export async function reconcileStalePendingRunners(input: { store: AgentRunStore postObservationState: postObservationError ? "failed" : "completed", postObservationError, activeRunRisk: protectedActiveRunners.length > 0, - reason: postObservationError ? "stale-pending-post-observation-failed" : selected.length === 0 ? "no-stale-pending-candidate" : failedRunnerCount > 0 ? "stale-pending-reconcile-partial" : "stale-pending-reconciled", + reason: postObservationError ? "inactive-runner-post-observation-failed" : selected.length === 0 ? "no-inactive-runner-candidate" : failedRunnerCount > 0 ? "inactive-runner-reconcile-partial" : "inactive-runner-reconciled", cleanupPolicy: cleanupPolicySummary(), valuesPrinted: false, } satisfies RunnerStalePendingReconcileSummary; diff --git a/src/selftest/cases/22-runner-retention-stale-pending.ts b/src/selftest/cases/22-runner-retention-stale-pending.ts index 69f139b..e2c559f 100644 --- a/src/selftest/cases/22-runner-retention-stale-pending.ts +++ b/src/selftest/cases/22-runner-retention-stale-pending.ts @@ -138,6 +138,7 @@ const selfTest: SelfTestCase = async (context) => { await assertMemoryDbFenceRejectsConcurrentClaim(); await assertRunnerCreateFenceSerializesNamespace(); await assertPeriodicReconcilerCleansAtCapacity({ fixturePath, statePath, kubectlCommand }); + await assertPeriodicReconcilerCleansTerminalRunners({ fixturePath, statePath, kubectlCommand }); await assertPeriodicReconcilerProtectsUnsafeRunners({ fixturePath, statePath, kubectlCommand }); await assertPeriodicReconcilerCasConflictFailsClosed({ fixturePath, statePath, kubectlCommand }); await assertConcurrentPeriodicReconcileIsSerialized({ fixturePath, statePath, kubectlCommand }); @@ -185,6 +186,7 @@ const selfTest: SelfTestCase = async (context) => { "memory-db-fence-rejects-concurrent-claim", "namespace-runner-create-fence-prevents-capacity-overshoot", "periodic-reconciler-cleans-stale-pending-at-capacity", + "periodic-reconciler-cleans-terminal-inactive-runners", "periodic-reconciler-protects-unsafe-runners", "periodic-reconciler-cas-conflict-fails-closed", "periodic-reconciler-namespace-fence-prevents-duplicate-terminalization", @@ -677,7 +679,7 @@ async function assertPeriodicReconcilerCleansAtCapacity(input: { fixturePath: st assert.equal(retention.terminalizedCommandCount, 3); assert.equal(retention.deletedRunnerJobCount, 3); assert.equal(retention.failedRunnerCount, 0); - assert.equal(retention.reason, "stale-pending-reconciled"); + assert.equal(retention.reason, "inactive-runner-reconciled"); assert.equal(store.terminalizationCount, 3); for (const fact of facts.slice(0, 3)) { assert.equal(store.getRun(fact.run.id).status, "failed"); @@ -693,6 +695,37 @@ async function assertPeriodicReconcilerCleansAtCapacity(input: { fixturePath: st ); } +async function assertPeriodicReconcilerCleansTerminalRunners(input: { fixturePath: string; statePath: string; kubectlCommand: string }): Promise { + const oldAt = new Date(Date.now() - 60 * 60_000).toISOString(); + const facts = [360, 361].map((index) => { + const fact = pendingFact(index, oldAt, "claimed"); + fact.run.status = "completed"; + fact.run.terminalStatus = "completed"; + fact.command.state = "completed"; + return fact; + }); + await writeFixture(input, { + jobs: facts.map((fact, index) => runnerJob(fact, index + 360)), + pods: facts.map((fact, index) => runnerPod(fact, index + 360, "running")), + }); + const store = new RetentionFactStore(facts); + const summary = await reconcileRunnerJobsOnce({ + store, + namespace: "agentrun-nc01-v02", + kubectlCommand: input.kubectlCommand, + retention: retentionOptions(input.kubectlCommand, 20), + limit: 20, + }); + const retention = summary.stalePendingRetention as JsonRecord; + assert.equal(retention.inactiveCandidateCount, 2); + assert.equal(retention.stalePendingCandidateCount, 0); + assert.equal(retention.selectedRunnerCount, 2); + assert.equal(retention.deletedRunnerJobCount, 2); + assert.equal(retention.reason, "inactive-runner-reconciled"); + assert.equal(store.getRun(facts[0]!.run.id).claimedBy, null); + assert.equal(store.getRun(facts[1]!.run.id).claimedBy, null); +} + async function assertPeriodicReconcilerProtectsUnsafeRunners(input: { fixturePath: string; statePath: string; kubectlCommand: string }): Promise { const oldAt = new Date(Date.now() - 60 * 60_000).toISOString(); const freshAt = new Date(Date.now() - 60_000).toISOString();