From d42ca43c78519fb7bc1d0f464b7544c3b23058a8 Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 20 Jul 2026 08:08:52 +0200 Subject: [PATCH] fix: terminalize runner jobs without reports --- src/mgr/runner-reconciler.ts | 15 +++++++++++ src/selftest/cases/24-runner-startup-retry.ts | 26 +++++++++++++++++++ .../fixtures/fake-kubectl-runner-startup.mjs | 12 +++++++-- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/mgr/runner-reconciler.ts b/src/mgr/runner-reconciler.ts index f52d85c..b97a734 100644 --- a/src/mgr/runner-reconciler.ts +++ b/src/mgr/runner-reconciler.ts @@ -667,6 +667,21 @@ function startupFailureFromObservation(observation: JsonRecord): StartupFailure retryable: true, }; } + if (phase === "k8s:failed" || phase === "k8s:succeeded") { + const terminalOutboxState = stringValue(recordValue(observation.terminalOutbox)?.state); + if (terminalOutboxState === "recovered" || terminalOutboxState === "command-terminal") return null; + return { + failureDomain: "infrastructure", + component: "runner-terminal-report", + code: phase === "k8s:failed" + ? "runner-job-failed-without-terminal-report" + : "runner-job-succeeded-without-terminal-report", + summary: phase === "k8s:failed" + ? "Runner Job 已失败且未提交可恢复终态" + : "Runner Job 已结束但未提交可恢复终态", + retryable: false, + }; + } if (phase !== "k8s:pending") return null; const waiting = recordValue(recordValue(observation.k8s)?.waiting); const reason = stringValue(waiting?.reason) ?? ""; diff --git a/src/selftest/cases/24-runner-startup-retry.ts b/src/selftest/cases/24-runner-startup-retry.ts index 5fb4ec0..1767b0c 100644 --- a/src/selftest/cases/24-runner-startup-retry.ts +++ b/src/selftest/cases/24-runner-startup-retry.ts @@ -25,6 +25,8 @@ const selfTest: SelfTestCase = async (context) => { await assertTransientRegistryFailureExhausts(context, fixturePath, kubectlCommand); await assertStartupRecoveryIsVisible(context, fixturePath, kubectlCommand); await assertDeterministicFailureDoesNotRetry(context, fixturePath, kubectlCommand); + await assertFailedJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand); + await assertSucceededJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand); } finally { if (previousFixture === undefined) delete process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH; else process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH = previousFixture; @@ -35,6 +37,8 @@ const selfTest: SelfTestCase = async (context) => { "registry-unreachable-emits-bounded-exponential-retry", "startup-recovery-emits-durable-recovered-event", "image-not-found-fails-without-retry", + "failed-job-without-terminal-outbox-fails", + "succeeded-job-without-terminal-outbox-fails", "semantic-failure-events-carry-workbench-fields", ], }; @@ -64,6 +68,28 @@ async function assertTransientRegistryFailureExhausts(context: SelfTestContext, assert.equal(fact.store.getRun(fact.runId).status, "failed"); } +async function assertFailedJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise { + const fact = createRunnerFact("failed-without-outbox"); + await writeFixture(fixturePath, "", "", "failed"); + await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } }); + const events = fact.store.listEvents(fact.runId, 0, 100); + const exhausted = phase(events, "runner-startup-retry-exhausted"); + assert.equal(exhausted.payload.code, "runner-job-failed-without-terminal-report"); + assert.equal(exhausted.payload.retryable, false); + assert.equal(fact.store.getCommand(fact.commandId).state, "failed"); + assert.equal(fact.store.getRun(fact.runId).status, "failed"); +} + +async function assertSucceededJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise { + const fact = createRunnerFact("succeeded-without-outbox"); + await writeFixture(fixturePath, "", "", "succeeded"); + await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } }); + const exhausted = phase(fact.store.listEvents(fact.runId, 0, 100), "runner-startup-retry-exhausted"); + assert.equal(exhausted.payload.code, "runner-job-succeeded-without-terminal-report"); + assert.equal(fact.store.getCommand(fact.commandId).state, "failed"); + assert.equal(fact.store.getRun(fact.runId).status, "failed"); +} + async function assertStartupRecoveryIsVisible(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise { const fact = createRunnerFact("recovered"); await writeFixture(fixturePath, "ErrImagePull", "failed to pull image: i/o timeout"); diff --git a/src/selftest/fixtures/fake-kubectl-runner-startup.mjs b/src/selftest/fixtures/fake-kubectl-runner-startup.mjs index 1645765..7f20bb1 100755 --- a/src/selftest/fixtures/fake-kubectl-runner-startup.mjs +++ b/src/selftest/fixtures/fake-kubectl-runner-startup.mjs @@ -16,6 +16,8 @@ if (args[0] !== "get") { } if (resource === "job") { + const failed = fixture.mode === "failed"; + const succeeded = fixture.mode === "succeeded"; process.stdout.write(JSON.stringify({ kind: "Job", metadata: { @@ -24,13 +26,19 @@ if (resource === "job") { uid: "job-startup-selftest", creationTimestamp: fixture.createdAt, }, - status: { active: 1, startTime: fixture.createdAt }, + status: failed + ? { failed: 1, startTime: fixture.createdAt, conditions: [{ type: "Failed", status: "True", reason: "BackoffLimitExceeded" }] } + : succeeded + ? { succeeded: 1, startTime: fixture.createdAt, conditions: [{ type: "Complete", status: "True" }] } + : { active: 1, startTime: fixture.createdAt }, })); process.exit(0); } if (resource === "pods") { const running = fixture.mode === "running"; + const failed = fixture.mode === "failed"; + const succeeded = fixture.mode === "succeeded"; const waiting = running ? undefined : { reason: fixture.reason, message: fixture.message, @@ -44,7 +52,7 @@ if (resource === "pods") { creationTimestamp: fixture.createdAt, }, status: { - phase: running ? "Running" : "Pending", + phase: running ? "Running" : failed ? "Failed" : succeeded ? "Succeeded" : "Pending", containerStatuses: [{ name: "runner", state: running ? { running: { startedAt: fixture.createdAt } } : { waiting },