fix: terminalize runner jobs without reports

This commit is contained in:
AgentRun Codex
2026-07-20 08:08:52 +02:00
parent c34fefcedc
commit d42ca43c78
3 changed files with 51 additions and 2 deletions
+15
View File
@@ -667,6 +667,21 @@ function startupFailureFromObservation(observation: JsonRecord): StartupFailure
retryable: true,
};
}
if (phase === "k8s:failed" || phase === "k8s:succeeded") {
const terminalOutboxState = stringValue(recordValue(observation.terminalOutbox)?.state);
if (terminalOutboxState === "recovered" || terminalOutboxState === "command-terminal") return null;
return {
failureDomain: "infrastructure",
component: "runner-terminal-report",
code: phase === "k8s:failed"
? "runner-job-failed-without-terminal-report"
: "runner-job-succeeded-without-terminal-report",
summary: phase === "k8s:failed"
? "Runner Job 已失败且未提交可恢复终态"
: "Runner Job 已结束但未提交可恢复终态",
retryable: false,
};
}
if (phase !== "k8s:pending") return null;
const waiting = recordValue(recordValue(observation.k8s)?.waiting);
const reason = stringValue(waiting?.reason) ?? "";
@@ -25,6 +25,8 @@ const selfTest: SelfTestCase = async (context) => {
await assertTransientRegistryFailureExhausts(context, fixturePath, kubectlCommand);
await assertStartupRecoveryIsVisible(context, fixturePath, kubectlCommand);
await assertDeterministicFailureDoesNotRetry(context, fixturePath, kubectlCommand);
await assertFailedJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand);
await assertSucceededJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand);
} finally {
if (previousFixture === undefined) delete process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH;
else process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH = previousFixture;
@@ -35,6 +37,8 @@ const selfTest: SelfTestCase = async (context) => {
"registry-unreachable-emits-bounded-exponential-retry",
"startup-recovery-emits-durable-recovered-event",
"image-not-found-fails-without-retry",
"failed-job-without-terminal-outbox-fails",
"succeeded-job-without-terminal-outbox-fails",
"semantic-failure-events-carry-workbench-fields",
],
};
@@ -64,6 +68,28 @@ async function assertTransientRegistryFailureExhausts(context: SelfTestContext,
assert.equal(fact.store.getRun(fact.runId).status, "failed");
}
async function assertFailedJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
const fact = createRunnerFact("failed-without-outbox");
await writeFixture(fixturePath, "", "", "failed");
await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } });
const events = fact.store.listEvents(fact.runId, 0, 100);
const exhausted = phase(events, "runner-startup-retry-exhausted");
assert.equal(exhausted.payload.code, "runner-job-failed-without-terminal-report");
assert.equal(exhausted.payload.retryable, false);
assert.equal(fact.store.getCommand(fact.commandId).state, "failed");
assert.equal(fact.store.getRun(fact.runId).status, "failed");
}
async function assertSucceededJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
const fact = createRunnerFact("succeeded-without-outbox");
await writeFixture(fixturePath, "", "", "succeeded");
await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } });
const exhausted = phase(fact.store.listEvents(fact.runId, 0, 100), "runner-startup-retry-exhausted");
assert.equal(exhausted.payload.code, "runner-job-succeeded-without-terminal-report");
assert.equal(fact.store.getCommand(fact.commandId).state, "failed");
assert.equal(fact.store.getRun(fact.runId).status, "failed");
}
async function assertStartupRecoveryIsVisible(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
const fact = createRunnerFact("recovered");
await writeFixture(fixturePath, "ErrImagePull", "failed to pull image: i/o timeout");
@@ -16,6 +16,8 @@ if (args[0] !== "get") {
}
if (resource === "job") {
const failed = fixture.mode === "failed";
const succeeded = fixture.mode === "succeeded";
process.stdout.write(JSON.stringify({
kind: "Job",
metadata: {
@@ -24,13 +26,19 @@ if (resource === "job") {
uid: "job-startup-selftest",
creationTimestamp: fixture.createdAt,
},
status: { active: 1, startTime: fixture.createdAt },
status: failed
? { failed: 1, startTime: fixture.createdAt, conditions: [{ type: "Failed", status: "True", reason: "BackoffLimitExceeded" }] }
: succeeded
? { succeeded: 1, startTime: fixture.createdAt, conditions: [{ type: "Complete", status: "True" }] }
: { active: 1, startTime: fixture.createdAt },
}));
process.exit(0);
}
if (resource === "pods") {
const running = fixture.mode === "running";
const failed = fixture.mode === "failed";
const succeeded = fixture.mode === "succeeded";
const waiting = running ? undefined : {
reason: fixture.reason,
message: fixture.message,
@@ -44,7 +52,7 @@ if (resource === "pods") {
creationTimestamp: fixture.createdAt,
},
status: {
phase: running ? "Running" : "Pending",
phase: running ? "Running" : failed ? "Failed" : succeeded ? "Succeeded" : "Pending",
containerStatuses: [{
name: "runner",
state: running ? { running: { startedAt: fixture.createdAt } } : { waiting },