fix: terminalize runner jobs without reports
This commit is contained in:
@@ -667,6 +667,21 @@ function startupFailureFromObservation(observation: JsonRecord): StartupFailure
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
if (phase === "k8s:failed" || phase === "k8s:succeeded") {
|
||||
const terminalOutboxState = stringValue(recordValue(observation.terminalOutbox)?.state);
|
||||
if (terminalOutboxState === "recovered" || terminalOutboxState === "command-terminal") return null;
|
||||
return {
|
||||
failureDomain: "infrastructure",
|
||||
component: "runner-terminal-report",
|
||||
code: phase === "k8s:failed"
|
||||
? "runner-job-failed-without-terminal-report"
|
||||
: "runner-job-succeeded-without-terminal-report",
|
||||
summary: phase === "k8s:failed"
|
||||
? "Runner Job 已失败且未提交可恢复终态"
|
||||
: "Runner Job 已结束但未提交可恢复终态",
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
if (phase !== "k8s:pending") return null;
|
||||
const waiting = recordValue(recordValue(observation.k8s)?.waiting);
|
||||
const reason = stringValue(waiting?.reason) ?? "";
|
||||
|
||||
@@ -25,6 +25,8 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
await assertTransientRegistryFailureExhausts(context, fixturePath, kubectlCommand);
|
||||
await assertStartupRecoveryIsVisible(context, fixturePath, kubectlCommand);
|
||||
await assertDeterministicFailureDoesNotRetry(context, fixturePath, kubectlCommand);
|
||||
await assertFailedJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand);
|
||||
await assertSucceededJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand);
|
||||
} finally {
|
||||
if (previousFixture === undefined) delete process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH;
|
||||
else process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH = previousFixture;
|
||||
@@ -35,6 +37,8 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
"registry-unreachable-emits-bounded-exponential-retry",
|
||||
"startup-recovery-emits-durable-recovered-event",
|
||||
"image-not-found-fails-without-retry",
|
||||
"failed-job-without-terminal-outbox-fails",
|
||||
"succeeded-job-without-terminal-outbox-fails",
|
||||
"semantic-failure-events-carry-workbench-fields",
|
||||
],
|
||||
};
|
||||
@@ -64,6 +68,28 @@ async function assertTransientRegistryFailureExhausts(context: SelfTestContext,
|
||||
assert.equal(fact.store.getRun(fact.runId).status, "failed");
|
||||
}
|
||||
|
||||
async function assertFailedJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
|
||||
const fact = createRunnerFact("failed-without-outbox");
|
||||
await writeFixture(fixturePath, "", "", "failed");
|
||||
await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } });
|
||||
const events = fact.store.listEvents(fact.runId, 0, 100);
|
||||
const exhausted = phase(events, "runner-startup-retry-exhausted");
|
||||
assert.equal(exhausted.payload.code, "runner-job-failed-without-terminal-report");
|
||||
assert.equal(exhausted.payload.retryable, false);
|
||||
assert.equal(fact.store.getCommand(fact.commandId).state, "failed");
|
||||
assert.equal(fact.store.getRun(fact.runId).status, "failed");
|
||||
}
|
||||
|
||||
async function assertSucceededJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
|
||||
const fact = createRunnerFact("succeeded-without-outbox");
|
||||
await writeFixture(fixturePath, "", "", "succeeded");
|
||||
await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } });
|
||||
const exhausted = phase(fact.store.listEvents(fact.runId, 0, 100), "runner-startup-retry-exhausted");
|
||||
assert.equal(exhausted.payload.code, "runner-job-succeeded-without-terminal-report");
|
||||
assert.equal(fact.store.getCommand(fact.commandId).state, "failed");
|
||||
assert.equal(fact.store.getRun(fact.runId).status, "failed");
|
||||
}
|
||||
|
||||
async function assertStartupRecoveryIsVisible(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
|
||||
const fact = createRunnerFact("recovered");
|
||||
await writeFixture(fixturePath, "ErrImagePull", "failed to pull image: i/o timeout");
|
||||
|
||||
@@ -16,6 +16,8 @@ if (args[0] !== "get") {
|
||||
}
|
||||
|
||||
if (resource === "job") {
|
||||
const failed = fixture.mode === "failed";
|
||||
const succeeded = fixture.mode === "succeeded";
|
||||
process.stdout.write(JSON.stringify({
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
@@ -24,13 +26,19 @@ if (resource === "job") {
|
||||
uid: "job-startup-selftest",
|
||||
creationTimestamp: fixture.createdAt,
|
||||
},
|
||||
status: { active: 1, startTime: fixture.createdAt },
|
||||
status: failed
|
||||
? { failed: 1, startTime: fixture.createdAt, conditions: [{ type: "Failed", status: "True", reason: "BackoffLimitExceeded" }] }
|
||||
: succeeded
|
||||
? { succeeded: 1, startTime: fixture.createdAt, conditions: [{ type: "Complete", status: "True" }] }
|
||||
: { active: 1, startTime: fixture.createdAt },
|
||||
}));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (resource === "pods") {
|
||||
const running = fixture.mode === "running";
|
||||
const failed = fixture.mode === "failed";
|
||||
const succeeded = fixture.mode === "succeeded";
|
||||
const waiting = running ? undefined : {
|
||||
reason: fixture.reason,
|
||||
message: fixture.message,
|
||||
@@ -44,7 +52,7 @@ if (resource === "pods") {
|
||||
creationTimestamp: fixture.createdAt,
|
||||
},
|
||||
status: {
|
||||
phase: running ? "Running" : "Pending",
|
||||
phase: running ? "Running" : failed ? "Failed" : succeeded ? "Succeeded" : "Pending",
|
||||
containerStatuses: [{
|
||||
name: "runner",
|
||||
state: running ? { running: { startedAt: fixture.createdAt } } : { waiting },
|
||||
|
||||
Reference in New Issue
Block a user