fix: terminalize runner jobs without reports
This commit is contained in:
@@ -667,6 +667,21 @@ function startupFailureFromObservation(observation: JsonRecord): StartupFailure
|
|||||||
retryable: true,
|
retryable: true,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (phase === "k8s:failed" || phase === "k8s:succeeded") {
|
||||||
|
const terminalOutboxState = stringValue(recordValue(observation.terminalOutbox)?.state);
|
||||||
|
if (terminalOutboxState === "recovered" || terminalOutboxState === "command-terminal") return null;
|
||||||
|
return {
|
||||||
|
failureDomain: "infrastructure",
|
||||||
|
component: "runner-terminal-report",
|
||||||
|
code: phase === "k8s:failed"
|
||||||
|
? "runner-job-failed-without-terminal-report"
|
||||||
|
: "runner-job-succeeded-without-terminal-report",
|
||||||
|
summary: phase === "k8s:failed"
|
||||||
|
? "Runner Job 已失败且未提交可恢复终态"
|
||||||
|
: "Runner Job 已结束但未提交可恢复终态",
|
||||||
|
retryable: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
if (phase !== "k8s:pending") return null;
|
if (phase !== "k8s:pending") return null;
|
||||||
const waiting = recordValue(recordValue(observation.k8s)?.waiting);
|
const waiting = recordValue(recordValue(observation.k8s)?.waiting);
|
||||||
const reason = stringValue(waiting?.reason) ?? "";
|
const reason = stringValue(waiting?.reason) ?? "";
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ const selfTest: SelfTestCase = async (context) => {
|
|||||||
await assertTransientRegistryFailureExhausts(context, fixturePath, kubectlCommand);
|
await assertTransientRegistryFailureExhausts(context, fixturePath, kubectlCommand);
|
||||||
await assertStartupRecoveryIsVisible(context, fixturePath, kubectlCommand);
|
await assertStartupRecoveryIsVisible(context, fixturePath, kubectlCommand);
|
||||||
await assertDeterministicFailureDoesNotRetry(context, fixturePath, kubectlCommand);
|
await assertDeterministicFailureDoesNotRetry(context, fixturePath, kubectlCommand);
|
||||||
|
await assertFailedJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand);
|
||||||
|
await assertSucceededJobWithoutTerminalOutboxFails(context, fixturePath, kubectlCommand);
|
||||||
} finally {
|
} finally {
|
||||||
if (previousFixture === undefined) delete process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH;
|
if (previousFixture === undefined) delete process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH;
|
||||||
else process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH = previousFixture;
|
else process.env.AGENTRUN_SELFTEST_STARTUP_FIXTURE_PATH = previousFixture;
|
||||||
@@ -35,6 +37,8 @@ const selfTest: SelfTestCase = async (context) => {
|
|||||||
"registry-unreachable-emits-bounded-exponential-retry",
|
"registry-unreachable-emits-bounded-exponential-retry",
|
||||||
"startup-recovery-emits-durable-recovered-event",
|
"startup-recovery-emits-durable-recovered-event",
|
||||||
"image-not-found-fails-without-retry",
|
"image-not-found-fails-without-retry",
|
||||||
|
"failed-job-without-terminal-outbox-fails",
|
||||||
|
"succeeded-job-without-terminal-outbox-fails",
|
||||||
"semantic-failure-events-carry-workbench-fields",
|
"semantic-failure-events-carry-workbench-fields",
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
@@ -64,6 +68,28 @@ async function assertTransientRegistryFailureExhausts(context: SelfTestContext,
|
|||||||
assert.equal(fact.store.getRun(fact.runId).status, "failed");
|
assert.equal(fact.store.getRun(fact.runId).status, "failed");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function assertFailedJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
|
||||||
|
const fact = createRunnerFact("failed-without-outbox");
|
||||||
|
await writeFixture(fixturePath, "", "", "failed");
|
||||||
|
await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } });
|
||||||
|
const events = fact.store.listEvents(fact.runId, 0, 100);
|
||||||
|
const exhausted = phase(events, "runner-startup-retry-exhausted");
|
||||||
|
assert.equal(exhausted.payload.code, "runner-job-failed-without-terminal-report");
|
||||||
|
assert.equal(exhausted.payload.retryable, false);
|
||||||
|
assert.equal(fact.store.getCommand(fact.commandId).state, "failed");
|
||||||
|
assert.equal(fact.store.getRun(fact.runId).status, "failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function assertSucceededJobWithoutTerminalOutboxFails(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
|
||||||
|
const fact = createRunnerFact("succeeded-without-outbox");
|
||||||
|
await writeFixture(fixturePath, "", "", "succeeded");
|
||||||
|
await reconcileRunnerJobsOnce({ store: fact.store, namespace: "agentrun-v02", kubectlCommand, startupRecovery: { ...policy, maxAttempts: 5 } });
|
||||||
|
const exhausted = phase(fact.store.listEvents(fact.runId, 0, 100), "runner-startup-retry-exhausted");
|
||||||
|
assert.equal(exhausted.payload.code, "runner-job-succeeded-without-terminal-report");
|
||||||
|
assert.equal(fact.store.getCommand(fact.commandId).state, "failed");
|
||||||
|
assert.equal(fact.store.getRun(fact.runId).status, "failed");
|
||||||
|
}
|
||||||
|
|
||||||
async function assertStartupRecoveryIsVisible(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
|
async function assertStartupRecoveryIsVisible(context: SelfTestContext, fixturePath: string, kubectlCommand: string): Promise<void> {
|
||||||
const fact = createRunnerFact("recovered");
|
const fact = createRunnerFact("recovered");
|
||||||
await writeFixture(fixturePath, "ErrImagePull", "failed to pull image: i/o timeout");
|
await writeFixture(fixturePath, "ErrImagePull", "failed to pull image: i/o timeout");
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ if (args[0] !== "get") {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (resource === "job") {
|
if (resource === "job") {
|
||||||
|
const failed = fixture.mode === "failed";
|
||||||
|
const succeeded = fixture.mode === "succeeded";
|
||||||
process.stdout.write(JSON.stringify({
|
process.stdout.write(JSON.stringify({
|
||||||
kind: "Job",
|
kind: "Job",
|
||||||
metadata: {
|
metadata: {
|
||||||
@@ -24,13 +26,19 @@ if (resource === "job") {
|
|||||||
uid: "job-startup-selftest",
|
uid: "job-startup-selftest",
|
||||||
creationTimestamp: fixture.createdAt,
|
creationTimestamp: fixture.createdAt,
|
||||||
},
|
},
|
||||||
status: { active: 1, startTime: fixture.createdAt },
|
status: failed
|
||||||
|
? { failed: 1, startTime: fixture.createdAt, conditions: [{ type: "Failed", status: "True", reason: "BackoffLimitExceeded" }] }
|
||||||
|
: succeeded
|
||||||
|
? { succeeded: 1, startTime: fixture.createdAt, conditions: [{ type: "Complete", status: "True" }] }
|
||||||
|
: { active: 1, startTime: fixture.createdAt },
|
||||||
}));
|
}));
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (resource === "pods") {
|
if (resource === "pods") {
|
||||||
const running = fixture.mode === "running";
|
const running = fixture.mode === "running";
|
||||||
|
const failed = fixture.mode === "failed";
|
||||||
|
const succeeded = fixture.mode === "succeeded";
|
||||||
const waiting = running ? undefined : {
|
const waiting = running ? undefined : {
|
||||||
reason: fixture.reason,
|
reason: fixture.reason,
|
||||||
message: fixture.message,
|
message: fixture.message,
|
||||||
@@ -44,7 +52,7 @@ if (resource === "pods") {
|
|||||||
creationTimestamp: fixture.createdAt,
|
creationTimestamp: fixture.createdAt,
|
||||||
},
|
},
|
||||||
status: {
|
status: {
|
||||||
phase: running ? "Running" : "Pending",
|
phase: running ? "Running" : failed ? "Failed" : succeeded ? "Succeeded" : "Pending",
|
||||||
containerStatuses: [{
|
containerStatuses: [{
|
||||||
name: "runner",
|
name: "runner",
|
||||||
state: running ? { running: { startedAt: fixture.createdAt } } : { waiting },
|
state: running ? { running: { startedAt: fixture.createdAt } } : { waiting },
|
||||||
|
|||||||
Reference in New Issue
Block a user