mirrors/pikasTech-agentrun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(codex): 仅按无活动时间终止 turn

Browse Source
This commit is contained in:
AgentRun Codex
2026-07-19 19:26:21 +02:00
parent a1cb170241
commit 822d1fbda2
5 changed files with 107 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backend/codex-stdio.ts
+1 -39
View File
@@ -826,44 +826,10 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
resolveTerminalNow();
};
options.abortSignal?.addEventListener("abort", abortTurn, { once: true });
const turnHardTimeoutMs = positiveTimeout(options.timeoutMs);
const turnIdleTimeoutMs = turnHardTimeoutMs;
const turnIdleTimeoutMs = positiveTimeout(options.timeoutMs);
const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs);
let hardTimeout: NodeJS.Timeout | null = null;
let idleTimeout: NodeJS.Timeout | null = null;
let idleWarningTimeout: NodeJS.Timeout | null = null;
let lastToolCallAt: number | null = null;
const turnHardTimeoutAttrs = (): JsonRecord => ({
waitingFor,
elapsedMs: Math.max(0, Date.now() - turnStartedAt),
idleMs: Math.max(0, Date.now() - lastActivityAt),
timeoutMs: turnHardTimeoutMs,
lastNotificationMethod,
threadId: threadId ?? null,
turnId: turnId ?? null,
terminalStatus: terminal?.status ?? null,
retryable: false,
retryAttempt: null,
retryMaxAttempts: 0,
retryExhausted: true,
lastToolCall,
toolIdleMs: lastToolCallAt === null ? null : Math.max(0, Date.now() - lastToolCallAt),
});
const failTurnHardTimeout = (): void => {
if (terminal) return;
const elapsedMs = Math.max(0, Date.now() - turnStartedAt);
terminal = { status: "failed", failureKind: "backend-timeout", message: `codex stdio turn hard timed out after ${turnHardTimeoutMs}ms total budget` };
const attrs = { ...turnHardTimeoutAttrs(), elapsedMs, terminalStatus: terminal.status, failureKind: terminal.failureKind };
emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:hard-timeout", timeoutMs: turnHardTimeoutMs, elapsedMs, idleMs: Math.max(0, Date.now() - lastActivityAt), waitingFor, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } });
emitCodexOtelSpan("codex_stdio.turn_hard_timeout", options, env, attrs, { status: "error", error: terminal.message });
beginInterruptAndStop("turn hard timeout", "turn:hard-timeout");
resolveTerminalNow();
};
const scheduleTurnHardTimeout = (): void => {
if (hardTimeout) clearTimeout(hardTimeout);
hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs);
hardTimeout.unref?.();
};
const scheduleIdleWarning = (): void => {
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
idleWarningTimeout = setTimeout(() => {
@@ -890,15 +856,12 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
idleTimeout.unref?.();
};
const stopTurnIdleTimeout = (): void => {
if (hardTimeout) clearTimeout(hardTimeout);
hardTimeout = null;
if (!idleTimeout) return;
clearTimeout(idleTimeout);
idleTimeout = null;
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
idleWarningTimeout = null;
};
scheduleTurnHardTimeout();
refreshTurnActivity();
const stopNotifications = session.addNotificationHandler((message) => {
refreshTurnActivity();
@@ -911,7 +874,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
const toolSummary = toolCallSummaryFromNotification(message);
if (toolSummary?.status === "completed" || toolSummary?.status === "failed") {
lastToolCall = toolSummary;
lastToolCallAt = Date.now();
}
exposeActiveTurn(normalized.turnId ? "turn-notification" : "notification");
emitEvents(normalized.events);
src/selftest/cases/30-codex-stdio.ts
+3 -2
View File
@@ -27,6 +27,8 @@ const selfTest: SelfTestCase = async (context) => {
const finalCommand = await client.get(`/api/v1/runs/${happy.runId}/commands/${happy.commandId}`) as { state?: string };
assert.equal(finalCommand.state, "completed");
await runSlowProgressIdleCase({ client, managerUrl: server.baseUrl, context });
const sandboxOverride = await createRunWithCommand(client, context, "hello sandbox override", "selftest-sandbox-override", 15_000);
const sandboxOverrideResult = await runOnce({ managerUrl: server.baseUrl, runId: sandboxOverride.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_CODEX_SHELL_SANDBOX: "danger-full-access", AGENTRUN_FAKE_CODEX_MODE: "require-danger-sandbox" }, oneShot: true });
assert.equal(sandboxOverrideResult.terminalStatus, "completed");
@@ -311,7 +313,6 @@ const selfTest: SelfTestCase = async (context) => {
await runRetryThenCompletedCase({ client, managerUrl: server.baseUrl, context });
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "invalid-json", expectedStatus: "failed", expectedFailureKind: "backend-json-parse-error" });
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-terminal", expectedStatus: "failed", expectedFailureKind: "backend-timeout", timeoutMs: 500 });
await runSlowProgressIdleCase({ client, managerUrl: server.baseUrl, context });
await runFailureDoesNotTerminalRunCase({ client, managerUrl: server.baseUrl, context });
await runSecretFailureCase({ client, managerUrl: server.baseUrl, context });
await runSpawnFailureCase({ client, managerUrl: server.baseUrl, context });
@@ -384,7 +385,7 @@ async function runLeaseConflictRecoveryCase(options: { client: ManagerClient; ma
}
async function runSlowProgressIdleCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 250);
const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 1_000);
const result = await runOnce({
managerUrl: options.managerUrl,
runId: item.runId,
src/selftest/cases/31-codex-stdio-idle-timeout.ts
+88
View File
@@ -0,0 +1,88 @@
import assert from "node:assert/strict";
import { startManagerServer } from "../../mgr/server.js";
import { ManagerClient } from "../../mgr/client.js";
import { MemoryAgentRunStore } from "../../mgr/store.js";
import { runOnce } from "../../runner/run-once.js";
import { createRunWithCommand, type SelfTestCase } from "../harness.js";
const selfTest: SelfTestCase = async (context) => {
const server = await startManagerServer({
port: 0,
host: "127.0.0.1",
sourceCommit: "self-test",
store: new MemoryAgentRunStore(),
});
try {
const client = new ManagerClient(server.baseUrl);
const active = await createRunWithCommand(
client,
context,
"keep working while progress remains visible",
"selftest-idle-timeout-active-progress",
1_000,
);
const activeResult = await runOnce({
managerUrl: server.baseUrl,
runId: active.runId,
codexCommand: context.fakeCodexCommand,
codexArgs: context.fakeCodexArgs,
codexHome: context.codexHome,
env: {
CODEX_HOME: context.codexHome,
AGENTRUN_FAKE_CODEX_MODE: "slow-progress-before-terminal",
},
oneShot: true,
});
assert.equal(activeResult.terminalStatus, "completed");
assert.equal(activeResult.failureKind, null);
const activeEvents = await client.get(`/api/v1/runs/${active.runId}/events?afterSeq=0&limit=100`) as {
items?: Array<{ type: string; payload: Record<string, unknown> }>;
};
assert.equal(
activeEvents.items?.some((event) => event.type === "error" && event.payload.phase === "turn:hard-timeout"),
false,
);
const idle = await createRunWithCommand(
client,
context,
"remain silent until the idle deadline",
"selftest-idle-timeout-silence",
1_000,
);
const idleResult = await runOnce({
managerUrl: server.baseUrl,
runId: idle.runId,
codexCommand: context.fakeCodexCommand,
codexArgs: context.fakeCodexArgs,
codexHome: context.codexHome,
env: {
CODEX_HOME: context.codexHome,
AGENTRUN_FAKE_CODEX_MODE: "missing-terminal",
},
oneShot: true,
});
assert.equal(idleResult.terminalStatus, "failed");
assert.equal(idleResult.failureKind, "backend-timeout");
const idleEvents = await client.get(`/api/v1/runs/${idle.runId}/events?afterSeq=0&limit=100`) as {
items?: Array<{ type: string; payload: Record<string, unknown> }>;
};
assert.equal(
idleEvents.items?.some((event) => event.type === "error" && event.payload.phase === "turn:idle-timeout"),
true,
);
return {
name: "codex-stdio-idle-timeout",
tests: [
"active-progress-resets-idle-timeout",
"silent-turn-reaches-idle-timeout",
],
};
} finally {
await new Promise<void>((resolve) => server.server.close(() => resolve()));
}
};
export default selfTest;
src/selftest/cases/50-hwlab-manual-dispatch.ts
+11 -7
View File
@@ -447,14 +447,18 @@ process.exit(1);
const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 });
await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "acknowledged");
await waitForEvent(client, outputWithoutTerminal.runId, (event) => event.type === "command_output", "command output without terminal");
await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "failed");
await new Promise((resolve) => setTimeout(resolve, 750));
const outputWithoutTerminalActive = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/commands/${outputWithoutTerminal.commandId}`) as JsonRecord;
assert.equal(outputWithoutTerminalActive.state, "acknowledged", "continuous command output must keep the turn active beyond its idle budget");
const activeOutputEvents = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: JsonRecord }> };
assert.equal(activeOutputEvents.items?.some((event) => event.type === "error" && event.payload.phase === "turn:hard-timeout"), false);
await client.post(`/api/v1/commands/${outputWithoutTerminal.commandId}/cancel`, { reason: "self-test continuous output cleanup" });
const outputWithoutTerminalResult = await outputWithoutTerminalRunner as JsonRecord;
assert.equal(outputWithoutTerminalResult.terminalStatus, "failed");
assert.equal(outputWithoutTerminalResult.failureKind, "backend-timeout");
assert.equal(outputWithoutTerminalResult.terminalStatus, "cancelled");
assert.equal(outputWithoutTerminalResult.failureKind, "cancelled");
const outputWithoutTerminalEnvelope = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/commands/${outputWithoutTerminal.commandId}/result`) as JsonRecord;
assert.equal(outputWithoutTerminalEnvelope.terminalStatus, "failed");
assert.equal(outputWithoutTerminalEnvelope.failureKind, "backend-timeout");
assert.match(String(outputWithoutTerminalEnvelope.failureMessage ?? outputWithoutTerminalEnvelope.message ?? ""), /hard timed out/u);
assert.equal(outputWithoutTerminalEnvelope.terminalStatus, "cancelled");
assert.equal(outputWithoutTerminalEnvelope.failureKind, "cancelled");
const runningCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-running", "cancel running", "hwlab-command-cancel-running", 10_000);
const running = runOnce({ managerUrl: server.baseUrl, runId: runningCancel.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "missing-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-running-cancel") }, oneShot: true });
@@ -463,7 +467,7 @@ process.exit(1);
const runningResult = await running;
assert.equal(runningResult.terminalStatus, "cancelled");
return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-warning", "same-run-runner-multiturn", "running-steer", "running-steer-user-message-target-trace", "backend-retry-replays-durable-steers", "backend-replay-barrier-fast-terminal", "backend-replay-rpc-failure-retries", "rebuilt-runner-resumes-acknowledged-turn", "no-event-watchdog-after-tool", "tool-output-hard-timeout", "running-cancel"] };
return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-warning", "same-run-runner-multiturn", "running-steer", "running-steer-user-message-target-trace", "backend-retry-replays-durable-steers", "backend-replay-barrier-fast-terminal", "backend-replay-rpc-failure-retries", "rebuilt-runner-resumes-acknowledged-turn", "no-event-watchdog-after-tool", "tool-output-refreshes-idle-timeout", "running-cancel"] };
} finally {
await new Promise<void>((resolve) => server.server.close(() => resolve()));
}
src/selftest/fake-codex-app-server.ts
+4 -2
View File
@@ -136,11 +136,13 @@ for await (const line of rl) {
const turn = { id: `turn_selftest_${turnCounter}`, status: "completed" };
notify("turn/started", { turn: { id: turn.id, status: "running" } });
respond(message.id, { turn: { id: turn.id, status: "running" } });
setTimeout(() => notify("item/agentMessage/delta", { itemId: "msg_slow_progress", delta: "still working" }), 40);
for (const delayMs of [600, 1200, 1800, 2400]) {
setTimeout(() => notify("item/agentMessage/delta", { itemId: "msg_slow_progress", delta: "still working" }), delayMs);
}
setTimeout(() => {
notify("item/completed", { item: { id: "msg_slow_progress", type: "agentMessage", text: "slow progress final" } });
notify("turn/completed", { turn });
}, 90);
}, 3000);
continue;
}
if (mode === "provider-503-terminal") {