diff --git a/src/backend/codex-stdio.ts b/src/backend/codex-stdio.ts index 23a980a..4d0233a 100644 --- a/src/backend/codex-stdio.ts +++ b/src/backend/codex-stdio.ts @@ -826,44 +826,10 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess resolveTerminalNow(); }; options.abortSignal?.addEventListener("abort", abortTurn, { once: true }); - const turnHardTimeoutMs = positiveTimeout(options.timeoutMs); - const turnIdleTimeoutMs = turnHardTimeoutMs; + const turnIdleTimeoutMs = positiveTimeout(options.timeoutMs); const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs); - let hardTimeout: NodeJS.Timeout | null = null; let idleTimeout: NodeJS.Timeout | null = null; let idleWarningTimeout: NodeJS.Timeout | null = null; - let lastToolCallAt: number | null = null; - const turnHardTimeoutAttrs = (): JsonRecord => ({ - waitingFor, - elapsedMs: Math.max(0, Date.now() - turnStartedAt), - idleMs: Math.max(0, Date.now() - lastActivityAt), - timeoutMs: turnHardTimeoutMs, - lastNotificationMethod, - threadId: threadId ?? null, - turnId: turnId ?? null, - terminalStatus: terminal?.status ?? null, - retryable: false, - retryAttempt: null, - retryMaxAttempts: 0, - retryExhausted: true, - lastToolCall, - toolIdleMs: lastToolCallAt === null ? null : Math.max(0, Date.now() - lastToolCallAt), - }); - const failTurnHardTimeout = (): void => { - if (terminal) return; - const elapsedMs = Math.max(0, Date.now() - turnStartedAt); - terminal = { status: "failed", failureKind: "backend-timeout", message: `codex stdio turn hard timed out after ${turnHardTimeoutMs}ms total budget` }; - const attrs = { ...turnHardTimeoutAttrs(), elapsedMs, terminalStatus: terminal.status, failureKind: terminal.failureKind }; - emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:hard-timeout", timeoutMs: turnHardTimeoutMs, elapsedMs, idleMs: Math.max(0, Date.now() - lastActivityAt), waitingFor, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } }); - emitCodexOtelSpan("codex_stdio.turn_hard_timeout", options, env, attrs, { status: "error", error: terminal.message }); - beginInterruptAndStop("turn hard timeout", "turn:hard-timeout"); - resolveTerminalNow(); - }; - const scheduleTurnHardTimeout = (): void => { - if (hardTimeout) clearTimeout(hardTimeout); - hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs); - hardTimeout.unref?.(); - }; const scheduleIdleWarning = (): void => { if (idleWarningTimeout) clearTimeout(idleWarningTimeout); idleWarningTimeout = setTimeout(() => { @@ -890,15 +856,12 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess idleTimeout.unref?.(); }; const stopTurnIdleTimeout = (): void => { - if (hardTimeout) clearTimeout(hardTimeout); - hardTimeout = null; if (!idleTimeout) return; clearTimeout(idleTimeout); idleTimeout = null; if (idleWarningTimeout) clearTimeout(idleWarningTimeout); idleWarningTimeout = null; }; - scheduleTurnHardTimeout(); refreshTurnActivity(); const stopNotifications = session.addNotificationHandler((message) => { refreshTurnActivity(); @@ -911,7 +874,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess const toolSummary = toolCallSummaryFromNotification(message); if (toolSummary?.status === "completed" || toolSummary?.status === "failed") { lastToolCall = toolSummary; - lastToolCallAt = Date.now(); } exposeActiveTurn(normalized.turnId ? "turn-notification" : "notification"); emitEvents(normalized.events); diff --git a/src/selftest/cases/30-codex-stdio.ts b/src/selftest/cases/30-codex-stdio.ts index a2593a2..354c5dc 100644 --- a/src/selftest/cases/30-codex-stdio.ts +++ b/src/selftest/cases/30-codex-stdio.ts @@ -27,6 +27,8 @@ const selfTest: SelfTestCase = async (context) => { const finalCommand = await client.get(`/api/v1/runs/${happy.runId}/commands/${happy.commandId}`) as { state?: string }; assert.equal(finalCommand.state, "completed"); + await runSlowProgressIdleCase({ client, managerUrl: server.baseUrl, context }); + const sandboxOverride = await createRunWithCommand(client, context, "hello sandbox override", "selftest-sandbox-override", 15_000); const sandboxOverrideResult = await runOnce({ managerUrl: server.baseUrl, runId: sandboxOverride.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_CODEX_SHELL_SANDBOX: "danger-full-access", AGENTRUN_FAKE_CODEX_MODE: "require-danger-sandbox" }, oneShot: true }); assert.equal(sandboxOverrideResult.terminalStatus, "completed"); @@ -311,7 +313,6 @@ const selfTest: SelfTestCase = async (context) => { await runRetryThenCompletedCase({ client, managerUrl: server.baseUrl, context }); await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "invalid-json", expectedStatus: "failed", expectedFailureKind: "backend-json-parse-error" }); await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-terminal", expectedStatus: "failed", expectedFailureKind: "backend-timeout", timeoutMs: 500 }); - await runSlowProgressIdleCase({ client, managerUrl: server.baseUrl, context }); await runFailureDoesNotTerminalRunCase({ client, managerUrl: server.baseUrl, context }); await runSecretFailureCase({ client, managerUrl: server.baseUrl, context }); await runSpawnFailureCase({ client, managerUrl: server.baseUrl, context }); @@ -384,7 +385,7 @@ async function runLeaseConflictRecoveryCase(options: { client: ManagerClient; ma } async function runSlowProgressIdleCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise { - const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 250); + const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 1_000); const result = await runOnce({ managerUrl: options.managerUrl, runId: item.runId, diff --git a/src/selftest/cases/31-codex-stdio-idle-timeout.ts b/src/selftest/cases/31-codex-stdio-idle-timeout.ts new file mode 100644 index 0000000..c89955a --- /dev/null +++ b/src/selftest/cases/31-codex-stdio-idle-timeout.ts @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; + +import { startManagerServer } from "../../mgr/server.js"; +import { ManagerClient } from "../../mgr/client.js"; +import { MemoryAgentRunStore } from "../../mgr/store.js"; +import { runOnce } from "../../runner/run-once.js"; +import { createRunWithCommand, type SelfTestCase } from "../harness.js"; + +const selfTest: SelfTestCase = async (context) => { + const server = await startManagerServer({ + port: 0, + host: "127.0.0.1", + sourceCommit: "self-test", + store: new MemoryAgentRunStore(), + }); + try { + const client = new ManagerClient(server.baseUrl); + const active = await createRunWithCommand( + client, + context, + "keep working while progress remains visible", + "selftest-idle-timeout-active-progress", + 1_000, + ); + const activeResult = await runOnce({ + managerUrl: server.baseUrl, + runId: active.runId, + codexCommand: context.fakeCodexCommand, + codexArgs: context.fakeCodexArgs, + codexHome: context.codexHome, + env: { + CODEX_HOME: context.codexHome, + AGENTRUN_FAKE_CODEX_MODE: "slow-progress-before-terminal", + }, + oneShot: true, + }); + assert.equal(activeResult.terminalStatus, "completed"); + assert.equal(activeResult.failureKind, null); + const activeEvents = await client.get(`/api/v1/runs/${active.runId}/events?afterSeq=0&limit=100`) as { + items?: Array<{ type: string; payload: Record }>; + }; + assert.equal( + activeEvents.items?.some((event) => event.type === "error" && event.payload.phase === "turn:hard-timeout"), + false, + ); + + const idle = await createRunWithCommand( + client, + context, + "remain silent until the idle deadline", + "selftest-idle-timeout-silence", + 1_000, + ); + const idleResult = await runOnce({ + managerUrl: server.baseUrl, + runId: idle.runId, + codexCommand: context.fakeCodexCommand, + codexArgs: context.fakeCodexArgs, + codexHome: context.codexHome, + env: { + CODEX_HOME: context.codexHome, + AGENTRUN_FAKE_CODEX_MODE: "missing-terminal", + }, + oneShot: true, + }); + assert.equal(idleResult.terminalStatus, "failed"); + assert.equal(idleResult.failureKind, "backend-timeout"); + const idleEvents = await client.get(`/api/v1/runs/${idle.runId}/events?afterSeq=0&limit=100`) as { + items?: Array<{ type: string; payload: Record }>; + }; + assert.equal( + idleEvents.items?.some((event) => event.type === "error" && event.payload.phase === "turn:idle-timeout"), + true, + ); + + return { + name: "codex-stdio-idle-timeout", + tests: [ + "active-progress-resets-idle-timeout", + "silent-turn-reaches-idle-timeout", + ], + }; + } finally { + await new Promise((resolve) => server.server.close(() => resolve())); + } +}; + +export default selfTest; diff --git a/src/selftest/cases/50-hwlab-manual-dispatch.ts b/src/selftest/cases/50-hwlab-manual-dispatch.ts index 010ab06..304aec1 100644 --- a/src/selftest/cases/50-hwlab-manual-dispatch.ts +++ b/src/selftest/cases/50-hwlab-manual-dispatch.ts @@ -447,14 +447,18 @@ process.exit(1); const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 }); await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "acknowledged"); await waitForEvent(client, outputWithoutTerminal.runId, (event) => event.type === "command_output", "command output without terminal"); - await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "failed"); + await new Promise((resolve) => setTimeout(resolve, 750)); + const outputWithoutTerminalActive = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/commands/${outputWithoutTerminal.commandId}`) as JsonRecord; + assert.equal(outputWithoutTerminalActive.state, "acknowledged", "continuous command output must keep the turn active beyond its idle budget"); + const activeOutputEvents = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: JsonRecord }> }; + assert.equal(activeOutputEvents.items?.some((event) => event.type === "error" && event.payload.phase === "turn:hard-timeout"), false); + await client.post(`/api/v1/commands/${outputWithoutTerminal.commandId}/cancel`, { reason: "self-test continuous output cleanup" }); const outputWithoutTerminalResult = await outputWithoutTerminalRunner as JsonRecord; - assert.equal(outputWithoutTerminalResult.terminalStatus, "failed"); - assert.equal(outputWithoutTerminalResult.failureKind, "backend-timeout"); + assert.equal(outputWithoutTerminalResult.terminalStatus, "cancelled"); + assert.equal(outputWithoutTerminalResult.failureKind, "cancelled"); const outputWithoutTerminalEnvelope = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/commands/${outputWithoutTerminal.commandId}/result`) as JsonRecord; - assert.equal(outputWithoutTerminalEnvelope.terminalStatus, "failed"); - assert.equal(outputWithoutTerminalEnvelope.failureKind, "backend-timeout"); - assert.match(String(outputWithoutTerminalEnvelope.failureMessage ?? outputWithoutTerminalEnvelope.message ?? ""), /hard timed out/u); + assert.equal(outputWithoutTerminalEnvelope.terminalStatus, "cancelled"); + assert.equal(outputWithoutTerminalEnvelope.failureKind, "cancelled"); const runningCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-running", "cancel running", "hwlab-command-cancel-running", 10_000); const running = runOnce({ managerUrl: server.baseUrl, runId: runningCancel.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "missing-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-running-cancel") }, oneShot: true }); @@ -463,7 +467,7 @@ process.exit(1); const runningResult = await running; assert.equal(runningResult.terminalStatus, "cancelled"); - return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-warning", "same-run-runner-multiturn", "running-steer", "running-steer-user-message-target-trace", "backend-retry-replays-durable-steers", "backend-replay-barrier-fast-terminal", "backend-replay-rpc-failure-retries", "rebuilt-runner-resumes-acknowledged-turn", "no-event-watchdog-after-tool", "tool-output-hard-timeout", "running-cancel"] }; + return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-warning", "same-run-runner-multiturn", "running-steer", "running-steer-user-message-target-trace", "backend-retry-replays-durable-steers", "backend-replay-barrier-fast-terminal", "backend-replay-rpc-failure-retries", "rebuilt-runner-resumes-acknowledged-turn", "no-event-watchdog-after-tool", "tool-output-refreshes-idle-timeout", "running-cancel"] }; } finally { await new Promise((resolve) => server.server.close(() => resolve())); } diff --git a/src/selftest/fake-codex-app-server.ts b/src/selftest/fake-codex-app-server.ts index 13f6dbb..5390f43 100644 --- a/src/selftest/fake-codex-app-server.ts +++ b/src/selftest/fake-codex-app-server.ts @@ -136,11 +136,13 @@ for await (const line of rl) { const turn = { id: `turn_selftest_${turnCounter}`, status: "completed" }; notify("turn/started", { turn: { id: turn.id, status: "running" } }); respond(message.id, { turn: { id: turn.id, status: "running" } }); - setTimeout(() => notify("item/agentMessage/delta", { itemId: "msg_slow_progress", delta: "still working" }), 40); + for (const delayMs of [600, 1200, 1800, 2400]) { + setTimeout(() => notify("item/agentMessage/delta", { itemId: "msg_slow_progress", delta: "still working" }), delayMs); + } setTimeout(() => { notify("item/completed", { item: { id: "msg_slow_progress", type: "agentMessage", text: "slow progress final" } }); notify("turn/completed", { turn }); - }, 90); + }, 3000); continue; } if (mode === "provider-503-terminal") {