From 6173b5e5c6f16201a2847743d37499a612a0434e Mon Sep 17 00:00:00 2001 From: lyon Date: Tue, 16 Jun 2026 03:45:33 +0800 Subject: [PATCH] fix: mark evicted agent sessions non-reusable --- internal/cloud/access-control.ts | 6 +- internal/cloud/code-agent-agentrun-adapter.ts | 16 ++- .../cloud/code-agent-session-lifecycle.ts | 12 +- .../cloud/code-agent-session-registry.test.ts | 2 +- internal/cloud/server-agent-chat.test.ts | 122 ++++++++++++++++++ internal/cloud/server-code-agent-http.ts | 2 + 6 files changed, 150 insertions(+), 10 deletions(-) diff --git a/internal/cloud/access-control.ts b/internal/cloud/access-control.ts index 388c3558..bcb41feb 100644 --- a/internal/cloud/access-control.ts +++ b/internal/cloud/access-control.ts @@ -2411,8 +2411,8 @@ function isThreadResumeFailedResult(result = {}) { result.agentRun?.providerTrace?.failureKind, result.agentRun?.failureKind ].map((value) => textOr(value, "").toLowerCase().replace(/_/gu, "-")); - return values.some((value) => value === "thread-resume-failed") - || /no rollout found for thread id|thread\/resume failed/iu.test(String(result.error?.message ?? result.blocker?.message ?? result.providerTrace?.failureMessage ?? result.agentRun?.providerTrace?.failureMessage ?? "")); + return values.some((value) => value === "thread-resume-failed" || value === "session-store-evicted") + || /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|thread\/resume failed/iu.test(String(result.error?.message ?? result.blocker?.message ?? result.providerTrace?.failureMessage ?? result.agentRun?.providerTrace?.failureMessage ?? "")); } function threadResumeFailureThreadId(result = {}) { return boundedText(textOr( @@ -2425,7 +2425,7 @@ function threadResumeFailureThreadId(result = {}) { ), 240) || null; } function terminalWorkbenchSessionStatus(result = {}) { - if (isThreadResumeFailedResult(result)) return "failed"; + if (isThreadResumeFailedResult(result)) return "thread-resume-failed"; const sessionStatus = textOr(result.session?.status ?? result.sessionSummary?.status ?? result.sessionLifecycleStatus ?? result.runnerTrace?.sessionStatus, "").toLowerCase(); if (sessionStatus && sessionStatus !== "running" && sessionStatus !== "busy" && sessionStatus !== "pending") return sessionStatus === "cancelled" ? "canceled" : sessionStatus; const status = textOr(result.status ?? result.agentRun?.terminalStatus, "").toLowerCase(); diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index ce6d5d53..a2fe6b9c 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -807,7 +807,13 @@ function agentRunProjectIdForEnv(env = process.env) { } async function shouldResetSessionAfterEviction(failureKind, failureMessage) { - return failureKind === "session-store-evicted" || /session store evicted/i.test(failureMessage ?? ""); + return agentRunFailureRequiresFreshSession(failureKind, failureMessage); +} + +function agentRunFailureRequiresFreshSession(failureKind, failureMessage) { + const kind = String(failureKind ?? "").trim().toLowerCase().replace(/_/gu, "-"); + const message = String(failureMessage ?? ""); + return kind === "session-store-evicted" || kind === "thread-resume-failed" || /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|thread\/resume failed/iu.test(message); } function newSessionIdAfterEviction(baseSessionId, traceId) { @@ -1456,14 +1462,16 @@ function agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, a }); } const partialContext = partialAgentRunContext(runnerTrace); - const resumableAfterFailure = !canceled && Boolean(safeOpaqueId(base.threadId ?? base.agentRun?.threadId)); + const freshSessionRequired = !canceled && agentRunFailureRequiresFreshSession(code, message); + const failedSessionStatus = freshSessionRequired ? "thread-resume-failed" : "failed"; + const resumableAfterFailure = !canceled && !freshSessionRequired && Boolean(safeOpaqueId(base.threadId ?? base.agentRun?.threadId)); return { ...base, status: canceled ? "canceled" : "failed", canceled, updatedAt: now, - session: agentRunSessionSummary(base, canceled ? "canceled" : "failed"), - sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true, { status: canceled ? undefined : resumableAfterFailure ? "failed-resumable" : "failed-requires-new-session" }), + session: agentRunSessionSummary(base, canceled ? "canceled" : failedSessionStatus), + sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true, { status: canceled ? undefined : freshSessionRequired ? "thread-resume-failed" : resumableAfterFailure ? "failed-resumable" : "failed-requires-new-session" }), runner: agentRunRunnerSummary(base.agentRun), runnerTrace: partialContext ? { ...runnerTrace, partialContext } : runnerTrace, toolCalls: agentRunToolCalls(result, canceled ? "canceled" : "failed"), diff --git a/internal/cloud/code-agent-session-lifecycle.ts b/internal/cloud/code-agent-session-lifecycle.ts index 1b30e483..d20ba3e3 100644 --- a/internal/cloud/code-agent-session-lifecycle.ts +++ b/internal/cloud/code-agent-session-lifecycle.ts @@ -5,7 +5,8 @@ export const CODE_AGENT_SESSION_LIFECYCLE_STATUSES = Object.freeze([ "idle", "interrupted", "expired", - "failed" + "failed", + "thread-resume-failed" ]); export const CODE_AGENT_SESSION_STATUS_ALIASES = Object.freeze({ @@ -18,7 +19,7 @@ export const CODE_AGENT_SESSION_STATUS_ALIASES = Object.freeze({ }); const ACTIVE_LIFECYCLE_STATUSES = new Set(["creating", "ready", "busy", "idle"]); -const TERMINAL_LIFECYCLE_STATUSES = new Set(["expired"]); +const TERMINAL_LIFECYCLE_STATUSES = new Set(["expired", "thread-resume-failed"]); const RECOVERABLE_LIFECYCLE_STATUSES = new Set(["failed", "interrupted"]); export function normalizeCodeAgentSessionLifecycleStatus(value, { fallback = null } = {}) { @@ -175,6 +176,13 @@ function lifecyclePresentation({ status, rawStatus, reused, newSession, degraded userMessage: "当前 Code Agent session 上一轮失败；可继续发送以尝试恢复同一 conversation/session/thread。" }; } + if (status === "thread-resume-failed") { + return { + label: "会话存储已失效", + action: "new_session", + userMessage: "当前 Code Agent session/thread 的后端存储已失效；下一轮应从新的 session/thread 继续。" + }; + } if (status === "interrupted") { return { label: rawStatus === "canceled" ? "会话已取消" : "会话已中断", diff --git a/internal/cloud/code-agent-session-registry.test.ts b/internal/cloud/code-agent-session-registry.test.ts index 91e4f2b5..d1e8b546 100644 --- a/internal/cloud/code-agent-session-registry.test.ts +++ b/internal/cloud/code-agent-session-registry.test.ts @@ -66,7 +66,7 @@ test("code agent session registry creates, reuses, and expires sessions without assert.equal(describedCreating.recentSessions[0].sessionMode, "controlled-readonly-session-registry"); assert.equal(describedCreating.recentSessions[0].workspace, "/workspace/hwlab"); assert.equal(describedCreating.recentSessions[0].secretMaterialStored, false); - assert.deepEqual(describedCreating.lifecycleStatuses, ["creating", "ready", "busy", "idle", "interrupted", "expired", "failed"]); + assert.deepEqual(describedCreating.lifecycleStatuses, ["creating", "ready", "busy", "idle", "interrupted", "expired", "failed", "thread-resume-failed"]); assert.equal(describedCreating.statusAliases.canceled, "interrupted"); registry.release(first.session.sessionId, { conversationId: "cnv_registry", diff --git a/internal/cloud/server-agent-chat.test.ts b/internal/cloud/server-agent-chat.test.ts index 6aed353a..c01f0aca 100644 --- a/internal/cloud/server-agent-chat.test.ts +++ b/internal/cloud/server-agent-chat.test.ts @@ -1038,6 +1038,128 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result } }); +test("cloud api AgentRun adapter marks evicted session storage as non-reusable", async () => { + const agentRunServer = createHttpServer(async (request, response) => { + const url = new URL(request.url || "/", "http://127.0.0.1"); + const chunks = []; + for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; + const send = (data) => { + response.writeHead(200, { "content-type": "application/json" }); + response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_agentrun_evicted" })}\n`); + }; + if (request.method === "POST" && url.pathname === "/api/v1/runs") { + return send({ id: "run_evicted", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); + } + if (request.method === "POST" && url.pathname === "/api/v1/runs/run_evicted/commands") { + return send({ id: "cmd_evicted", runId: "run_evicted", state: "queued" }); + } + if (request.method === "POST" && url.pathname === "/api/v1/runs/run_evicted/runner-jobs") { + return send({ + action: "create-kubernetes-job", + runId: "run_evicted", + commandId: "cmd_evicted", + attemptId: "attempt_evicted", + runnerId: "runner_evicted", + namespace: "agentrun-v01", + jobName: "agentrun-v01-runner-evicted", + runner: { attemptId: "attempt_evicted", runnerId: "runner_evicted" } + }); + } + if (request.method === "GET" && url.pathname === "/api/v1/runs/run_evicted/events") { + return send({ items: [ + { id: "evt_evicted", runId: "run_evicted", seq: 1, type: "error", payload: { commandId: "cmd_evicted", failureKind: "session-store-evicted", message: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted" }, createdAt: "2026-06-02T00:00:00.000Z" }, + { id: "evt_evicted_terminal", runId: "run_evicted", seq: 2, type: "terminal_status", payload: { commandId: "cmd_evicted", terminalStatus: "failed", failureKind: "session-store-evicted" }, createdAt: "2026-06-02T00:00:01.000Z" } + ] }); + } + if (request.method === "GET" && url.pathname === "/api/v1/runs/run_evicted/commands") { + return send({ items: [{ id: "cmd_evicted", runId: "run_evicted", state: "failed", terminalStatus: "failed", idempotencyKey: "trc_server-test-agentrun-evicted", payload: { traceId: "trc_server-test-agentrun-evicted" } }] }); + } + if (request.method === "GET" && url.pathname === "/api/v1/runs/run_evicted/commands/cmd_evicted/result") { + return send({ + runId: "run_evicted", + commandId: "cmd_evicted", + attemptId: "attempt_evicted", + runnerId: "runner_evicted", + jobName: "agentrun-v01-runner-evicted", + namespace: "agentrun-v01", + status: "failed", + runStatus: "failed", + commandState: "failed", + terminalStatus: "failed", + failureKind: "session-store-evicted", + failureMessage: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted", + completed: false, + lastSeq: 2, + eventCount: 2, + sessionRef: { sessionId: "ses_agentrun_evicted", conversationId: "cnv_evicted", threadId: "thread_evicted" } + }); + } + response.writeHead(404, { "content-type": "application/json" }); + response.end(`${JSON.stringify({ ok: false, failureKind: "schema-invalid", message: `unexpected ${request.method} ${url.pathname}`, traceId: "trc_fake_agentrun_evicted" })}\n`); + }); + await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); + const agentRunPort = agentRunServer.address().port; + const ownerSessions = new Map([["ses_server-test-evicted", testAgentSessionRecord({ + sessionId: "ses_server-test-evicted", + conversationId: "cnv_evicted", + threadId: "thread_evicted", + status: "idle" + })]]); + const server = createCloudApiServer({ + env: { + HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", + AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, + HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", + HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601", + HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "0123456789abcdef0123456789abcdef01234567", + HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "deepseek", + HWLAB_ENVIRONMENT: "v02", + HWLAB_GITOPS_PROFILE: "v02" + }, + accessController: { + required: false, + async authenticate() { + return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; + }, + async recordAgentSessionOwner(input) { + const record = testAgentSessionRecord(input); + ownerSessions.set(record.id, record); + return record; + }, + async getAgentSession(sessionId) { + return ownerSessions.get(sessionId) ?? null; + } + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const traceId = "trc_server-test-agentrun-evicted"; + const submit = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, { + method: "POST", + headers: { "content-type": "application/json", "x-trace-id": traceId, cookie: "hwlab_session=test-stub-session" }, + body: JSON.stringify({ conversationId: "cnv_evicted", sessionId: "ses_server-test-evicted", threadId: "thread_evicted", message: "resume evicted session" }) + }); + assert.equal(submit.status, 202); + const payload = await pollAgentResult(port, traceId); + validateCodeAgentChatSchema(payload); + assert.equal(payload.status, "failed"); + assert.equal(payload.error.code, "session-store-evicted"); + assert.equal(payload.session.status, "thread-resume-failed"); + assert.equal(payload.session.lifecycle.requiresNewSession, true); + assert.equal(payload.sessionReuse.threadId, "thread_evicted"); + assert.equal(payload.sessionReuse.status, "thread-resume-failed"); + assert.equal(payload.agentRun.reuseEligible, false); + assert.equal(payload.reuseEligible, false); + assert.equal(ownerSessions.get("ses_server-test-evicted")?.status, "thread-resume-failed"); + } finally { + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); + } +}); + test("cloud api AgentRun adapter maps minimax-m3 provider profile to AgentRun backend", async () => { const calls = []; const agentRunServer = createHttpServer(async (request, response) => { diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 2f69ed5c..ceb2bf2c 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -1644,6 +1644,8 @@ async function recordCodeAgentSessionOwner({ payload = {}, params = {}, options } function codeAgentOwnerStatusForResult(result = {}) { + const sessionStatus = textValue(result?.session?.status ?? result?.sessionSummary?.status ?? result?.sessionLifecycleStatus); + if (sessionStatus === "thread-resume-failed") return "thread-resume-failed"; if (result?.status === "completed") return "active"; if (result?.status === "canceled" || result?.status === "cancelled") return "canceled"; return result?.status ?? "active";