diff --git a/docs/reference/spec-v02-hwlab-cloud-web.md b/docs/reference/spec-v02-hwlab-cloud-web.md index 84ec51d7..796053de 100644 --- a/docs/reference/spec-v02-hwlab-cloud-web.md +++ b/docs/reference/spec-v02-hwlab-cloud-web.md @@ -91,6 +91,32 @@ Cloud Web check 通过后仍需执行 bundle build 和 dist freshness 校验, 阅读 docs/reference/spec-v02-hwlab-cloud-web.md,然后用 cli 手动测试以下内容:确认 trace 渲染相关单测覆盖 request、setup、commandExecution、assistant markdown 和 completion row;该测试必须能在无浏览器、无 Playwright、无公网、无真实 provider 的环境中执行。 +## Session state 持久化与 eviction reset + +HWLAB v0.2 Code Agent chat turn 默认走 `sessionStorage=persistent`:每次 turn 由 `code-agent-agentrun-adapter.ts` 在创建 run 之前显式调 `POST /api/v1/sessions`,让 AgentRun v0.1 同步创建 RWO PVC(`agentrun-v01-session-`,1Gi,StorageClass 走 env `AGENTRUN_SESSION_STORAGE_CLASS` 默认 `local-path`)。runner Job manifest 渲染时多挂一个 `agentrun-sessions` volume + `AGENTRUN_SESSION_PVC_NAME` / `_NAMESPACE` / `_MOUNT_PATH` / `AGENTRUN_CODEX_ROLLOUT_SUBDIR` env,使 codex app-server 自己把 rollout JSONL 写进 PVC,跨 runner pod 重建天然 `thread/resume:completed`。 + +### Eviction reset UX + +当 AgentRun v0.1 上报 `failureKind=session-store-evicted`(PVC 被删或 TTL 到期)时,HWLAB cloud-api 走 reset UX: + +- 同 `conversationId`,发新 `sessionId`(`newSessionIdAfterEviction(baseSessionId, traceId)` = `-reset-`),`threadId=null` 强制走 `thread/start`。 +- 旧 session 的 `storageKind` 已被 AgentRun 标为 `evicted`,HWLAB adapter 不再 reuse 旧 mapping。 +- Workbench 把 `errorCode=session_storage_evicted` 归到 `session-blocked` category,UI 文案与 `session_failed` 区分:「Code Agent session 存储已失效(PVC 被回收 / TTL 到期),HWLAB 已为你开新 sessionId,可继续发送下一条消息。」 + +禁止路径: + +- 不允许 fake `thread/resume:completed`(PR #78 已锁定的 v0.1 contract)。 +- 不允许 `idleTimeoutMs` 拉成永驻当成本特性。 +- 不允许 runner Job 启动后再做 copy/restore(本方案撤掉的路径,禁止复活)。 + +### 实现情况 + +| 规格项 | 状态 | 说明 | +| --- | --- | --- | +| 调 POST /api/v1/sessions 同步建 session + PVC | 已实现 | `code-agent-agentrun-adapter.ts::ensureAgentRunSessionPersistent` 默认 `sessionStorage=persistent`;env `HWLAB_CODE_AGENT_AGENTRUN_SESSION_STORAGE=ephemeral` 可退回 metadata-only 模式。 | +| runner Job 直接挂载 PVC | 已实现(agentrun-v01 侧 PR A/B/C 已上线) | HWLAB 侧无需额外代码,runner Job manifest 由 AgentRun v0.1 渲染。 | +| eviction reset UX | 已实现 | submitAgentRunChatTurn 走两步尝试,第一步在 ensureSession 或 runner-jobs POST 收到 `session-store-evicted` 时用新 sessionId + `threadId=null` 重试。 | +| UX 错误码 `session_storage_evicted` | 待 PR D 收口 | `code-agent-response-contract.mjs` 把 `session-store-evicted` 归 `session-blocked`;`code-agent-chat.ts` 加 userMessage。 | ## 规格的实现情况 | 规格项 | 状态 | 说明 | diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index f98de73f..ff70548d 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -159,51 +159,61 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI } } - const sessionId = scopedAgentRunSessionIdForParams(params, traceId, backendProfile); - const runInput = buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId }); - const run = await agentRunJson(fetchImpl, managerUrl, "/api/v1/runs", { - method: "POST", - body: runInput, - timeoutMs - }); - const runId = requiredString(run?.id, "run.id"); - traceStore.append(traceId, { - type: "backend", - status: "running", - label: "agentrun:run:created", - message: `AgentRun run ${runId} created through internal k3s Service DNS.`, - runId, - backendProfile, - waitingFor: "agentrun-command-create", - valuesPrinted: false - }); - - const commandInput = buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId }); - const command = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands`, { - method: "POST", - body: commandInput, - timeoutMs - }); - const commandId = requiredString(command?.id, "command.id"); - traceStore.append(traceId, { - type: "backend", - status: "running", - label: "agentrun:command:created", - message: `AgentRun command ${commandId} created; hwlab-cloud-api will start a runner Job explicitly without relying on scheduler automation.`, - runId, - commandId, - backendProfile, - waitingFor: "agentrun-runner-job-create", - valuesPrinted: false - }); - - const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId }); - const runnerJob = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, { - method: "POST", - body: runnerJobInput, - timeoutMs - }); - const mapping = agentRunMapping({ env, managerUrl, backendProfile, run, command, runnerJob, traceId, startedAt }); + const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId, backendProfile); + let sessionId = baseSessionId; + let sessionReset = false; + let run, command, runnerJob, mapping; + for (let attempt = 0; attempt < 2; attempt += 1) { + if (sessionReset) { + sessionId = newSessionIdAfterEviction(baseSessionId, traceId); + const resetParams = { ...params, threadId: null }; + traceStore.append(traceId, { + type: "backend", status: "running", + label: "agentrun:session-reset", + message: "AgentRun session storage was evicted; hwlab-cloud-api is creating a fresh sessionId " + sessionId + " with threadId=null.", + sessionId, previousSessionId: baseSessionId, backendProfile, waitingFor: "agentrun-run-create", valuesPrinted: false, + }); + } + try { + await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }); + } catch (error) { + if (attempt === 0 && await shouldResetSessionAfterEviction("session-store-evicted", error?.message)) { + sessionReset = true; + continue; + } + throw error; + } + const runInput = buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId }); + run = await agentRunJson(fetchImpl, managerUrl, "/api/v1/runs", { method: "POST", body: runInput, timeoutMs }); + const runId = requiredString(run?.id, "run.id"); + traceStore.append(traceId, { + type: "backend", status: "running", + label: "agentrun:run:created", + message: "AgentRun run " + runId + " created through internal k3s Service DNS.", + runId, backendProfile, waitingFor: "agentrun-command-create", valuesPrinted: false, + }); + const commandInput = buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId }); + command = await agentRunJson(fetchImpl, managerUrl, "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", { method: "POST", body: commandInput, timeoutMs }); + const commandId = requiredString(command?.id, "command.id"); + traceStore.append(traceId, { + type: "backend", status: "running", + label: "agentrun:command:created", + message: "AgentRun command " + commandId + " created; hwlab-cloud-api will start a runner Job explicitly without relying on scheduler automation.", + runId, commandId, backendProfile, waitingFor: "agentrun-runner-job-create", valuesPrinted: false, + }); + const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId }); + try { + runnerJob = await agentRunJson(fetchImpl, managerUrl, "/api/v1/runs/" + encodeURIComponent(runId) + "/runner-jobs", { method: "POST", body: runnerJobInput, timeoutMs }); + } catch (error) { + if (attempt === 0 && await shouldResetSessionAfterEviction("session-store-evicted", error?.message)) { + sessionReset = true; + continue; + } + throw error; + } + mapping = agentRunMapping({ env, managerUrl, backendProfile, run, command, runnerJob, traceId, startedAt }); + break; + } traceStore.append(traceId, { type: "backend", status: "running", @@ -463,6 +473,47 @@ export function agentRunSessionEvidence(payload = {}) { }; } +async function ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }) { + const defaultPolicy = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SESSION_STORAGE, "persistent"); + if (defaultPolicy !== "persistent") return; + try { + const tenantId = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TENANT_ID, DEFAULT_TENANT_ID); + const projectId = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECT_ID, DEFAULT_PROJECT_ID); + const expiresInDays = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_SESSION_TTL_DAYS, 30); + const expiresAt = new Date(Date.now() + Math.max(1, expiresInDays) * 24 * 60 * 60 * 1000).toISOString(); + await agentRunJson(fetchImpl, managerUrl, "/api/v1/sessions", { + method: "POST", + timeoutMs: 15_000, + body: { sessionId, tenantId, projectId, backendProfile, expiresAt, codexRolloutSubdir: "sessions" }, + }); + } catch (error) { + const message = error?.message ?? String(error); + if (/evicted/i.test(message)) { + traceStore.append(traceId, { + type: "backend", status: "running", + label: "agentrun:session-storage-evicted", + message: "AgentRun session " + sessionId + " storage was previously evicted; hwlab-cloud-api will create a fresh sessionId for this turn.", + sessionId, backendProfile, waitingFor: "agentrun-session-reset", valuesPrinted: false, + }); + throw error; + } + traceStore.append(traceId, { + type: "backend", status: "running", + label: "agentrun:session-storage-recover-warning", + message: "AgentRun session " + sessionId + " pre-flight could not ensure PVC; falling back to legacy metadata-only session (" + message + ").", + sessionId, backendProfile, valuesPrinted: false, + }); + } +} + +async function shouldResetSessionAfterEviction(failureKind, failureMessage) { + return failureKind === "session-store-evicted" || /session store evicted/i.test(failureMessage ?? ""); +} + +function newSessionIdAfterEviction(baseSessionId, traceId) { + const profile = String(traceId).replace(/[^A-Za-z0-9]/gu, "").slice(0, 12) || "fresh"; + return baseSessionId + "-reset-" + profile; +} function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId }) { const commitId = fullSourceCommit(env); const threadId = safeOpaqueId(params.threadId); diff --git a/internal/cloud/code-agent-chat.ts b/internal/cloud/code-agent-chat.ts index 306f4540..9dba7276 100644 --- a/internal/cloud/code-agent-chat.ts +++ b/internal/cloud/code-agent-chat.ts @@ -1328,6 +1328,12 @@ function errorTaxonomy(code, error = {}) { retryable: true, userMessage: "Code Agent 当前 turn 已失败,session/thread 已保留,可继续发送下一条消息。" }, + session_storage_evicted: { + layer: "session", + category: "session_blocked", + retryable: true, + userMessage: "Code Agent session 存储已失效(PVC 被回收或 TTL 到期),HWLAB 已为你开新 sessionId,可继续发送下一条消息。" + }, session_interrupted: { layer: "session", category: "session_blocked", diff --git a/scripts/src/code-agent-response-contract.mjs b/scripts/src/code-agent-response-contract.mjs index 0277e487..4a246486 100644 --- a/scripts/src/code-agent-response-contract.mjs +++ b/scripts/src/code-agent-response-contract.mjs @@ -481,7 +481,7 @@ export function classifyCodeAgentRuntimeBlock(payload, { httpStatus = null } = { }; } - if (["session_expired", "session_reuse_conflict", "session_failed", "session_interrupted", "session_canceled", "session_timeout", "session_error"].includes(errorCode)) { + if (["session_expired", "session_reuse_conflict", "session_failed", "session_interrupted", "session_canceled", "session_timeout", "session_error", "session_storage_evicted"].includes(errorCode)) { return { blocked: true, status: "blocked",