From a56ea7e33ef67040cc9a8748c3759b4f9c87ca46 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 27 May 2026 07:43:53 +0800 Subject: [PATCH] fix: extend code agent timeout recovery --- cmd/hwlab-cloud-api/main.mjs | 8 +- cmd/hwlab-edge-proxy/main.mjs | 4 +- deploy/deploy.json | 6 +- deploy/k8s/base/workloads.yaml | 6 +- scripts/artifact-publish.mjs | 4 +- .../artifact-runtime-readiness-guard.test.mjs | 28 +++--- .../src/artifact-runtime-readiness-guard.mjs | 17 ++-- scripts/src/deploy-contract-plan.mjs | 4 +- scripts/src/dev-cloud-workbench-smoke-lib.mjs | 8 +- web/hwlab-cloud-web/app.mjs | 36 ++++++-- web/hwlab-cloud-web/index.html | 4 +- web/hwlab-cloud-web/scripts/check.mjs | 6 +- .../scripts/trace-scroll.test.mjs | 88 +++++++++++++++++++ 13 files changed, 172 insertions(+), 47 deletions(-) diff --git a/cmd/hwlab-cloud-api/main.mjs b/cmd/hwlab-cloud-api/main.mjs index 38b43d3a..03fc52ab 100644 --- a/cmd/hwlab-cloud-api/main.mjs +++ b/cmd/hwlab-cloud-api/main.mjs @@ -4,13 +4,13 @@ import { createCloudApiServer } from "../../internal/cloud/server.mjs"; const host = process.env.HWLAB_CLOUD_API_HOST || "0.0.0.0"; const port = parsePort(process.env.HWLAB_CLOUD_API_PORT, parsePort(process.env.PORT, 6667)); const commitId = process.env.HWLAB_COMMIT_ID || process.env.HWLAB_GIT_SHA; -const codeAgentTimeoutMs = parseTimeout(process.env.HWLAB_CODE_AGENT_TIMEOUT_MS, 600000, { +const codeAgentTimeoutMs = parseTimeout(process.env.HWLAB_CODE_AGENT_TIMEOUT_MS, 1200000, { min: 1000, - max: 900000 + max: 2400000 }); -const codeAgentHardTimeoutMs = parseTimeout(process.env.HWLAB_CODE_AGENT_HARD_TIMEOUT_MS, Math.max(600000, codeAgentTimeoutMs * 4), { +const codeAgentHardTimeoutMs = parseTimeout(process.env.HWLAB_CODE_AGENT_HARD_TIMEOUT_MS, Math.max(1200000, codeAgentTimeoutMs * 4), { min: codeAgentTimeoutMs, - max: 1800000 + max: 3600000 }); if (!process.env.HWLAB_IMAGE && commitId) { diff --git a/cmd/hwlab-edge-proxy/main.mjs b/cmd/hwlab-edge-proxy/main.mjs index 38ecd5c0..dc82757b 100644 --- a/cmd/hwlab-edge-proxy/main.mjs +++ b/cmd/hwlab-edge-proxy/main.mjs @@ -10,9 +10,9 @@ import { createServer } from "node:http"; const serviceId = "hwlab-edge-proxy"; const upstream = process.env.HWLAB_EDGE_UPSTREAM || "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667"; -const proxyTimeoutMs = parseTimeout(process.env.HWLAB_EDGE_PROXY_TIMEOUT_MS, 660000, { +const proxyTimeoutMs = parseTimeout(process.env.HWLAB_EDGE_PROXY_TIMEOUT_MS, 1260000, { min: 1000, - max: 900000 + max: 2400000 }); const { host, port } = resolveHostPort({ listenEnv: "HWLAB_EDGE_LISTEN", diff --git a/deploy/deploy.json b/deploy/deploy.json index 126fbe75..5905d976 100644 --- a/deploy/deploy.json +++ b/deploy/deploy.json @@ -205,7 +205,7 @@ "HWLAB_M3_PATCH_PANEL_URL": "http://hwlab-patch-panel.hwlab-dev.svc.cluster.local:7301", "HWLAB_CODE_AGENT_PROVIDER": "codex-stdio", "HWLAB_CODE_AGENT_MODEL": "gpt-5.5", - "HWLAB_CODE_AGENT_TIMEOUT_MS": "600000", + "HWLAB_CODE_AGENT_TIMEOUT_MS": "1200000", "HWLAB_CODE_AGENT_OPENAI_BASE_URL": "http://127.0.0.1:49280/responses", "HWLAB_CODE_AGENT_HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", "HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED": "1", @@ -238,7 +238,7 @@ "env": { "HWLAB_ENVIRONMENT": "dev", "HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", - "HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS": "660000", + "HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS": "1260000", "HWLAB_COMMIT_ID": "b59d5b0", "HWLAB_IMAGE": "127.0.0.1:5000/hwlab/hwlab-cloud-web:b59d5b0", "HWLAB_IMAGE_TAG": "b59d5b0" @@ -376,7 +376,7 @@ "env": { "HWLAB_EDGE_LISTEN": "0.0.0.0:6667", "HWLAB_EDGE_UPSTREAM": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", - "HWLAB_EDGE_PROXY_TIMEOUT_MS": "660000" + "HWLAB_EDGE_PROXY_TIMEOUT_MS": "1260000" } }, { diff --git a/deploy/k8s/base/workloads.yaml b/deploy/k8s/base/workloads.yaml index 327a5b92..02e0677d 100644 --- a/deploy/k8s/base/workloads.yaml +++ b/deploy/k8s/base/workloads.yaml @@ -119,7 +119,7 @@ }, { "name": "HWLAB_CODE_AGENT_TIMEOUT_MS", - "value": "600000" + "value": "1200000" }, { "name": "HWLAB_CODE_AGENT_OPENAI_BASE_URL", @@ -381,7 +381,7 @@ }, { "name": "HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS", - "value": "660000" + "value": "1260000" }, { "name": "HWLAB_COMMIT_ID", @@ -958,7 +958,7 @@ }, { "name": "HWLAB_EDGE_PROXY_TIMEOUT_MS", - "value": "660000" + "value": "1260000" } ], "readinessProbe": { diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index d1c46725..681fb08c 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -490,9 +490,9 @@ const runtimeKind = process.env.HWLAB_ARTIFACT_KIND || "health-placeholder"; const entrypoint = process.env.HWLAB_SERVICE_ENTRYPOINT || ""; const port = Number.parseInt(process.env.PORT || process.env.HWLAB_PORT || "8080", 10); const cloudApiBaseUrl = process.env.HWLAB_API_BASE_URL || ""; -const cloudApiProxyTimeoutMs = parseTimeout(process.env.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS, 660000, { +const cloudApiProxyTimeoutMs = parseTimeout(process.env.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS, 1260000, { min: 1000, - max: 900000 + max: 2400000 }); const authUsername = nonEmptyEnv("HWLAB_CLOUD_WEB_AUTH_USERNAME", "admin"); const authPassword = nonEmptyEnv("HWLAB_CLOUD_WEB_AUTH_PASSWORD", "hwlab2026"); diff --git a/scripts/artifact-runtime-readiness-guard.test.mjs b/scripts/artifact-runtime-readiness-guard.test.mjs index 111af27f..5639abbc 100644 --- a/scripts/artifact-runtime-readiness-guard.test.mjs +++ b/scripts/artifact-runtime-readiness-guard.test.mjs @@ -258,10 +258,10 @@ test("compact commander summary exposes canPublish/canApply and dry-run devLiveC cloudWebDistFreshness: { status: "pass", mismatches: [] }, codeAgentTimeoutReadiness: { status: "pass", - frontendTimeoutMs: 660000, - backendTimeoutMs: 600000, + frontendTimeoutMs: 1800000, + backendTimeoutMs: 1200000, requiredFrontendTimeoutMs: 35000, - requiredBackendTimeoutMs: 600000, + requiredBackendTimeoutMs: 1200000, transportWaitsForBackend: true, summary: "ready" }, @@ -292,10 +292,10 @@ test("compact summary blocks publish on stale Cloud Web source dist", () => { cloudWebDistFreshness: { status: "blocked", mismatches: ["index.html"] }, codeAgentTimeoutReadiness: { status: "pass", - frontendTimeoutMs: 660000, - backendTimeoutMs: 600000, + frontendTimeoutMs: 1800000, + backendTimeoutMs: 1200000, requiredFrontendTimeoutMs: 35000, - requiredBackendTimeoutMs: 600000, + requiredBackendTimeoutMs: 1200000, transportWaitsForBackend: true, summary: "ready" }, @@ -326,9 +326,9 @@ test("compact summary separates Code Agent timeout and durable runtime blockers codeAgentTimeoutReadiness: { status: "blocked", frontendTimeoutMs: 4500, - backendTimeoutMs: 600000, + backendTimeoutMs: 1200000, requiredFrontendTimeoutMs: 35000, - requiredBackendTimeoutMs: 600000, + requiredBackendTimeoutMs: 1200000, transportWaitsForBackend: false, summary: "Code Agent browser timeout is too short." }, @@ -373,10 +373,10 @@ test("compact summary blocks publish when Code Agent transport timeout does not cloudWebDistFreshness: { status: "pass", mismatches: [] }, codeAgentTimeoutReadiness: { status: "blocked", - frontendTimeoutMs: 600000, - backendTimeoutMs: 600000, + frontendTimeoutMs: 1200000, + backendTimeoutMs: 1200000, requiredFrontendTimeoutMs: 35000, - requiredBackendTimeoutMs: 600000, + requiredBackendTimeoutMs: 1200000, transportWaitsForBackend: false, summary: "Code Agent timeout layering is unsafe." }, @@ -405,10 +405,10 @@ test("compact summary keeps provider-secret blockers out of publish but blocks a cloudWebDistFreshness: { status: "pass", mismatches: [] }, codeAgentTimeoutReadiness: { status: "pass", - frontendTimeoutMs: 660000, - backendTimeoutMs: 600000, + frontendTimeoutMs: 1800000, + backendTimeoutMs: 1200000, requiredFrontendTimeoutMs: 35000, - requiredBackendTimeoutMs: 600000, + requiredBackendTimeoutMs: 1200000, transportWaitsForBackend: true, summary: "ready" }, diff --git a/scripts/src/artifact-runtime-readiness-guard.mjs b/scripts/src/artifact-runtime-readiness-guard.mjs index 46d990da..5cdee0f2 100644 --- a/scripts/src/artifact-runtime-readiness-guard.mjs +++ b/scripts/src/artifact-runtime-readiness-guard.mjs @@ -22,8 +22,8 @@ const expectedBlockedGuardCommand = `${guardCommand} --expect-blocked`; const digestPattern = /^sha256:[a-f0-9]{64}$/u; const commitPattern = /^[a-f0-9]{7,40}$/u; const minCodeAgentFrontendTimeoutMs = 35000; -const defaultCodeAgentBackendTimeoutMs = 600000; -const minCodeAgentBackendTimeoutMs = 600000; +const defaultCodeAgentBackendTimeoutMs = 1200000; +const minCodeAgentBackendTimeoutMs = 1200000; export function normalizeCommit(value) { if (typeof value !== "string") return "unknown"; @@ -915,12 +915,12 @@ function summarizeCloudWebBuildFreshness(distFreshness, artifactCloudWeb) { } async function inspectCodeAgentTimeoutReadiness(repoRoot) { - const [appSource, chatSource] = await Promise.all([ + const [appSource, cloudApiEntrySource] = await Promise.all([ readFile(path.join(repoRoot, "web/hwlab-cloud-web/app.mjs"), "utf8"), - readFile(path.join(repoRoot, "internal/cloud/code-agent-chat.mjs"), "utf8") + readFile(path.join(repoRoot, "cmd/hwlab-cloud-api/main.mjs"), "utf8") ]); const frontendTimeoutMs = numericSourceConstant(appSource, "DEFAULT_CODE_AGENT_TIMEOUT_MS"); - const backendTimeoutMs = numericSourceConstant(chatSource, "DEFAULT_CODE_AGENT_TIMEOUT_MS"); + const backendTimeoutMs = parseTimeoutFallbackMs(cloudApiEntrySource, "HWLAB_CODE_AGENT_TIMEOUT_MS"); const frontendReady = Number.isInteger(frontendTimeoutMs) && frontendTimeoutMs >= minCodeAgentFrontendTimeoutMs; const backendReady = Number.isInteger(backendTimeoutMs) && backendTimeoutMs >= minCodeAgentBackendTimeoutMs; const transportWaitsForBackend = Number.isInteger(frontendTimeoutMs) && Number.isInteger(backendTimeoutMs) && frontendTimeoutMs > backendTimeoutMs; @@ -943,6 +943,13 @@ function numericSourceConstant(source, name) { return Number.parseInt(match[1].replace(/_/gu, ""), 10); } +function parseTimeoutFallbackMs(source, envName) { + const escaped = envName.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const match = source.match(new RegExp(`parseTimeout\\(process\\.env\\.${escaped},\\s*([0-9_]+)`, "u")); + if (!match) return null; + return Number.parseInt(match[1].replace(/_/gu, ""), 10); +} + async function inspectM3IoSourceReadiness(repoRoot) { const files = await listTrackedFiles(repoRoot); const candidateFiles = files.filter((file) => diff --git a/scripts/src/deploy-contract-plan.mjs b/scripts/src/deploy-contract-plan.mjs index 270f50f6..b0d01e14 100644 --- a/scripts/src/deploy-contract-plan.mjs +++ b/scripts/src/deploy-contract-plan.mjs @@ -13,8 +13,8 @@ import { const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const expectedPorts = Object.freeze({ frontend: 16666, api: 16667 }); -const codeAgentBackendTimeoutMs = "600000"; -const codeAgentTransportTimeoutMs = "660000"; +const codeAgentBackendTimeoutMs = "1200000"; +const codeAgentTransportTimeoutMs = "1260000"; const legacyPublicPorts = new Set([6666, 6667]); const duplicateMappingAllowed = new Set(["hwlab-box-simu", "hwlab-gateway-simu"]); const tomlBareKeyPattern = /^[A-Za-z0-9_-]+$/u; diff --git a/scripts/src/dev-cloud-workbench-smoke-lib.mjs b/scripts/src/dev-cloud-workbench-smoke-lib.mjs index cd06645e..b2c465ed 100644 --- a/scripts/src/dev-cloud-workbench-smoke-lib.mjs +++ b/scripts/src/dev-cloud-workbench-smoke-lib.mjs @@ -23,7 +23,7 @@ const webRoot = path.join(repoRoot, "web/hwlab-cloud-web"); const defaultLiveUrl = "http://74.48.78.17:16666/"; const helpOwner = "codex_1779444232735_1"; const legacyFailureWindowMs = 4500; -const codeAgentLongTimeoutMs = 600000; +const codeAgentLongTimeoutMs = 1800000; const localAgentFixtureDelayMs = 5200; const codeAgentE2ePrompts = Object.freeze([ { @@ -2657,7 +2657,8 @@ function hasCodeAgentLongTimeoutContract(files) { const app = files.app; return ( /DEFAULT_API_TIMEOUT_MS\s*=\s*4500/u.test(app) && - /DEFAULT_CODE_AGENT_TIMEOUT_MS\s*=\s*600000/u.test(app) && + /DEFAULT_CODE_AGENT_TIMEOUT_MS\s*=\s*1800000/u.test(app) && + /MAX_CODE_AGENT_TIMEOUT_MS\s*=\s*2400000/u.test(app) && /DEFAULT_CODE_AGENT_SUBMIT_TIMEOUT_MS\s*=\s*60000/u.test(app) && /async function sendAgentMessage[\s\S]*?fetchJson\("\/v1\/agent\/chat"[\s\S]*?timeoutMs:\s*CODE_AGENT_SUBMIT_TIMEOUT_MS/u.test(app) && /async function sendAgentMessage[\s\S]*?shortConnection:\s*true/u.test(app) && @@ -2782,7 +2783,8 @@ function hasCodeAgentConversationUxStates({ app, styles }) { !/function\s+messageAttributionPanel\s*\(/u.test(app) && !/codeAgentAttributionFromMessage/u.test(app) && /DEFAULT_API_TIMEOUT_MS\s*=\s*4500/u.test(app) && - /DEFAULT_CODE_AGENT_TIMEOUT_MS\s*=\s*600000/u.test(app) && + /DEFAULT_CODE_AGENT_TIMEOUT_MS\s*=\s*1800000/u.test(app) && + /MAX_CODE_AGENT_TIMEOUT_MS\s*=\s*2400000/u.test(app) && /timeoutMs:\s*CODE_AGENT_SUBMIT_TIMEOUT_MS/u.test(app) && /HWLAB_CLOUD_WEB_CONFIG/u.test(app) && /正在处理这次 Code Agent 请求/u.test(app) && diff --git a/web/hwlab-cloud-web/app.mjs b/web/hwlab-cloud-web/app.mjs index 0755b82a..c2036964 100644 --- a/web/hwlab-cloud-web/app.mjs +++ b/web/hwlab-cloud-web/app.mjs @@ -20,7 +20,8 @@ import { const DEFAULT_API_TIMEOUT_MS = 4500; const DEFAULT_LIVE_SURFACE_TIMEOUT_MS = 12000; -const DEFAULT_CODE_AGENT_TIMEOUT_MS = 600000; +const DEFAULT_CODE_AGENT_TIMEOUT_MS = 1800000; +const MAX_CODE_AGENT_TIMEOUT_MS = 2400000; const DEFAULT_CODE_AGENT_SUBMIT_TIMEOUT_MS = 60000; const DEFAULT_CODE_AGENT_CANCEL_TIMEOUT_MS = 30000; const DEFAULT_GATEWAY_SHELL_TIMEOUT_MS = 120000; @@ -307,8 +308,8 @@ function resolveUserCodeAgentTimeoutMs() { } catch { stored = null; } - return clampTimeoutMs(stored, { min, max: 1200000 }) - ?? resolveTimeoutMs("codeAgentTimeoutMs", DEFAULT_CODE_AGENT_TIMEOUT_MS, { min, max: 1200000 }); + return clampTimeoutMs(stored, { min, max: MAX_CODE_AGENT_TIMEOUT_MS }) + ?? resolveTimeoutMs("codeAgentTimeoutMs", DEFAULT_CODE_AGENT_TIMEOUT_MS, { min, max: MAX_CODE_AGENT_TIMEOUT_MS }); } function resolveUserGatewayShellTimeoutMs() { @@ -350,7 +351,7 @@ function initCodeAgentTimeoutControl() { renderCodeAgentSummary(); }); el.codeAgentTimeout.addEventListener("change", () => { - const next = clampTimeoutMs(el.codeAgentTimeout.value, { min: 30000, max: 1200000 }) ?? DEFAULT_CODE_AGENT_TIMEOUT_MS; + const next = clampTimeoutMs(el.codeAgentTimeout.value, { min: 30000, max: MAX_CODE_AGENT_TIMEOUT_MS }) ?? DEFAULT_CODE_AGENT_TIMEOUT_MS; CODE_AGENT_TIMEOUT_MS = next; try { window.localStorage?.setItem(CODE_AGENT_TIMEOUT_STORAGE_KEY, String(next)); @@ -514,7 +515,7 @@ function refreshRestoredCodeAgentTraces() { function reconcileRestoredCodeAgentResults() { for (const message of state.chatMessages) { if (!message.traceId || message.role !== "agent") continue; - if (["completed", "failed", "timeout", "canceled", "error"].includes(String(message.status ?? "").toLowerCase())) continue; + if (!shouldReconcileRestoredCodeAgentResult(message)) continue; reconcileCodeAgentResult(message.id, { quiet: true }); } } @@ -1816,10 +1817,33 @@ async function refreshTraceAfterResultPollError(traceId) { function shouldReconcileCodeAgentResult(message, runnerTrace = message?.runnerTrace) { if (!message?.traceId || message.role !== "agent") return false; if (state.resultReconciliationInFlight.has(message.traceId)) return false; - if (["completed", "failed", "timeout", "canceled", "error"].includes(String(message.status ?? "").toLowerCase())) return false; + const status = String(message.status ?? "").toLowerCase(); + if (["completed", "canceled"].includes(status)) return false; + if (["failed", "timeout", "error"].includes(status)) return traceHasTerminalResultEvent(runnerTrace) || isRecoverableCodeAgentTerminalMessage(message); return traceHasTerminalResultEvent(runnerTrace); } +function shouldReconcileRestoredCodeAgentResult(message) { + if (!message?.traceId || message.role !== "agent") return false; + if (state.resultReconciliationInFlight.has(message.traceId)) return false; + const status = String(message.status ?? "").toLowerCase(); + if (["completed", "canceled"].includes(status)) return false; + if (["failed", "timeout", "error"].includes(status)) return isRecoverableCodeAgentTerminalMessage(message) || traceHasTerminalResultEvent(message.runnerTrace); + return true; +} + +function isRecoverableCodeAgentTerminalMessage(message) { + const code = normalizeErrorCode(message?.error?.code); + return [ + "client_trace_idle_timeout", + "client_timeout", + "proxy_timeout", + "cloud_api_proxy_timeout", + "code_agent_background_failed", + "code_agent_session_evidence_missing" + ].includes(code) || isCodeAgentTimeoutError(message?.error, message?.text); +} + function traceHasTerminalResultEvent(runnerTrace) { const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; const terminalLabel = runnerTrace?.lastEvent?.label ?? events.at(-1)?.label; diff --git a/web/hwlab-cloud-web/index.html b/web/hwlab-cloud-web/index.html index 0e2c5405..3e598f05 100644 --- a/web/hwlab-cloud-web/index.html +++ b/web/hwlab-cloud-web/index.html @@ -213,9 +213,11 @@