From 6bf9f15fbcc29e9b74035ccff5e69e6585229a95 Mon Sep 17 00:00:00 2001 From: Code Queue Review Date: Sat, 23 May 2026 19:35:28 +0000 Subject: [PATCH] fix: harden code agent timeout discovery path --- cmd/hwlab-cloud-api/main.mjs | 14 +++++++- deploy/deploy.json | 5 ++- deploy/k8s/base/workloads.yaml | 12 +++++++ internal/cloud/server.test.mjs | 5 +++ internal/dev-entrypoint/http.mjs | 33 +++++++++++++++---- internal/dev-entrypoint/http.test.mjs | 9 ++++- scripts/dev-artifact-publish.mjs | 33 +++++++++++++++---- scripts/src/deploy-contract-plan.mjs | 19 +++++++++++ scripts/src/dev-cloud-workbench-smoke-lib.mjs | 14 +++++--- web/hwlab-cloud-web/scripts/check.mjs | 7 +++- 10 files changed, 131 insertions(+), 20 deletions(-) diff --git a/cmd/hwlab-cloud-api/main.mjs b/cmd/hwlab-cloud-api/main.mjs index 9bad80e3..7dcbd5a4 100644 --- a/cmd/hwlab-cloud-api/main.mjs +++ b/cmd/hwlab-cloud-api/main.mjs @@ -4,6 +4,10 @@ import { createCloudApiServer } from "../../internal/cloud/server.mjs"; const host = process.env.HWLAB_CLOUD_API_HOST || "0.0.0.0"; const port = Number.parseInt(process.env.HWLAB_CLOUD_API_PORT || process.env.PORT || "6667", 10); const commitId = process.env.HWLAB_COMMIT_ID || process.env.HWLAB_GIT_SHA; +const codeAgentTimeoutMs = parseTimeout(process.env.HWLAB_CODE_AGENT_TIMEOUT_MS, 120000, { + min: 1000, + max: 300000 +}); if (!process.env.HWLAB_IMAGE && commitId) { process.env.HWLAB_IMAGE = `ghcr.io/pikastech/hwlab-cloud-api:${commitId.slice(0, 7)}`; @@ -13,7 +17,15 @@ if (!process.env.HWLAB_IMAGE_TAG && commitId) { process.env.HWLAB_IMAGE_TAG = commitId.slice(0, 7); } -const server = createCloudApiServer(); +const server = createCloudApiServer({ + codeAgentTimeoutMs +}); + +function parseTimeout(value, fallback, { min, max }) { + const parsed = Number.parseInt(value || "", 10); + if (!Number.isInteger(parsed)) return fallback; + return Math.min(Math.max(parsed, min), max); +} server.listen(port, host, () => { const address = server.address(); diff --git a/deploy/deploy.json b/deploy/deploy.json index 444c12f5..f0924404 100644 --- a/deploy/deploy.json +++ b/deploy/deploy.json @@ -204,6 +204,7 @@ "HWLAB_M3_PATCH_PANEL_URL": "http://hwlab-patch-panel.hwlab-dev.svc.cluster.local:7301", "HWLAB_CODE_AGENT_PROVIDER": "openai", "HWLAB_CODE_AGENT_MODEL": "gpt-5.5", + "HWLAB_CODE_AGENT_TIMEOUT_MS": "120000", "HWLAB_CODE_AGENT_OPENAI_BASE_URL": "http://172.26.26.227:17680/v1/responses", "HWLAB_CODE_AGENT_HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", "HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED": "1", @@ -226,6 +227,7 @@ "env": { "HWLAB_ENVIRONMENT": "dev", "HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", + "HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS": "150000", "HWLAB_COMMIT_ID": "9190860", "HWLAB_IMAGE": "127.0.0.1:5000/hwlab/hwlab-cloud-web:9190860", "HWLAB_IMAGE_TAG": "9190860" @@ -362,7 +364,8 @@ "replicas": 1, "env": { "HWLAB_EDGE_LISTEN": "0.0.0.0:6667", - "HWLAB_EDGE_UPSTREAM": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667" + "HWLAB_EDGE_UPSTREAM": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", + "HWLAB_EDGE_PROXY_TIMEOUT_MS": "150000" } }, { diff --git a/deploy/k8s/base/workloads.yaml b/deploy/k8s/base/workloads.yaml index 623855ee..26368fc1 100644 --- a/deploy/k8s/base/workloads.yaml +++ b/deploy/k8s/base/workloads.yaml @@ -113,6 +113,10 @@ "name": "HWLAB_CODE_AGENT_MODEL", "value": "gpt-5.5" }, + { + "name": "HWLAB_CODE_AGENT_TIMEOUT_MS", + "value": "120000" + }, { "name": "HWLAB_CODE_AGENT_OPENAI_BASE_URL", "value": "http://172.26.26.227:17680/v1/responses" @@ -219,6 +223,10 @@ "name": "HWLAB_API_BASE_URL", "value": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667" }, + { + "name": "HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS", + "value": "150000" + }, { "name": "HWLAB_COMMIT_ID", "value": "9190860" @@ -791,6 +799,10 @@ { "name": "HWLAB_EDGE_UPSTREAM", "value": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667" + }, + { + "name": "HWLAB_EDGE_PROXY_TIMEOUT_MS", + "value": "150000" } ], "readinessProbe": { diff --git a/internal/cloud/server.test.mjs b/internal/cloud/server.test.mjs index 66da77a7..f773fa81 100644 --- a/internal/cloud/server.test.mjs +++ b/internal/cloud/server.test.mjs @@ -1268,6 +1268,8 @@ test("cloud api /v1/agent/chat parse and params errors use structured blocker en assert.equal(parse.body.error.traceId, "trc_parse_error"); assert.equal(parse.body.error.route, "/v1/agent/chat"); assert.match(parse.body.error.userMessage, /JSON/u); + assert.equal(Object.hasOwn(parse.body, "reply"), false); + assert.equal(JSON.stringify(parse.body).includes("sk-"), false); const invalid = await postAgentRaw(port, JSON.stringify([]), { "x-trace-id": "trc_invalid_params" }); assert.equal(invalid.status, 400); @@ -2155,6 +2157,9 @@ test("cloud api /v1/agent/chat reports OpenAI provider 502 and 503 as failed blo assert.match(payload.error.userMessage, /provider/u); assert.equal(payload.error.providerStatus, status); assert.match(payload.error.message, new RegExp(`HTTP ${status}`)); + assert.equal(payload.error.blocker.traceId, `trc_server-test-agent-chat-provider-${status}`); + assert.equal(payload.error.blocker.retryable, true); + assert.equal(payload.error.blocker.capabilityLevel, "blocked"); assert.equal(Object.hasOwn(payload, "reply"), false); } finally { await new Promise((resolve, reject) => { diff --git a/internal/dev-entrypoint/http.mjs b/internal/dev-entrypoint/http.mjs index e8d7278f..ff83684e 100644 --- a/internal/dev-entrypoint/http.mjs +++ b/internal/dev-entrypoint/http.mjs @@ -106,6 +106,7 @@ export function listen(server, { serviceId, host, port }) { export function proxyHttpRequest({ request, response, upstream, timeoutMs = DEFAULT_PROXY_TIMEOUT_MS }) { const target = new URL(request.url || "/", upstream); + const traceId = request.headers["x-trace-id"] ?? null; let timedOut = false; const timeout = setTimeout(() => { timedOut = true; @@ -136,17 +137,37 @@ export function proxyHttpRequest({ request, response, upstream, timeoutMs = DEFA return; } settled = true; + const code = timedOut ? "proxy_timeout" : "upstream_unavailable"; + const category = timedOut ? "timeout" : "proxy"; + const userMessage = timedOut + ? `Code Agent 代理等待上游超过 ${timeoutMs}ms；输入已保留，可稍后重试。` + : "Code Agent 代理暂时无法连接上游；输入已保留，可稍后重试。"; sendJson(response, 502, { status: "failed", error: { - code: timedOut ? "proxy_timeout" : "upstream_unavailable", - category: timedOut ? "timeout" : "proxy", - message: error.message, - timeoutMs: timedOut ? timeoutMs : null + code, + layer: "proxy", + category, + retryable: true, + userMessage, + message: userMessage, + timeoutMs: timedOut ? timeoutMs : null, + traceId, + route: request.url || null, + blocker: { + code, + layer: "proxy", + category, + retryable: true, + summary: userMessage, + userMessage, + traceId, + route: request.url || null + } }, upstream, - traceId: request.headers["x-trace-id"] ?? null, - message: error.message + traceId, + message: userMessage }); }); diff --git a/internal/dev-entrypoint/http.test.mjs b/internal/dev-entrypoint/http.test.mjs index 66e8ce3f..eb39249b 100644 --- a/internal/dev-entrypoint/http.test.mjs +++ b/internal/dev-entrypoint/http.test.mjs @@ -81,10 +81,17 @@ test("dev entrypoint proxy returns 502 when hard timeout expires", async () => { const payload = await response.json(); assert.equal(payload.status, "failed"); assert.equal(payload.error.code, "proxy_timeout"); + assert.equal(payload.error.layer, "proxy"); assert.equal(payload.error.category, "timeout"); + assert.equal(payload.error.retryable, true); assert.equal(payload.error.timeoutMs, 50); + assert.equal(payload.error.traceId, "trc_dev-entrypoint-proxy-timeout"); + assert.match(payload.error.userMessage, /超过 50ms/u); + assert.match(payload.error.message, /输入已保留/u); + assert.equal(payload.error.blocker.code, "proxy_timeout"); + assert.equal(payload.error.blocker.retryable, true); assert.equal(payload.traceId, "trc_dev-entrypoint-proxy-timeout"); - assert.match(payload.message, /timed out after 50ms/u); + assert.match(payload.message, /Code Agent 代理等待上游超过 50ms/u); } finally { await close(proxy); await close(upstream); diff --git a/scripts/dev-artifact-publish.mjs b/scripts/dev-artifact-publish.mjs index 6f619f79..f4d636b9 100644 --- a/scripts/dev-artifact-publish.mjs +++ b/scripts/dev-artifact-publish.mjs @@ -671,18 +671,39 @@ async function proxyCloudApi(request, response, url) { response.end(upstream.body); } catch (error) { const timedOut = /timed out after \d+ms/iu.test(error.message); + const code = timedOut ? "cloud_api_proxy_timeout" : "cloud_api_proxy_failed"; + const category = timedOut ? "timeout" : "proxy"; + const traceId = request.headers["x-trace-id"] || null; + const userMessage = timedOut + ? "Code Agent 代理等待 cloud-api 超过 " + cloudApiProxyTimeoutMs + "ms；输入已保留，可稍后重试。" + : "Code Agent 代理暂时无法连接 cloud-api；输入已保留，可稍后重试。"; sendJson(response, 502, { status: "failed", error: { - code: timedOut ? "cloud_api_proxy_timeout" : "cloud_api_proxy_failed", - category: timedOut ? "timeout" : "proxy", - message: error.message, - timeoutMs: timedOut ? cloudApiProxyTimeoutMs : null + code, + layer: "proxy", + category, + retryable: true, + userMessage, + message: userMessage, + timeoutMs: timedOut ? cloudApiProxyTimeoutMs : null, + traceId, + route: url.pathname, + blocker: { + code, + layer: "proxy", + category, + retryable: true, + summary: userMessage, + userMessage, + traceId, + route: url.pathname + } }, serviceId, target: target.href.replace(/\/\/[^/@]*@/u, "//***@"), - traceId: request.headers["x-trace-id"] || null, - reason: error.message + traceId, + reason: userMessage }); } } diff --git a/scripts/src/deploy-contract-plan.mjs b/scripts/src/deploy-contract-plan.mjs index 1290acdd..56dcd4c1 100644 --- a/scripts/src/deploy-contract-plan.mjs +++ b/scripts/src/deploy-contract-plan.mjs @@ -227,9 +227,12 @@ function validateSource(ctx, manifest) { const tunnelClient = servicesById.get("hwlab-tunnel-client"); const cli = servicesById.get("hwlab-cli"); const patchPanel = servicesById.get("hwlab-patch-panel"); + const cloudWeb = servicesById.get("hwlab-cloud-web"); + const edgeProxy = servicesById.get("hwlab-edge-proxy"); expectEqual(ctx, cloudApi?.env?.HWLAB_PUBLIC_ENDPOINT, endpoints.api?.url, "$.services.hwlab-cloud-api.env.HWLAB_PUBLIC_ENDPOINT", "cloud API public endpoint env"); validateCloudApiDbSource(ctx, cloudApi?.env ?? {}); validateCloudApiCodeAgentSource(ctx, cloudApi?.env ?? {}); + validateCodeAgentProxyTimeoutSource(ctx, { cloudWeb, edgeProxy }); validateM3PatchPanelSource(ctx, patchPanel); expectEqual(ctx, cli?.env?.HWLAB_CLI_ENDPOINT, endpoints.api?.url, "$.services.hwlab-cli.env.HWLAB_CLI_ENDPOINT", "CLI endpoint env"); expectEqual(ctx, tunnelClient?.env?.HWLAB_FRP_PUBLIC_PORT, String(expectedPorts.api), "$.services.hwlab-tunnel-client.env.HWLAB_FRP_PUBLIC_PORT", "tunnel API public port env"); @@ -374,6 +377,7 @@ function validateCloudApiCodeAgentSource(ctx, env) { const contract = DEV_CODE_AGENT_PROVIDER_CONTRACT; expectEqual(ctx, env.HWLAB_CODE_AGENT_PROVIDER, contract.provider, "$.services.hwlab-cloud-api.env.HWLAB_CODE_AGENT_PROVIDER", "cloud API Code Agent provider"); expectEqual(ctx, env.HWLAB_CODE_AGENT_MODEL, contract.model, "$.services.hwlab-cloud-api.env.HWLAB_CODE_AGENT_MODEL", "cloud API Code Agent model"); + expectEqual(ctx, env.HWLAB_CODE_AGENT_TIMEOUT_MS, "120000", "$.services.hwlab-cloud-api.env.HWLAB_CODE_AGENT_TIMEOUT_MS", "cloud API Code Agent hard timeout budget"); expectEqual( ctx, env.HWLAB_CODE_AGENT_OPENAI_BASE_URL, @@ -406,6 +410,11 @@ function validateCloudApiCodeAgentSource(ctx, env) { expectEqual(ctx, env.OPENAI_API_KEY, codeAgentSecretRefPlaceholder(), "$.services.hwlab-cloud-api.env.OPENAI_API_KEY", "cloud API Code Agent OpenAI Secret reference"); } +function validateCodeAgentProxyTimeoutSource(ctx, { cloudWeb, edgeProxy }) { + expectEqual(ctx, cloudWeb?.env?.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS, "150000", "$.services.hwlab-cloud-web.env.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS", "Cloud Web same-origin Code Agent proxy timeout budget"); + expectEqual(ctx, edgeProxy?.env?.HWLAB_EDGE_PROXY_TIMEOUT_MS, "150000", "$.services.hwlab-edge-proxy.env.HWLAB_EDGE_PROXY_TIMEOUT_MS", "edge proxy Code Agent timeout budget"); +} + function renderPlan(source) { const proxyPorts = [...new Set(source.proxies.map((proxy) => Number(proxy.remotePort)))].sort((a, b) => a - b); return { @@ -505,6 +514,7 @@ function validateK3sArtifacts(ctx, source, rendered, services, workloads, health validateCloudApiDbOptionalAliasArtifacts(ctx, services, workloads); validateCloudApiCodeAgentArtifacts(ctx, workloads); + validateCodeAgentProxyTimeoutArtifacts(ctx, workloads); expectEqual(ctx, healthContract.data?.endpoint, source.endpoints.api?.url, "deploy/k8s/dev/health-contract.yaml.data.endpoint", "health contract endpoint"); expectEqual(ctx, masterEdge.endpoint, source.endpoints.api?.url, "deploy/master-edge/health-contract.json.endpoint", "master edge endpoint"); @@ -543,6 +553,7 @@ function validateCloudApiCodeAgentArtifacts(ctx, workloads) { const env = new Map((container?.env ?? []).map((entry) => [entry.name, entry])); expectEqual(ctx, env.get("HWLAB_CODE_AGENT_PROVIDER")?.value, contract.provider, "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.HWLAB_CODE_AGENT_PROVIDER", "cloud API workload Code Agent provider"); expectEqual(ctx, env.get("HWLAB_CODE_AGENT_MODEL")?.value, contract.model, "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.HWLAB_CODE_AGENT_MODEL", "cloud API workload Code Agent model"); + expectEqual(ctx, env.get("HWLAB_CODE_AGENT_TIMEOUT_MS")?.value, "120000", "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.HWLAB_CODE_AGENT_TIMEOUT_MS", "cloud API workload Code Agent hard timeout budget"); expectEqual( ctx, env.get("HWLAB_CODE_AGENT_OPENAI_BASE_URL")?.value, @@ -576,6 +587,14 @@ function validateCloudApiCodeAgentArtifacts(ctx, workloads) { expectEqual(ctx, env.get(secretRef.env)?.valueFrom?.secretKeyRef?.key, secretRef.secretKey, "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.OPENAI_API_KEY.secretKeyRef.key", "cloud API workload Code Agent Secret key"); } +function validateCodeAgentProxyTimeoutArtifacts(ctx, workloads) { + const workloadItems = mapByServiceId(listItems(workloads)); + const cloudWebEnv = new Map((workloadItems.get("hwlab-cloud-web")?.spec?.template?.spec?.containers?.[0]?.env ?? []).map((entry) => [entry.name, entry.value])); + const edgeProxyEnv = new Map((workloadItems.get("hwlab-edge-proxy")?.spec?.template?.spec?.containers?.[0]?.env ?? []).map((entry) => [entry.name, entry.value])); + expectEqual(ctx, cloudWebEnv.get("HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS"), "150000", "deploy/k8s/base/workloads.yaml.hwlab-cloud-web.env.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS", "Cloud Web workload same-origin Code Agent proxy timeout budget"); + expectEqual(ctx, edgeProxyEnv.get("HWLAB_EDGE_PROXY_TIMEOUT_MS"), "150000", "deploy/k8s/base/workloads.yaml.hwlab-edge-proxy.env.HWLAB_EDGE_PROXY_TIMEOUT_MS", "edge proxy workload Code Agent timeout budget"); +} + async function buildPlan() { const ctx = { checks: 0, diagnostics: [] }; const [manifest, services, workloads, healthContract, masterEdge, frpc, frps] = await Promise.all([ diff --git a/scripts/src/dev-cloud-workbench-smoke-lib.mjs b/scripts/src/dev-cloud-workbench-smoke-lib.mjs index 5fbd9137..df1ae903 100644 --- a/scripts/src/dev-cloud-workbench-smoke-lib.mjs +++ b/scripts/src/dev-cloud-workbench-smoke-lib.mjs @@ -604,7 +604,7 @@ function runStaticSmoke() { evidence: codeAgentQuickPromptFixtures.flatMap((prompt) => [prompt.id, prompt.label, prompt.prompt]) }); - addCheck(checks, blockers, "code-agent-long-timeout-contract", hasCodeAgentLongTimeoutContract(files.app), "Code Agent chat uses a dedicated long timeout and does not reuse the old 4500ms API timeout.", { + addCheck(checks, blockers, "code-agent-long-timeout-contract", hasCodeAgentLongTimeoutContract(files), "Code Agent chat uses a dedicated long timeout and does not reuse the old 4500ms API timeout.", { blocker: "runtime_blocker", evidence: ["DEFAULT_API_TIMEOUT_MS=4500 for light probes", `DEFAULT_CODE_AGENT_TIMEOUT_MS=${codeAgentLongTimeoutMs}`, "sendAgentMessage passes timeoutMs"] }); @@ -1689,7 +1689,8 @@ function readStaticFiles() { runtime: readText("web/hwlab-cloud-web/runtime.mjs"), panel: readText("web/hwlab-cloud-web/workbench-hardware-panel.mjs"), codeAgentM3Evidence: readText("web/hwlab-cloud-web/code-agent-m3-evidence.mjs"), - help: readText("web/hwlab-cloud-web/help.md") + help: readText("web/hwlab-cloud-web/help.md"), + artifactPublisher: readText("scripts/dev-artifact-publish.mjs") }; } @@ -2561,7 +2562,8 @@ function hasCodeAgentCompletedEvidenceVisibility({ app, codeAgentM3Evidence }) { ); } -function hasCodeAgentLongTimeoutContract(app) { +function hasCodeAgentLongTimeoutContract(files) { + const app = files.app; return ( /DEFAULT_API_TIMEOUT_MS\s*=\s*4500/u.test(app) && /DEFAULT_CODE_AGENT_TIMEOUT_MS\s*=\s*150000/u.test(app) && @@ -2569,7 +2571,11 @@ function hasCodeAgentLongTimeoutContract(app) { /timeoutMs\s*=\s*API_TIMEOUT_MS/u.test(functionBody(app, "fetchJson")) && /timeoutName\s*=\s*path/u.test(functionBody(app, "fetchJson")) && /HWLAB_CLOUD_WEB_CONFIG/u.test(app) && - /codeAgentTimeoutMs/u.test(app) + /codeAgentTimeoutMs/u.test(app) && + /Code Agent 代理等待 cloud-api 超过/u.test(files.artifactPublisher ?? "") && + /输入已保留，可稍后重试/u.test(files.artifactPublisher ?? "") && + /blocker:\s*\{/u.test(files.artifactPublisher ?? "") && + /retryable:\s*true/u.test(files.artifactPublisher ?? "") ); } diff --git a/web/hwlab-cloud-web/scripts/check.mjs b/web/hwlab-cloud-web/scripts/check.mjs index 37a93bd4..75be8345 100644 --- a/web/hwlab-cloud-web/scripts/check.mjs +++ b/web/hwlab-cloud-web/scripts/check.mjs @@ -934,7 +934,12 @@ assert.match(artifactPublisher, /"evidence\.record\.query"/); assert.match(artifactPublisher, /readonly_rpc_required/); assert.match(artifactPublisher, /cloud_api_proxy_failed/); assert.match(artifactPublisher, /cloud_api_proxy_timeout/); -assert.match(artifactPublisher, /traceId:\s*request\.headers\["x-trace-id"\]/); +assert.match(artifactPublisher, /Code Agent 代理等待 cloud-api 超过/); +assert.match(artifactPublisher, /输入已保留，可稍后重试/); +assert.match(artifactPublisher, /retryable:\s*true/); +assert.match(artifactPublisher, /blocker:\s*\{/); +assert.match(artifactPublisher, /const traceId = request\.headers\["x-trace-id"\] \|\| null/); +assert.match(artifactPublisher, /traceId,\s*\n\s*route: url\.pathname/); assert.match(artifactPublisher, /url\.pathname\.replace\(\/\\\/\+\$\/,\s*""\) \|\| "\/"/); assert.match(artifactPublisher, /routePath === "\/gate"/); assert.match(artifactPublisher, /routePath === "\/diagnostics\/gate"/);