mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden code agent timeout discovery path

Browse Source
This commit is contained in:
Code Queue Review
2026-05-23 19:35:28 +00:00
parent 182221a6a2
commit 6bf9f15fbc
10 changed files with 131 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/hwlab-cloud-api/main.mjs
+13 -1
View File
@@ -4,6 +4,10 @@ import { createCloudApiServer } from "../../internal/cloud/server.mjs";
const host = process.env.HWLAB_CLOUD_API_HOST || "0.0.0.0";
const port = Number.parseInt(process.env.HWLAB_CLOUD_API_PORT || process.env.PORT || "6667", 10);
const commitId = process.env.HWLAB_COMMIT_ID || process.env.HWLAB_GIT_SHA;
const codeAgentTimeoutMs = parseTimeout(process.env.HWLAB_CODE_AGENT_TIMEOUT_MS, 120000, {
min: 1000,
max: 300000
});
if (!process.env.HWLAB_IMAGE && commitId) {
process.env.HWLAB_IMAGE = `ghcr.io/pikastech/hwlab-cloud-api:${commitId.slice(0, 7)}`;
@@ -13,7 +17,15 @@ if (!process.env.HWLAB_IMAGE_TAG && commitId) {
process.env.HWLAB_IMAGE_TAG = commitId.slice(0, 7);
}
const server = createCloudApiServer();
const server = createCloudApiServer({
codeAgentTimeoutMs
});
function parseTimeout(value, fallback, { min, max }) {
const parsed = Number.parseInt(value || "", 10);
if (!Number.isInteger(parsed)) return fallback;
return Math.min(Math.max(parsed, min), max);
}
server.listen(port, host, () => {
const address = server.address();
deploy/deploy.json
+4 -1
View File
@@ -204,6 +204,7 @@
"HWLAB_M3_PATCH_PANEL_URL": "http://hwlab-patch-panel.hwlab-dev.svc.cluster.local:7301",
"HWLAB_CODE_AGENT_PROVIDER": "openai",
"HWLAB_CODE_AGENT_MODEL": "gpt-5.5",
"HWLAB_CODE_AGENT_TIMEOUT_MS": "120000",
"HWLAB_CODE_AGENT_OPENAI_BASE_URL": "http://172.26.26.227:17680/v1/responses",
"HWLAB_CODE_AGENT_HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667",
"HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED": "1",
@@ -226,6 +227,7 @@
"env": {
"HWLAB_ENVIRONMENT": "dev",
"HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667",
"HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS": "150000",
"HWLAB_COMMIT_ID": "9190860",
"HWLAB_IMAGE": "127.0.0.1:5000/hwlab/hwlab-cloud-web:9190860",
"HWLAB_IMAGE_TAG": "9190860"
@@ -362,7 +364,8 @@
"replicas": 1,
"env": {
"HWLAB_EDGE_LISTEN": "0.0.0.0:6667",
"HWLAB_EDGE_UPSTREAM": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667"
"HWLAB_EDGE_UPSTREAM": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667",
"HWLAB_EDGE_PROXY_TIMEOUT_MS": "150000"
}
},
{
deploy/k8s/base/workloads.yaml
+12
View File
@@ -113,6 +113,10 @@
"name": "HWLAB_CODE_AGENT_MODEL",
"value": "gpt-5.5"
},
{
"name": "HWLAB_CODE_AGENT_TIMEOUT_MS",
"value": "120000"
},
{
"name": "HWLAB_CODE_AGENT_OPENAI_BASE_URL",
"value": "http://172.26.26.227:17680/v1/responses"
@@ -219,6 +223,10 @@
"name": "HWLAB_API_BASE_URL",
"value": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667"
},
{
"name": "HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS",
"value": "150000"
},
{
"name": "HWLAB_COMMIT_ID",
"value": "9190860"
@@ -791,6 +799,10 @@
{
"name": "HWLAB_EDGE_UPSTREAM",
"value": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667"
},
{
"name": "HWLAB_EDGE_PROXY_TIMEOUT_MS",
"value": "150000"
}
],
"readinessProbe": {
internal/cloud/server.test.mjs
+5
View File
@@ -1268,6 +1268,8 @@ test("cloud api /v1/agent/chat parse and params errors use structured blocker en
assert.equal(parse.body.error.traceId, "trc_parse_error");
assert.equal(parse.body.error.route, "/v1/agent/chat");
assert.match(parse.body.error.userMessage, /JSON/u);
assert.equal(Object.hasOwn(parse.body, "reply"), false);
assert.equal(JSON.stringify(parse.body).includes("sk-"), false);
const invalid = await postAgentRaw(port, JSON.stringify([]), { "x-trace-id": "trc_invalid_params" });
assert.equal(invalid.status, 400);
@@ -2155,6 +2157,9 @@ test("cloud api /v1/agent/chat reports OpenAI provider 502 and 503 as failed blo
assert.match(payload.error.userMessage, /provider/u);
assert.equal(payload.error.providerStatus, status);
assert.match(payload.error.message, new RegExp(`HTTP ${status}`));
assert.equal(payload.error.blocker.traceId, `trc_server-test-agent-chat-provider-${status}`);
assert.equal(payload.error.blocker.retryable, true);
assert.equal(payload.error.blocker.capabilityLevel, "blocked");
assert.equal(Object.hasOwn(payload, "reply"), false);
} finally {
await new Promise((resolve, reject) => {
internal/dev-entrypoint/http.mjs
+27 -6
View File
@@ -106,6 +106,7 @@ export function listen(server, { serviceId, host, port }) {
export function proxyHttpRequest({ request, response, upstream, timeoutMs = DEFAULT_PROXY_TIMEOUT_MS }) {
const target = new URL(request.url || "/", upstream);
const traceId = request.headers["x-trace-id"] ?? null;
let timedOut = false;
const timeout = setTimeout(() => {
timedOut = true;
@@ -136,17 +137,37 @@ export function proxyHttpRequest({ request, response, upstream, timeoutMs = DEFA
return;
}
settled = true;
const code = timedOut ? "proxy_timeout" : "upstream_unavailable";
const category = timedOut ? "timeout" : "proxy";
const userMessage = timedOut
? `Code Agent 代理等待上游超过 ${timeoutMs}ms；输入已保留，可稍后重试。`
: "Code Agent 代理暂时无法连接上游；输入已保留，可稍后重试。";
sendJson(response, 502, {
status: "failed",
error: {
code: timedOut ? "proxy_timeout" : "upstream_unavailable",
category: timedOut ? "timeout" : "proxy",
message: error.message,
timeoutMs: timedOut ? timeoutMs : null
code,
layer: "proxy",
category,
retryable: true,
userMessage,
message: userMessage,
timeoutMs: timedOut ? timeoutMs : null,
traceId,
route: request.url || null,
blocker: {
code,
layer: "proxy",
category,
retryable: true,
summary: userMessage,
userMessage,
traceId,
route: request.url || null
}
},
upstream,
traceId: request.headers["x-trace-id"] ?? null,
message: error.message
traceId,
message: userMessage
});
});
internal/dev-entrypoint/http.test.mjs
+8 -1
View File
@@ -81,10 +81,17 @@ test("dev entrypoint proxy returns 502 when hard timeout expires", async () => {
const payload = await response.json();
assert.equal(payload.status, "failed");
assert.equal(payload.error.code, "proxy_timeout");
assert.equal(payload.error.layer, "proxy");
assert.equal(payload.error.category, "timeout");
assert.equal(payload.error.retryable, true);
assert.equal(payload.error.timeoutMs, 50);
assert.equal(payload.error.traceId, "trc_dev-entrypoint-proxy-timeout");
assert.match(payload.error.userMessage, /超过 50ms/u);
assert.match(payload.error.message, /输入已保留/u);
assert.equal(payload.error.blocker.code, "proxy_timeout");
assert.equal(payload.error.blocker.retryable, true);
assert.equal(payload.traceId, "trc_dev-entrypoint-proxy-timeout");
assert.match(payload.message, /timed out after 50ms/u);
assert.match(payload.message, /Code Agent 代理等待上游超过 50ms/u);
} finally {
await close(proxy);
await close(upstream);
scripts/dev-artifact-publish.mjs
+27 -6
View File
@@ -671,18 +671,39 @@ async function proxyCloudApi(request, response, url) {
response.end(upstream.body);
} catch (error) {
const timedOut = /timed out after \d+ms/iu.test(error.message);
const code = timedOut ? "cloud_api_proxy_timeout" : "cloud_api_proxy_failed";
const category = timedOut ? "timeout" : "proxy";
const traceId = request.headers["x-trace-id"] || null;
const userMessage = timedOut
? "Code Agent 代理等待 cloud-api 超过 " + cloudApiProxyTimeoutMs + "ms；输入已保留，可稍后重试。"
: "Code Agent 代理暂时无法连接 cloud-api；输入已保留，可稍后重试。";
sendJson(response, 502, {
status: "failed",
error: {
code: timedOut ? "cloud_api_proxy_timeout" : "cloud_api_proxy_failed",
category: timedOut ? "timeout" : "proxy",
message: error.message,
timeoutMs: timedOut ? cloudApiProxyTimeoutMs : null
code,
layer: "proxy",
category,
retryable: true,
userMessage,
message: userMessage,
timeoutMs: timedOut ? cloudApiProxyTimeoutMs : null,
traceId,
route: url.pathname,
blocker: {
code,
layer: "proxy",
category,
retryable: true,
summary: userMessage,
userMessage,
traceId,
route: url.pathname
}
},
serviceId,
target: target.href.replace(/\/\/[^/@]*@/u, "//***@"),
traceId: request.headers["x-trace-id"] || null,
reason: error.message
traceId,
reason: userMessage
});
}
}
scripts/src/deploy-contract-plan.mjs
+19
View File
@@ -227,9 +227,12 @@ function validateSource(ctx, manifest) {
const tunnelClient = servicesById.get("hwlab-tunnel-client");
const cli = servicesById.get("hwlab-cli");
const patchPanel = servicesById.get("hwlab-patch-panel");
const cloudWeb = servicesById.get("hwlab-cloud-web");
const edgeProxy = servicesById.get("hwlab-edge-proxy");
expectEqual(ctx, cloudApi?.env?.HWLAB_PUBLIC_ENDPOINT, endpoints.api?.url, "$.services.hwlab-cloud-api.env.HWLAB_PUBLIC_ENDPOINT", "cloud API public endpoint env");
validateCloudApiDbSource(ctx, cloudApi?.env ?? {});
validateCloudApiCodeAgentSource(ctx, cloudApi?.env ?? {});
validateCodeAgentProxyTimeoutSource(ctx, { cloudWeb, edgeProxy });
validateM3PatchPanelSource(ctx, patchPanel);
expectEqual(ctx, cli?.env?.HWLAB_CLI_ENDPOINT, endpoints.api?.url, "$.services.hwlab-cli.env.HWLAB_CLI_ENDPOINT", "CLI endpoint env");
expectEqual(ctx, tunnelClient?.env?.HWLAB_FRP_PUBLIC_PORT, String(expectedPorts.api), "$.services.hwlab-tunnel-client.env.HWLAB_FRP_PUBLIC_PORT", "tunnel API public port env");
@@ -374,6 +377,7 @@ function validateCloudApiCodeAgentSource(ctx, env) {
const contract = DEV_CODE_AGENT_PROVIDER_CONTRACT;
expectEqual(ctx, env.HWLAB_CODE_AGENT_PROVIDER, contract.provider, "$.services.hwlab-cloud-api.env.HWLAB_CODE_AGENT_PROVIDER", "cloud API Code Agent provider");
expectEqual(ctx, env.HWLAB_CODE_AGENT_MODEL, contract.model, "$.services.hwlab-cloud-api.env.HWLAB_CODE_AGENT_MODEL", "cloud API Code Agent model");
expectEqual(ctx, env.HWLAB_CODE_AGENT_TIMEOUT_MS, "120000", "$.services.hwlab-cloud-api.env.HWLAB_CODE_AGENT_TIMEOUT_MS", "cloud API Code Agent hard timeout budget");
expectEqual(
ctx,
env.HWLAB_CODE_AGENT_OPENAI_BASE_URL,
@@ -406,6 +410,11 @@ function validateCloudApiCodeAgentSource(ctx, env) {
expectEqual(ctx, env.OPENAI_API_KEY, codeAgentSecretRefPlaceholder(), "$.services.hwlab-cloud-api.env.OPENAI_API_KEY", "cloud API Code Agent OpenAI Secret reference");
}
function validateCodeAgentProxyTimeoutSource(ctx, { cloudWeb, edgeProxy }) {
expectEqual(ctx, cloudWeb?.env?.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS, "150000", "$.services.hwlab-cloud-web.env.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS", "Cloud Web same-origin Code Agent proxy timeout budget");
expectEqual(ctx, edgeProxy?.env?.HWLAB_EDGE_PROXY_TIMEOUT_MS, "150000", "$.services.hwlab-edge-proxy.env.HWLAB_EDGE_PROXY_TIMEOUT_MS", "edge proxy Code Agent timeout budget");
}
function renderPlan(source) {
const proxyPorts = [...new Set(source.proxies.map((proxy) => Number(proxy.remotePort)))].sort((a, b) => a - b);
return {
@@ -505,6 +514,7 @@ function validateK3sArtifacts(ctx, source, rendered, services, workloads, health
validateCloudApiDbOptionalAliasArtifacts(ctx, services, workloads);
validateCloudApiCodeAgentArtifacts(ctx, workloads);
validateCodeAgentProxyTimeoutArtifacts(ctx, workloads);
expectEqual(ctx, healthContract.data?.endpoint, source.endpoints.api?.url, "deploy/k8s/dev/health-contract.yaml.data.endpoint", "health contract endpoint");
expectEqual(ctx, masterEdge.endpoint, source.endpoints.api?.url, "deploy/master-edge/health-contract.json.endpoint", "master edge endpoint");
@@ -543,6 +553,7 @@ function validateCloudApiCodeAgentArtifacts(ctx, workloads) {
const env = new Map((container?.env ?? []).map((entry) => [entry.name, entry]));
expectEqual(ctx, env.get("HWLAB_CODE_AGENT_PROVIDER")?.value, contract.provider, "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.HWLAB_CODE_AGENT_PROVIDER", "cloud API workload Code Agent provider");
expectEqual(ctx, env.get("HWLAB_CODE_AGENT_MODEL")?.value, contract.model, "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.HWLAB_CODE_AGENT_MODEL", "cloud API workload Code Agent model");
expectEqual(ctx, env.get("HWLAB_CODE_AGENT_TIMEOUT_MS")?.value, "120000", "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.HWLAB_CODE_AGENT_TIMEOUT_MS", "cloud API workload Code Agent hard timeout budget");
expectEqual(
ctx,
env.get("HWLAB_CODE_AGENT_OPENAI_BASE_URL")?.value,
@@ -576,6 +587,14 @@ function validateCloudApiCodeAgentArtifacts(ctx, workloads) {
expectEqual(ctx, env.get(secretRef.env)?.valueFrom?.secretKeyRef?.key, secretRef.secretKey, "deploy/k8s/base/workloads.yaml.hwlab-cloud-api.env.OPENAI_API_KEY.secretKeyRef.key", "cloud API workload Code Agent Secret key");
}
function validateCodeAgentProxyTimeoutArtifacts(ctx, workloads) {
const workloadItems = mapByServiceId(listItems(workloads));
const cloudWebEnv = new Map((workloadItems.get("hwlab-cloud-web")?.spec?.template?.spec?.containers?.[0]?.env ?? []).map((entry) => [entry.name, entry.value]));
const edgeProxyEnv = new Map((workloadItems.get("hwlab-edge-proxy")?.spec?.template?.spec?.containers?.[0]?.env ?? []).map((entry) => [entry.name, entry.value]));
expectEqual(ctx, cloudWebEnv.get("HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS"), "150000", "deploy/k8s/base/workloads.yaml.hwlab-cloud-web.env.HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS", "Cloud Web workload same-origin Code Agent proxy timeout budget");
expectEqual(ctx, edgeProxyEnv.get("HWLAB_EDGE_PROXY_TIMEOUT_MS"), "150000", "deploy/k8s/base/workloads.yaml.hwlab-edge-proxy.env.HWLAB_EDGE_PROXY_TIMEOUT_MS", "edge proxy workload Code Agent timeout budget");
}
async function buildPlan() {
const ctx = { checks: 0, diagnostics: [] };
const [manifest, services, workloads, healthContract, masterEdge, frpc, frps] = await Promise.all([
scripts/src/dev-cloud-workbench-smoke-lib.mjs
+10 -4
View File
@@ -604,7 +604,7 @@ function runStaticSmoke() {
evidence: codeAgentQuickPromptFixtures.flatMap((prompt) => [prompt.id, prompt.label, prompt.prompt])
});
addCheck(checks, blockers, "code-agent-long-timeout-contract", hasCodeAgentLongTimeoutContract(files.app), "Code Agent chat uses a dedicated long timeout and does not reuse the old 4500ms API timeout.", {
addCheck(checks, blockers, "code-agent-long-timeout-contract", hasCodeAgentLongTimeoutContract(files), "Code Agent chat uses a dedicated long timeout and does not reuse the old 4500ms API timeout.", {
blocker: "runtime_blocker",
evidence: ["DEFAULT_API_TIMEOUT_MS=4500 for light probes", `DEFAULT_CODE_AGENT_TIMEOUT_MS=${codeAgentLongTimeoutMs}`, "sendAgentMessage passes timeoutMs"]
});
@@ -1689,7 +1689,8 @@ function readStaticFiles() {
runtime: readText("web/hwlab-cloud-web/runtime.mjs"),
panel: readText("web/hwlab-cloud-web/workbench-hardware-panel.mjs"),
codeAgentM3Evidence: readText("web/hwlab-cloud-web/code-agent-m3-evidence.mjs"),
help: readText("web/hwlab-cloud-web/help.md")
help: readText("web/hwlab-cloud-web/help.md"),
artifactPublisher: readText("scripts/dev-artifact-publish.mjs")
};
}
@@ -2561,7 +2562,8 @@ function hasCodeAgentCompletedEvidenceVisibility({ app, codeAgentM3Evidence }) {
);
}
function hasCodeAgentLongTimeoutContract(app) {
function hasCodeAgentLongTimeoutContract(files) {
const app = files.app;
return (
/DEFAULT_API_TIMEOUT_MS\s*=\s*4500/u.test(app) &&
/DEFAULT_CODE_AGENT_TIMEOUT_MS\s*=\s*150000/u.test(app) &&
@@ -2569,7 +2571,11 @@ function hasCodeAgentLongTimeoutContract(app) {
/timeoutMs\s*=\s*API_TIMEOUT_MS/u.test(functionBody(app, "fetchJson")) &&
/timeoutName\s*=\s*path/u.test(functionBody(app, "fetchJson")) &&
/HWLAB_CLOUD_WEB_CONFIG/u.test(app) &&
/codeAgentTimeoutMs/u.test(app)
/codeAgentTimeoutMs/u.test(app) &&
/Code Agent 代理等待 cloud-api 超过/u.test(files.artifactPublisher ?? "") &&
/输入已保留，可稍后重试/u.test(files.artifactPublisher ?? "") &&
/blocker:\s*\{/u.test(files.artifactPublisher ?? "") &&
/retryable:\s*true/u.test(files.artifactPublisher ?? "")
);
}
web/hwlab-cloud-web/scripts/check.mjs
+6 -1
View File
@@ -934,7 +934,12 @@ assert.match(artifactPublisher, /"evidence\.record\.query"/);
assert.match(artifactPublisher, /readonly_rpc_required/);
assert.match(artifactPublisher, /cloud_api_proxy_failed/);
assert.match(artifactPublisher, /cloud_api_proxy_timeout/);
assert.match(artifactPublisher, /traceId:\s*request\.headers\["x-trace-id"\]/);
assert.match(artifactPublisher, /Code Agent 代理等待 cloud-api 超过/);
assert.match(artifactPublisher, /输入已保留，可稍后重试/);
assert.match(artifactPublisher, /retryable:\s*true/);
assert.match(artifactPublisher, /blocker:\s*\{/);
assert.match(artifactPublisher, /const traceId = request\.headers\["x-trace-id"\] \|\| null/);
assert.match(artifactPublisher, /traceId,\s*\n\s*route: url\.pathname/);
assert.match(artifactPublisher, /url\.pathname\.replace\(\/\\\/\+\$\/,\s*""\) \|\| "\/"/);
assert.match(artifactPublisher, /routePath === "\/gate"/);
assert.match(artifactPublisher, /routePath === "\/diagnostics\/gate"/);