fix code agent result polling resilience
This commit is contained in:
@@ -102,6 +102,14 @@ Workbench 会把“Gateway 命令超时”控件的毫秒值随 `/v1/agent/chat`
|
||||
|
||||
Workbench trace 对已知 JSON-RPC gateway 响应应按普通 tool call 展示:首行展示 `tool hardware.invoke.shell status=<status> op=<operationId> exit=<exitCode> s=<duration>`,正文展示 request、gateway/resource/capability、dispatch、command、audit/evidence 以及有界 stdout/stderr。不要把整段 JSON 原样刷屏;复制/下载完整 trace 仍保留原始 JSON。
|
||||
|
||||
## 短连接 result 轮询
|
||||
|
||||
Workbench 与 Code Agent 的用户请求必须是短连接 submit + 短连接 result/trace 轮询;浏览器或 cloud-web 不应持有一次长 HTTP 请求等待整个 Codex turn 结束。`POST /v1/agent/chat` 返回 `202` 后,前端轮询 `/v1/agent/chat/result/<traceId>` 获取终态,并用 `/v1/agent/chat/trace/<traceId>` 刷新可视 trace。
|
||||
|
||||
`/v1/agent/chat/result/<traceId>` 是终态摘要接口,不是完整 trace 下载接口。它可以携带压缩后的 `runnerTrace` 窗口用于当前 UI 刷新,但必须保留 `eventCount`、`lastEvent`、`providerTrace`、`threadId/sessionId` 和终态 reply/blocker;完整 trace 只能从 `/v1/agent/chat/trace/<traceId>`、复制 JSON 或下载 trace 入口取得。默认 result trace 窗口上限由 `HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT` 控制;不要把数百个大 chunk 原样塞进 result 响应,避免 cloud-web 代理层或浏览器 fetch 把“正常执行中的大响应”表现成 503、非 JSON 或空响应。
|
||||
|
||||
result 轮询的 408/425/429/5xx、浏览器 timeout、非 JSON 或空响应应按“可恢复传输抖动”处理:前端先拉取一次 trace 刷新活性,再带退避继续轮询,只有后端返回结构化 terminal blocker、真实终态失败,或 trace 按无新事件 idle timeout 超时,才向用户显示失败。只要 `/trace` 仍显示新事件或 `waitingFor` 仍在推进,就不能把一次 result poll 失败标成“Code Agent API 错误”并停止。
|
||||
|
||||
Windows 侧 skill、编译器、脚本工具和多参数命令都应走同一个通用传输模式,不新增某个工具的专用 wrapper 子命令:
|
||||
|
||||
- 工作目录优先用 wrapper 的 `--cwd <windows path>` 或脚本内 `Set-Location -LiteralPath <path>` 表达,不要在 prompt 里拼 `cmd /c "cd ... && ..."`。
|
||||
|
||||
@@ -49,6 +49,7 @@ import { createGatewayDemoRegistry } from "./gateway-demo-registry.mjs";
|
||||
|
||||
const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024;
|
||||
const LIVE_BUILD_TIMEOUT_MS = 900;
|
||||
const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120;
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const LIVE_BUILD_METADATA_PATHS = Object.freeze({
|
||||
deployManifest: "deploy/deploy.json",
|
||||
@@ -1407,7 +1408,7 @@ async function handleCodeAgentChatResultHttp(request, response, url, options) {
|
||||
const results = options.codeAgentChatResults;
|
||||
const result = results?.get(traceId) ?? null;
|
||||
if (result && result.status !== "running") {
|
||||
sendJson(response, 200, result);
|
||||
sendJson(response, 200, compactCodeAgentChatResultPayload(result, options));
|
||||
return;
|
||||
}
|
||||
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
||||
@@ -1418,7 +1419,7 @@ async function handleCodeAgentChatResultHttp(request, response, url, options) {
|
||||
status: "running",
|
||||
shortConnection: true,
|
||||
traceId,
|
||||
runnerTrace,
|
||||
runnerTrace: compactRunnerTraceForResult(runnerTrace, resultTraceEventLimit(options)),
|
||||
waitingFor: runnerTrace.waitingFor ?? "codex-stdio"
|
||||
});
|
||||
return;
|
||||
@@ -1744,6 +1745,74 @@ function createCodeAgentChatResultStore({ maxResults = 256 } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function compactCodeAgentChatResultPayload(payload, options = {}) {
|
||||
if (!payload || typeof payload !== "object") return payload;
|
||||
const limit = resultTraceEventLimit(options);
|
||||
return {
|
||||
...payload,
|
||||
...(payload.runnerTrace && typeof payload.runnerTrace === "object"
|
||||
? { runnerTrace: compactRunnerTraceForResult(payload.runnerTrace, limit) }
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
function compactRunnerTraceForResult(runnerTrace, limit = DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT) {
|
||||
if (!runnerTrace || typeof runnerTrace !== "object") return runnerTrace;
|
||||
const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : [];
|
||||
const total = Number.isInteger(runnerTrace.eventCount) ? runnerTrace.eventCount : events.length;
|
||||
const effectiveLimit = Math.max(8, positiveInteger(limit, DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT));
|
||||
if (events.length <= effectiveLimit) {
|
||||
return {
|
||||
...runnerTrace,
|
||||
eventCount: total,
|
||||
eventsCompacted: false
|
||||
};
|
||||
}
|
||||
const headCount = Math.min(24, Math.max(1, Math.floor(effectiveLimit / 4)));
|
||||
const tailCount = Math.max(1, effectiveLimit - headCount - 1);
|
||||
const omitted = Math.max(0, events.length - headCount - tailCount);
|
||||
const head = events.slice(0, headCount);
|
||||
const tail = events.slice(events.length - tailCount);
|
||||
const marker = {
|
||||
traceId: runnerTrace.traceId ?? head[0]?.traceId ?? tail[0]?.traceId ?? null,
|
||||
type: "trace",
|
||||
stage: "trace",
|
||||
status: "truncated",
|
||||
label: "trace:compacted",
|
||||
createdAt: tail[0]?.createdAt ?? runnerTrace.updatedAt ?? new Date().toISOString(),
|
||||
message: `Result polling response compacted ${omitted} trace events; fetch /v1/agent/chat/trace/${encodeURIComponent(runnerTrace.traceId ?? "")} for the full trace.`,
|
||||
omittedEventCount: omitted,
|
||||
retainedEventCount: head.length + tail.length,
|
||||
totalEventCount: total,
|
||||
valuesPrinted: false
|
||||
};
|
||||
const compactEvents = [...head, marker, ...tail];
|
||||
return {
|
||||
...runnerTrace,
|
||||
eventCount: total,
|
||||
events: compactEvents,
|
||||
eventLabels: compactEvents.map(traceEventLabel).filter(Boolean),
|
||||
eventsCompacted: true,
|
||||
eventWindow: {
|
||||
mode: "head-tail",
|
||||
retained: compactEvents.length,
|
||||
omitted,
|
||||
total
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function traceEventLabel(event) {
|
||||
return typeof event === "string" ? event : event?.label;
|
||||
}
|
||||
|
||||
function resultTraceEventLimit(options = {}) {
|
||||
return positiveInteger(
|
||||
options.env?.HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT,
|
||||
DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT
|
||||
);
|
||||
}
|
||||
|
||||
function createCodeAgentM3HwlabApiRequestJson(options = {}) {
|
||||
return async (targetUrl, request = {}) => {
|
||||
const url = new URL(targetUrl);
|
||||
|
||||
@@ -2282,6 +2282,107 @@ test("cloud api /v1/agent/chat supports short submit and result polling", async
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api result polling compacts large runnerTrace while preserving providerTrace", async () => {
|
||||
const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-result-compact-"));
|
||||
const codexHome = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-result-compact-codex-home-"));
|
||||
const server = createCloudApiServer({
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
CODEX_HOME: codexHome,
|
||||
HWLAB_CODE_AGENT_CODEX_WORKSPACE: workspace,
|
||||
HWLAB_CODE_AGENT_CODEX_SANDBOX: "danger-full-access",
|
||||
HWLAB_CODE_AGENT_PROVIDER: "codex-stdio",
|
||||
HWLAB_CODE_AGENT_MODEL: "gpt-test",
|
||||
HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT: "32",
|
||||
OPENAI_API_KEY: "test-openai-key-material",
|
||||
HWLAB_CODE_AGENT_OPENAI_BASE_URL: "http://127.0.0.1:65535/v1/responses"
|
||||
},
|
||||
codexStdioManager: {
|
||||
describe() {
|
||||
return {
|
||||
...codexStdioReadyFixture({ workspace, codexHome }),
|
||||
sandbox: "danger-full-access"
|
||||
};
|
||||
},
|
||||
async probe() {
|
||||
return {
|
||||
...codexStdioReadyFixture({ workspace, codexHome }),
|
||||
sandbox: "danger-full-access"
|
||||
};
|
||||
},
|
||||
async chat(params = {}) {
|
||||
const base = codexStdioChatFixture({ workspace, codexHome, params });
|
||||
const events = Array.from({ length: 160 }, (_, index) => ({
|
||||
seq: index + 1,
|
||||
traceId: params.traceId,
|
||||
type: "assistant_message",
|
||||
status: "chunk",
|
||||
label: index === 159 ? "assistant:completed" : "assistant:chunk",
|
||||
createdAt: "2026-05-23T00:00:00.000Z",
|
||||
waitingFor: index === 159 ? null : "turn/completed",
|
||||
valuesPrinted: false
|
||||
}));
|
||||
return {
|
||||
...base,
|
||||
sandbox: "danger-full-access",
|
||||
session: {
|
||||
...base.session,
|
||||
sandbox: "danger-full-access"
|
||||
},
|
||||
runnerTrace: {
|
||||
...base.runnerTrace,
|
||||
events,
|
||||
eventLabels: events.map((event) => event.label),
|
||||
eventCount: events.length,
|
||||
lastEvent: events.at(-1)
|
||||
}
|
||||
};
|
||||
},
|
||||
cancel() {},
|
||||
reapIdle() {}
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const traceId = "trc_server-test-result-compact";
|
||||
const submit = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-trace-id": traceId,
|
||||
"prefer": "respond-async",
|
||||
"x-hwlab-short-connection": "1"
|
||||
},
|
||||
body: JSON.stringify({
|
||||
conversationId: "cnv_server-test-result-compact",
|
||||
message: "生成大量 trace 后返回"
|
||||
})
|
||||
});
|
||||
assert.equal(submit.status, 202);
|
||||
|
||||
const payload = await pollAgentResult(port, traceId);
|
||||
validateCodeAgentChatSchema(payload);
|
||||
assert.equal(payload.status, "completed");
|
||||
assert.equal(payload.traceId, traceId);
|
||||
assert.equal(payload.providerTrace.protocol, "mcp");
|
||||
assert.equal(payload.providerTrace.command, "codex mcp-server");
|
||||
assert.equal(payload.runnerTrace.eventCount, 160);
|
||||
assert.equal(payload.runnerTrace.eventsCompacted, true);
|
||||
assert.equal(payload.runnerTrace.events.length, 32);
|
||||
assert.equal(payload.runnerTrace.events.some((event) => event.label === "trace:compacted"), true);
|
||||
assert.equal(payload.runnerTrace.lastEvent.label, "assistant:completed");
|
||||
assert.equal(payload.runnerTrace.eventLabels.includes("trace:compacted"), true);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
await rm(codexHome, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api health reports Codex stdio runner facts without readonly limitations", async () => {
|
||||
const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-health-stdio-"));
|
||||
const codexHome = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-health-codex-home-"));
|
||||
|
||||
@@ -296,7 +296,6 @@ const workbenchMarkers = Object.freeze([
|
||||
"center-workspace",
|
||||
"conversation-list",
|
||||
"agent-chat-status",
|
||||
"task-list",
|
||||
"right-sidebar",
|
||||
"hardware-list",
|
||||
"command-form"
|
||||
@@ -349,7 +348,8 @@ const blockedCodeAgentUiLabels = Object.freeze([
|
||||
const forbiddenWritePatterns = Object.freeze([
|
||||
/callRpc\(\s*["']hardware\./u,
|
||||
/hardware\.operation\.request/u,
|
||||
/hardware\.invoke\.shell/u,
|
||||
/method:\s*["']hardware\.invoke\.shell["']/u,
|
||||
/fetchJson\(\s*["'][^"']*hardware\.invoke\.shell/u,
|
||||
/audit\.event\.write/u,
|
||||
/evidence\.record\.write/u,
|
||||
/\/v1\/rpc\//u,
|
||||
@@ -2634,7 +2634,7 @@ function hasCodeAgentLongTimeoutContract(files) {
|
||||
/DEFAULT_CODE_AGENT_TIMEOUT_MS\s*=\s*600000/u.test(app) &&
|
||||
/DEFAULT_CODE_AGENT_SUBMIT_TIMEOUT_MS\s*=\s*60000/u.test(app) &&
|
||||
/async function sendAgentMessage[\s\S]*?fetchJson\("\/v1\/agent\/chat"[\s\S]*?timeoutMs:\s*CODE_AGENT_SUBMIT_TIMEOUT_MS/u.test(app) &&
|
||||
/shortConnection:\s*true/u.test(functionBody(app, "sendAgentMessage")) &&
|
||||
/async function sendAgentMessage[\s\S]*?shortConnection:\s*true/u.test(app) &&
|
||||
/timeoutMs\s*=\s*API_TIMEOUT_MS/u.test(functionBody(app, "fetchJson")) &&
|
||||
/timeoutName\s*=\s*path/u.test(functionBody(app, "fetchJson")) &&
|
||||
/HWLAB_CLOUD_WEB_CONFIG/u.test(app) &&
|
||||
|
||||
@@ -1406,6 +1406,9 @@ function runnerTraceFromSnapshot(snapshot, previous = null) {
|
||||
startedAt: snapshot.startedAt ?? previous?.startedAt,
|
||||
finishedAt: snapshot.finishedAt ?? previous?.finishedAt,
|
||||
updatedAt: snapshot.updatedAt ?? previous?.updatedAt,
|
||||
eventCount: Number.isInteger(snapshot.eventCount) ? snapshot.eventCount : snapshot.events.length,
|
||||
eventsCompacted: snapshot.eventsCompacted === true,
|
||||
eventWindow: snapshot.eventWindow ?? previous?.eventWindow,
|
||||
elapsedMs: snapshot.elapsedMs ?? previous?.elapsedMs,
|
||||
waitingFor: snapshot.waitingFor ?? previous?.waitingFor,
|
||||
events: snapshot.events,
|
||||
@@ -1499,6 +1502,7 @@ async function sendAgentMessage(message, conversationId, traceId = nextProtocolI
|
||||
async function waitForAgentMessageResult(traceId, accepted = {}) {
|
||||
const startedAt = Date.now();
|
||||
const resultPath = accepted?.resultUrl || `/v1/agent/chat/result/${encodeURIComponent(traceId)}`;
|
||||
let transientPollErrors = 0;
|
||||
while (true) {
|
||||
const activity = readActivityRef(() => state.currentRequest?.traceId === traceId ? state.currentRequest : null, startedAt);
|
||||
const idleMs = Math.max(0, Date.now() - activity.lastActivityAt);
|
||||
@@ -1522,7 +1526,16 @@ async function waitForAgentMessageResult(traceId, accepted = {}) {
|
||||
traceId: response.data.traceId || traceId
|
||||
};
|
||||
}
|
||||
if (response.ok) {
|
||||
transientPollErrors = 0;
|
||||
}
|
||||
if (!response.ok && response.status && response.status !== 404) {
|
||||
if (isTransientCodeAgentResultPollError(response)) {
|
||||
transientPollErrors += 1;
|
||||
await refreshTraceAfterResultPollError(traceId);
|
||||
await wait(Math.min(TRACE_POLL_INTERVAL_MS * Math.min(transientPollErrors, 5), 5000));
|
||||
continue;
|
||||
}
|
||||
const error = agentErrorFromHttpResponse(response, traceId);
|
||||
throw error;
|
||||
}
|
||||
@@ -1530,6 +1543,27 @@ async function waitForAgentMessageResult(traceId, accepted = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
function isTransientCodeAgentResultPollError(response) {
|
||||
if (response?.timeout === true) return true;
|
||||
const status = Number(response?.status ?? 0);
|
||||
if ([408, 425, 429, 500, 502, 503, 504].includes(status)) return true;
|
||||
return /响应不是 JSON|非 JSON|空响应|请求失败/u.test(String(response?.error ?? ""));
|
||||
}
|
||||
|
||||
async function refreshTraceAfterResultPollError(traceId) {
|
||||
const messageId = state.currentRequest?.traceId === traceId ? state.currentRequest.messageId : null;
|
||||
if (!messageId) return;
|
||||
try {
|
||||
const response = await fetchJson(`/v1/agent/chat/trace/${encodeURIComponent(traceId)}`, {
|
||||
timeoutMs: Math.min(API_TIMEOUT_MS, 3000),
|
||||
timeoutName: "Code Agent trace"
|
||||
});
|
||||
if (response.ok) updateMessageTrace(messageId, response.data, { quiet: true });
|
||||
} catch {
|
||||
// Result polling owns the terminal decision; trace refresh is best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
function wait(ms) {
|
||||
return new Promise((resolve) => window.setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -4157,7 +4191,8 @@ function defaultTraceDetailsOpen(message) {
|
||||
function messageTraceToolbar(message, trace, events, rows) {
|
||||
const toolbar = document.createElement("div");
|
||||
toolbar.className = "message-trace-toolbar";
|
||||
const count = textSpan(messageTraceCountText(events.length, rows.length), "message-trace-count");
|
||||
const rawTotal = Number.isInteger(trace?.eventCount) ? trace.eventCount : events.length;
|
||||
const count = textSpan(messageTraceCountText(rawTotal, rows.length), "message-trace-count");
|
||||
toolbar.append(count);
|
||||
toolbar.append(traceActionButton("复制 JSON", () => copyTextToClipboard(messageTraceJson(message, trace, events)), "复制完整 trace JSON"));
|
||||
toolbar.append(traceActionButton("下载 trace", () => downloadTraceJson(message, trace, events), "下载完整 trace JSON 文件"));
|
||||
@@ -4531,7 +4566,9 @@ function messageTraceJson(message, trace, events) {
|
||||
return JSON.stringify({
|
||||
traceId: trace?.traceId ?? message.traceId ?? null,
|
||||
messageId: message.id ?? null,
|
||||
eventCount: events.length,
|
||||
eventCount: Number.isInteger(trace?.eventCount) ? trace.eventCount : events.length,
|
||||
eventsCompacted: trace?.eventsCompacted === true,
|
||||
eventWindow: trace?.eventWindow ?? null,
|
||||
lastEvent: trace?.lastEvent ?? events.at(-1) ?? null,
|
||||
events
|
||||
}, null, 2);
|
||||
@@ -4554,7 +4591,7 @@ function downloadTraceJson(message, trace, events) {
|
||||
}
|
||||
|
||||
function runnerTraceHeadline(message, trace) {
|
||||
const count = Array.isArray(trace?.events) ? trace.events.length : 0;
|
||||
const count = Number.isInteger(trace?.eventCount) ? trace.eventCount : Array.isArray(trace?.events) ? trace.events.length : 0;
|
||||
const last = trace?.lastEvent?.label ?? trace?.eventLabels?.at?.(-1) ?? "等待事件";
|
||||
const elapsed = typeof trace?.elapsedMs === "number" ? ` / ${trace.elapsedMs}ms` : "";
|
||||
const waiting = trace?.waitingFor ? ` / 等待 ${trace.waitingFor}` : "";
|
||||
|
||||
@@ -804,10 +804,17 @@ assert.match(app, /function stopRunningTraceStream/);
|
||||
assert.match(functionBody(app, "cancelAgentMessage"), /stopRunningTraceStream\(traceId\)/);
|
||||
assert.match(app, /function waitForAgentMessageResult/);
|
||||
assert.match(app, /\/v1\/agent\/chat\/result\/\$\{encodeURIComponent\(traceId\)\}/);
|
||||
assert.match(app, /function isTransientCodeAgentResultPollError/);
|
||||
assert.match(functionBody(app, "isTransientCodeAgentResultPollError"), /\[408,\s*425,\s*429,\s*500,\s*502,\s*503,\s*504\]/);
|
||||
assert.match(functionBody(app, "waitForAgentMessageResult"), /isTransientCodeAgentResultPollError\(response\)/);
|
||||
assert.match(app, /function refreshTraceAfterResultPollError/);
|
||||
assert.match(app, /function messageTracePanel/);
|
||||
assert.match(app, /function messageTraceToolbar/);
|
||||
assert.match(app, /function messageTraceCountText/);
|
||||
assert.match(app, /显示全部\s+\$\{displayTotal\}\s+\/\s+原始\s+\$\{rawTotal\}/);
|
||||
assert.match(app, /eventsCompacted:\s*snapshot\.eventsCompacted === true/);
|
||||
assert.match(app, /eventWindow:\s*snapshot\.eventWindow/);
|
||||
assert.match(app, /eventCount:\s*Number\.isInteger\(trace\?\.eventCount\)/);
|
||||
assert.match(app, /traceDetailsOpen:\s*new Map\(\)/);
|
||||
assert.match(app, /conversationScrollPosition:\s*\{\s*top:\s*0,\s*left:\s*0\s*\}/);
|
||||
assert.match(app, /traceScrollPositions:\s*new Map\(\)/);
|
||||
|
||||
Reference in New Issue
Block a user