fix: support case-level caserun agent timeouts

This commit is contained in:
Codex Agent
2026-06-06 20:50:56 +08:00
parent 844d4a3683
commit 991beeded6
3 changed files with 44 additions and 9 deletions
+2
View File
@@ -204,6 +204,8 @@ CaseRun 的 case registry repo 与 subject repo 必须分离。标准 registry r
}
```
每个 case 必须按任务难度独立配置 agent 阶段等待窗口,避免用单一全局 timeout 误判长任务或拖慢短任务。优先在 `agentTask.timeoutMs` 写入该 case 的 Code Agent 预期窗口;需要统一组织多个阶段 timeout 时,也可以写顶层 `timeouts.agentTimeoutMs`,但 CLI 显式参数 `--agent-timeout-ms` / `--timeout-ms` 仍作为人工单次覆盖。`timeoutMs` 只控制 CaseRun 等待 agent terminal/result 的窗口,不是 evidence 自动评价、门禁或成功判定。未配置时 CLI 兜底为 600000ms;case 可按难度显式收紧或放宽,当前 CLI 上限为 3600000ms。
### CaseRun 当前实现边界
当前 `d601-f103-v2-compile` 是 compile-only smoke,用于验证 case registry repo 与 subject repo 分离、`repoLocalPath``commitId` 的 subject provenance、隔离 subject worktree、run-local spec rewrite 和 D601 Keil 编译证据。它不创建 HWLAB Code Agent session,不调用 DS/DeepSeek/provider,不向 Code Agent 下发任务 prompt,不等待 agent trace,也不验证 agent 在源码仓库内完成修改。因此,compile-only CaseRun 是 HWPOD runner 和 Keil 编译底座 smoke,不是 Code Agent 能力评测。
+7
View File
@@ -275,6 +275,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
title: "D601-F103-V2 Keil compile-only CaseRun",
hwpodSpec: "hwpod-spec.yaml",
subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline" },
agentTask: { workspace: "subjectWorktree", prompt: "Make a tiny source change through HWPOD.", timeoutMs: 234000, pollIntervalMs: 10 },
runtime: { apiUrl: "http://api.test", webUrl: "http://web.test" }
}, null, 2), "utf8");
await writeFile(path.join(caseDir, "hwpod-spec.yaml"), hwpodSpecText(), "utf8");
@@ -304,6 +305,8 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
assert.equal(runState.agent.sessionId, "ses_case_run");
assert.equal(runState.agent.conversationId, "cnv_case_d601-f103-v2-compile_run-agent-flow");
assert.equal(runState.agent.threadId, "thread-case");
assert.equal(runState.agent.timeoutMs, 234000);
assert.equal(runState.agent.pollIntervalMs, 250);
assert.match(runState.agent.traceId, /^trc_case_/u);
assert.match(runState.agent.nextPollCommand, /client agent result trc_case_/u);
assert.equal(runState._control.stage, "agent-running");
@@ -324,11 +327,14 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
assert.equal(result.payload.evidence.ok, undefined);
assert.equal(result.payload.agent.stageStatus, "completed");
assert.equal(result.payload.agent.traceId.startsWith("trc_case_"), true);
assert.equal(result.payload.agent.timeoutMs, 234000);
assert.ok(result.payload.agentTrace.hwpodBuildCommandCount >= 1);
assert.ok(result.payload.evidence.agentTrace.commandCount > 0);
assert.ok(result.payload.evidence.agentTrace.commandCount <= 30);
assert.ok(result.payload.evidence.agentTrace.hwpodCommandCount >= 3);
assert.equal(result.payload.evidence.agentTrace.keilJobCandidates.includes("20260605_203835_798515c0"), true);
assert.equal(result.payload.evidence.agentTask.timeoutMs, 234000);
assert.equal(result.payload.agent.polls <= 3, true);
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod-ctl spec validate --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod inspect --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod build --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
@@ -370,6 +376,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
assert.equal(manifest.trace.conversationId, "cnv_case_d601-f103-v2-compile_run-agent-flow");
assert.equal(manifest.trace.agentRun.runId, "run_agent_trace");
assert.equal(manifest.prompt.sha256, result.payload.evidence.agent.promptSha256);
assert.equal(manifest.subject.commitId, SUBJECT_COMMIT_ID);
assert.equal(manifest.diff.sha256, result.payload.evidence.agentDiff.diffPatchSha256);
assert.equal(manifest.readableAgent.renderer, "tools/src/hwlab-cli/trace-renderer:traceDisplayRows");
assert.equal(manifest.readableAgent.messagesPath, "agent-messages.json");
+35 -9
View File
@@ -12,8 +12,8 @@ const DEFAULT_WEB_URL = "http://74.48.78.17:19666";
const DEFAULT_POLL_INTERVAL_MS = 1000;
const DEFAULT_JOB_TIMEOUT_MS = 50000;
const MAX_JOB_TIMEOUT_MS = 300000;
const DEFAULT_AGENT_TIMEOUT_MS = 120000;
const MAX_AGENT_TIMEOUT_MS = 600000;
const DEFAULT_AGENT_TIMEOUT_MS = 600000;
const MAX_AGENT_TIMEOUT_MS = 3600000;
const MAX_AGENT_TRACE_COMMANDS = 30;
const HWLAB_API_KEY_PREFIX = "hwl_live_";
const STANDARD_CASE_REPO_PATH = "/root/hwlab-case-registry";
@@ -80,6 +80,8 @@ type PreparedAgentTask = {
constraints: string[];
providerProfile: string;
projectId: string;
timeoutMs?: number;
pollIntervalMs?: number;
};
type AgentRunStage = {
@@ -97,6 +99,8 @@ type AgentRunStage = {
result: Record<string, unknown> | null;
polls: number;
timedOut: boolean;
timeoutMs?: number;
pollIntervalMs?: number;
resultUrl?: string;
lastPollAt?: string;
lastHttpStatus?: number;
@@ -614,15 +618,24 @@ function agentTaskFromDefinition(definition: Record<string, unknown>, caseId: st
if (!prompt) throw cliError("agent_task_prompt_required", "case.json agentTask.prompt is required for Code Agent CaseRun", { field: "agentTask.prompt" });
const workspace = text(source.workspace) || "subjectWorktree";
if (workspace !== "subjectWorktree") throw cliError("unsupported_agent_task_workspace", "case.json agentTask.workspace must be subjectWorktree", { workspace });
const timeouts = caseTimeoutsFromDefinition(definition);
return {
prompt,
workspace,
constraints: stringArray(source.constraints),
providerProfile: text(source.providerProfile) || "deepseek",
projectId: text(source.projectId) || "prj_hwpod_workbench"
projectId: text(source.projectId) || "prj_hwpod_workbench",
timeoutMs: numberOption(source.timeoutMs ?? source.agentTimeoutMs ?? timeouts.agentTimeoutMs),
pollIntervalMs: numberOption(source.pollIntervalMs ?? source.agentPollIntervalMs ?? timeouts.agentPollIntervalMs)
} as PreparedAgentTask;
}
function caseTimeoutsFromDefinition(definition: Record<string, unknown>) {
const value = definition.timeouts;
if (!value || typeof value !== "object" || Array.isArray(value)) return {} as Record<string, unknown>;
return value as Record<string, unknown>;
}
function defaultAgentTask(definition: Record<string, unknown>, caseId: string): PreparedAgentTask {
const title = text(definition.title) || caseId;
return {
@@ -686,6 +699,8 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) {
result: null,
polls: 0,
timedOut: false,
timeoutMs: effectiveAgentTimeoutMs(context, run.agentTask),
pollIntervalMs: effectiveAgentPollIntervalMs(context, run.agentTask),
resultUrl,
nextPollCommand: `hwlab-cli client agent result ${traceId} --base-url ${baseUrl}`,
sessionResponse: compactObject(session.body)
@@ -707,6 +722,8 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) {
result: compactObject(result.body),
polls: result.polls,
timedOut: result.timedOut,
timeoutMs: effectiveAgentTimeoutMs(context, run.agentTask),
pollIntervalMs: effectiveAgentPollIntervalMs(context, run.agentTask),
resultUrl,
lastPollAt: result.lastPollAt,
lastHttpStatus: result.lastHttpStatus,
@@ -812,9 +829,9 @@ async function agentWorkspaceFilesForRun(run: PreparedCaseRun): Promise<AgentWor
}
async function pollAgentResult(context: CaseContext, input: { baseUrl: string; traceId: string; acceptedBody: any; resultUrl?: string; run?: PreparedCaseRun; agent?: AgentRunStage }) {
const requestedTimeoutMs = numberOption(context.parsed.agentTimeoutMs ?? context.parsed.timeoutMs) ?? DEFAULT_AGENT_TIMEOUT_MS;
const requestedTimeoutMs = effectiveAgentTimeoutMs(context, input.run?.agentTask);
const timeoutMs = Math.max(Math.min(requestedTimeoutMs, MAX_AGENT_TIMEOUT_MS), 1000);
const pollIntervalMs = Math.max(numberOption(context.parsed.agentPollIntervalMs ?? context.parsed.pollIntervalMs) ?? DEFAULT_POLL_INTERVAL_MS, 250);
const pollIntervalMs = effectiveAgentPollIntervalMs(context, input.run?.agentTask);
const startedAt = Date.now();
const resultPath = text(input.acceptedBody?.resultUrl) || `/v1/agent/chat/result/${encodeURIComponent(input.traceId)}`;
const resultUrl = input.resultUrl || (resultPath.startsWith("http") ? resultPath : `${input.baseUrl}${resultPath}`);
@@ -828,7 +845,7 @@ async function pollAgentResult(context: CaseContext, input: { baseUrl: string; t
const response = await caseFetchJson(context, resultUrl, { method: "GET", headers: caseAgentHeaders(context) }, "agent_task_result");
lastBody = response.body;
lastStatus = response.status;
if (input.run && input.agent) await updateRun(context, input.run, { stage: "agent-running", agent: { ...input.agent, stageStatus: "running", result: compactObject(lastBody), polls, resultUrl, lastPollAt, lastHttpStatus: lastStatus } });
if (input.run && input.agent) await updateRun(context, input.run, { stage: "agent-running", agent: { ...input.agent, stageStatus: "running", result: compactObject(lastBody), polls, timeoutMs, pollIntervalMs, resultUrl, lastPollAt, lastHttpStatus: lastStatus } });
if (!response.ok) return { stageStatus: "result_request_failed", body: lastBody, polls, timedOut: false, lastPollAt, lastHttpStatus: lastStatus, error: response.error };
const status = text(lastBody?.status);
if (status && status !== "running") {
@@ -839,6 +856,14 @@ async function pollAgentResult(context: CaseContext, input: { baseUrl: string; t
return { stageStatus: "timeout", body: lastBody, polls, timedOut: true, lastPollAt, lastHttpStatus: lastStatus, error: { code: "agent_task_timeout", traceId: input.traceId, timeoutMs, lastHttpStatus: lastStatus, lastBody: compactObject(lastBody) } };
}
function effectiveAgentTimeoutMs(context: CaseContext, agentTask?: PreparedAgentTask | null) {
return numberOption(context.parsed.agentTimeoutMs ?? context.parsed.timeoutMs) ?? agentTask?.timeoutMs ?? DEFAULT_AGENT_TIMEOUT_MS;
}
function effectiveAgentPollIntervalMs(context: CaseContext, agentTask?: PreparedAgentTask | null) {
return Math.max(numberOption(context.parsed.agentPollIntervalMs ?? context.parsed.pollIntervalMs) ?? agentTask?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS, 250);
}
async function collectAgentDiff(context: CaseContext, run: PreparedCaseRun) {
const status = await requestSubjectCommand(context, run, ["-C", run.subject.worktreePath, "status", "--short"]);
const diffStat = await requestSubjectCommand(context, run, ["-C", run.subject.worktreePath, "diff", "--stat"]);
@@ -1135,11 +1160,11 @@ function requestSummary(label: string, value: any) {
}
function agentTaskSummary(agentTask: PreparedAgentTask) {
return { workspace: agentTask.workspace, providerProfile: agentTask.providerProfile, projectId: agentTask.projectId, promptSha256: sha256(agentTask.prompt), constraints: agentTask.constraints };
return clean({ workspace: agentTask.workspace, providerProfile: agentTask.providerProfile, projectId: agentTask.projectId, timeoutMs: agentTask.timeoutMs, pollIntervalMs: agentTask.pollIntervalMs, promptSha256: sha256(agentTask.prompt), constraints: agentTask.constraints });
}
function agentSummary(agent: AgentRunStage) {
return clean({ stageStatus: agent.stageStatus, baseUrl: agent.baseUrl, projectId: agent.projectId, providerProfile: agent.providerProfile, conversationId: agent.conversationId, sessionId: agent.sessionId, threadId: agent.threadId, traceId: agent.traceId, promptPath: agent.promptPath, promptSha256: agent.promptSha256, polls: agent.polls, timedOut: agent.timedOut, lastHttpStatus: agent.lastHttpStatus, error: agent.error });
return clean({ stageStatus: agent.stageStatus, baseUrl: agent.baseUrl, projectId: agent.projectId, providerProfile: agent.providerProfile, conversationId: agent.conversationId, sessionId: agent.sessionId, threadId: agent.threadId, traceId: agent.traceId, promptPath: agent.promptPath, promptSha256: agent.promptSha256, polls: agent.polls, timedOut: agent.timedOut, timeoutMs: agent.timeoutMs, pollIntervalMs: agent.pollIntervalMs, lastHttpStatus: agent.lastHttpStatus, error: agent.error });
}
function traceSummary(trace: AgentTraceStage) {
@@ -1313,7 +1338,8 @@ function rewriteHwpodSpecWorkspacePath(specText: string, workspacePath: string)
}
async function pollKeilJobStatus(context: CaseContext, run: PreparedCaseRun, jobId: string) {
const timeoutMs = Math.min(numberOption(context.parsed.jobTimeoutMs ?? context.parsed.timeoutMs) ?? DEFAULT_JOB_TIMEOUT_MS, MAX_JOB_TIMEOUT_MS);
const timeouts = caseTimeoutsFromDefinition(run.definition);
const timeoutMs = Math.min(numberOption(context.parsed.jobTimeoutMs ?? context.parsed.timeoutMs) ?? numberOption(timeouts.jobTimeoutMs ?? timeouts.keilJobTimeoutMs) ?? DEFAULT_JOB_TIMEOUT_MS, MAX_JOB_TIMEOUT_MS);
const pollIntervalMs = numberOption(context.parsed.pollIntervalMs) ?? DEFAULT_POLL_INTERVAL_MS;
const started = Date.now();
const polls: any[] = [];