From 991beeded6dfdc1db880461bcb882f1a984cc4a4 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Sat, 6 Jun 2026 20:50:56 +0800 Subject: [PATCH] fix: support case-level caserun agent timeouts --- docs/reference/spec-hwpod-harness.md | 2 ++ tools/hwlab-cli/caserun.test.ts | 7 +++++ tools/src/hwlab-caserun-lib.ts | 44 ++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/docs/reference/spec-hwpod-harness.md b/docs/reference/spec-hwpod-harness.md index 3b7bd072..12165426 100644 --- a/docs/reference/spec-hwpod-harness.md +++ b/docs/reference/spec-hwpod-harness.md @@ -204,6 +204,8 @@ CaseRun 的 case registry repo 与 subject repo 必须分离。标准 registry r } ``` +每个 case 必须按任务难度独立配置 agent 阶段等待窗口，避免用单一全局 timeout 误判长任务或拖慢短任务。优先在 `agentTask.timeoutMs` 写入该 case 的 Code Agent 预期窗口；需要统一组织多个阶段 timeout 时，也可以写顶层 `timeouts.agentTimeoutMs`，但 CLI 显式参数 `--agent-timeout-ms` / `--timeout-ms` 仍作为人工单次覆盖。`timeoutMs` 只控制 CaseRun 等待 agent terminal/result 的窗口，不是 evidence 自动评价、门禁或成功判定。未配置时 CLI 兜底为 600000ms；case 可按难度显式收紧或放宽，当前 CLI 上限为 3600000ms。 + ### CaseRun 当前实现边界 当前 `d601-f103-v2-compile` 是 compile-only smoke，用于验证 case registry repo 与 subject repo 分离、`repoLocalPath` 加 `commitId` 的 subject provenance、隔离 subject worktree、run-local spec rewrite 和 D601 Keil 编译证据。它不创建 HWLAB Code Agent session，不调用 DS/DeepSeek/provider，不向 Code Agent 下发任务 prompt，不等待 agent trace，也不验证 agent 在源码仓库内完成修改。因此，compile-only CaseRun 是 HWPOD runner 和 Keil 编译底座 smoke，不是 Code Agent 能力评测。 diff --git a/tools/hwlab-cli/caserun.test.ts b/tools/hwlab-cli/caserun.test.ts index 725b4b10..73ecbf8a 100644 --- a/tools/hwlab-cli/caserun.test.ts +++ b/tools/hwlab-cli/caserun.test.ts @@ -275,6 +275,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit title: "D601-F103-V2 Keil compile-only CaseRun", hwpodSpec: "hwpod-spec.yaml", subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline" }, + agentTask: { workspace: "subjectWorktree", prompt: "Make a tiny source change through HWPOD.", timeoutMs: 234000, pollIntervalMs: 10 }, runtime: { apiUrl: "http://api.test", webUrl: "http://web.test" } }, null, 2), "utf8"); await writeFile(path.join(caseDir, "hwpod-spec.yaml"), hwpodSpecText(), "utf8"); @@ -304,6 +305,8 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.equal(runState.agent.sessionId, "ses_case_run"); assert.equal(runState.agent.conversationId, "cnv_case_d601-f103-v2-compile_run-agent-flow"); assert.equal(runState.agent.threadId, "thread-case"); + assert.equal(runState.agent.timeoutMs, 234000); + assert.equal(runState.agent.pollIntervalMs, 250); assert.match(runState.agent.traceId, /^trc_case_/u); assert.match(runState.agent.nextPollCommand, /client agent result trc_case_/u); assert.equal(runState._control.stage, "agent-running"); @@ -324,11 +327,14 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.equal(result.payload.evidence.ok, undefined); assert.equal(result.payload.agent.stageStatus, "completed"); assert.equal(result.payload.agent.traceId.startsWith("trc_case_"), true); + assert.equal(result.payload.agent.timeoutMs, 234000); assert.ok(result.payload.agentTrace.hwpodBuildCommandCount >= 1); assert.ok(result.payload.evidence.agentTrace.commandCount > 0); assert.ok(result.payload.evidence.agentTrace.commandCount <= 30); assert.ok(result.payload.evidence.agentTrace.hwpodCommandCount >= 3); assert.equal(result.payload.evidence.agentTrace.keilJobCandidates.includes("20260605_203835_798515c0"), true); + assert.equal(result.payload.evidence.agentTask.timeoutMs, 234000); + assert.equal(result.payload.agent.polls <= 3, true); assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod-ctl spec validate --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true); assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod inspect --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true); assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod build --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true); @@ -370,6 +376,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.equal(manifest.trace.conversationId, "cnv_case_d601-f103-v2-compile_run-agent-flow"); assert.equal(manifest.trace.agentRun.runId, "run_agent_trace"); assert.equal(manifest.prompt.sha256, result.payload.evidence.agent.promptSha256); + assert.equal(manifest.subject.commitId, SUBJECT_COMMIT_ID); assert.equal(manifest.diff.sha256, result.payload.evidence.agentDiff.diffPatchSha256); assert.equal(manifest.readableAgent.renderer, "tools/src/hwlab-cli/trace-renderer:traceDisplayRows"); assert.equal(manifest.readableAgent.messagesPath, "agent-messages.json"); diff --git a/tools/src/hwlab-caserun-lib.ts b/tools/src/hwlab-caserun-lib.ts index 245e057d..b2234667 100644 --- a/tools/src/hwlab-caserun-lib.ts +++ b/tools/src/hwlab-caserun-lib.ts @@ -12,8 +12,8 @@ const DEFAULT_WEB_URL = "http://74.48.78.17:19666"; const DEFAULT_POLL_INTERVAL_MS = 1000; const DEFAULT_JOB_TIMEOUT_MS = 50000; const MAX_JOB_TIMEOUT_MS = 300000; -const DEFAULT_AGENT_TIMEOUT_MS = 120000; -const MAX_AGENT_TIMEOUT_MS = 600000; +const DEFAULT_AGENT_TIMEOUT_MS = 600000; +const MAX_AGENT_TIMEOUT_MS = 3600000; const MAX_AGENT_TRACE_COMMANDS = 30; const HWLAB_API_KEY_PREFIX = "hwl_live_"; const STANDARD_CASE_REPO_PATH = "/root/hwlab-case-registry"; @@ -80,6 +80,8 @@ type PreparedAgentTask = { constraints: string[]; providerProfile: string; projectId: string; + timeoutMs?: number; + pollIntervalMs?: number; }; type AgentRunStage = { @@ -97,6 +99,8 @@ type AgentRunStage = { result: Record | null; polls: number; timedOut: boolean; + timeoutMs?: number; + pollIntervalMs?: number; resultUrl?: string; lastPollAt?: string; lastHttpStatus?: number; @@ -614,15 +618,24 @@ function agentTaskFromDefinition(definition: Record, caseId: st if (!prompt) throw cliError("agent_task_prompt_required", "case.json agentTask.prompt is required for Code Agent CaseRun", { field: "agentTask.prompt" }); const workspace = text(source.workspace) || "subjectWorktree"; if (workspace !== "subjectWorktree") throw cliError("unsupported_agent_task_workspace", "case.json agentTask.workspace must be subjectWorktree", { workspace }); + const timeouts = caseTimeoutsFromDefinition(definition); return { prompt, workspace, constraints: stringArray(source.constraints), providerProfile: text(source.providerProfile) || "deepseek", - projectId: text(source.projectId) || "prj_hwpod_workbench" + projectId: text(source.projectId) || "prj_hwpod_workbench", + timeoutMs: numberOption(source.timeoutMs ?? source.agentTimeoutMs ?? timeouts.agentTimeoutMs), + pollIntervalMs: numberOption(source.pollIntervalMs ?? source.agentPollIntervalMs ?? timeouts.agentPollIntervalMs) } as PreparedAgentTask; } +function caseTimeoutsFromDefinition(definition: Record) { + const value = definition.timeouts; + if (!value || typeof value !== "object" || Array.isArray(value)) return {} as Record; + return value as Record; +} + function defaultAgentTask(definition: Record, caseId: string): PreparedAgentTask { const title = text(definition.title) || caseId; return { @@ -686,6 +699,8 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) { result: null, polls: 0, timedOut: false, + timeoutMs: effectiveAgentTimeoutMs(context, run.agentTask), + pollIntervalMs: effectiveAgentPollIntervalMs(context, run.agentTask), resultUrl, nextPollCommand: `hwlab-cli client agent result ${traceId} --base-url ${baseUrl}`, sessionResponse: compactObject(session.body) @@ -707,6 +722,8 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) { result: compactObject(result.body), polls: result.polls, timedOut: result.timedOut, + timeoutMs: effectiveAgentTimeoutMs(context, run.agentTask), + pollIntervalMs: effectiveAgentPollIntervalMs(context, run.agentTask), resultUrl, lastPollAt: result.lastPollAt, lastHttpStatus: result.lastHttpStatus, @@ -812,9 +829,9 @@ async function agentWorkspaceFilesForRun(run: PreparedCaseRun): Promise