fix: support case-level caserun agent timeouts
This commit is contained in:
@@ -204,6 +204,8 @@ CaseRun 的 case registry repo 与 subject repo 必须分离。标准 registry r
|
||||
}
|
||||
```
|
||||
|
||||
每个 case 必须按任务难度独立配置 agent 阶段等待窗口,避免用单一全局 timeout 误判长任务或拖慢短任务。优先在 `agentTask.timeoutMs` 写入该 case 的 Code Agent 预期窗口;需要统一组织多个阶段 timeout 时,也可以写顶层 `timeouts.agentTimeoutMs`,但 CLI 显式参数 `--agent-timeout-ms` / `--timeout-ms` 仍作为人工单次覆盖。`timeoutMs` 只控制 CaseRun 等待 agent terminal/result 的窗口,不是 evidence 自动评价、门禁或成功判定。未配置时 CLI 兜底为 600000ms;case 可按难度显式收紧或放宽,当前 CLI 上限为 3600000ms。
|
||||
|
||||
### CaseRun 当前实现边界
|
||||
|
||||
当前 `d601-f103-v2-compile` 是 compile-only smoke,用于验证 case registry repo 与 subject repo 分离、`repoLocalPath` 加 `commitId` 的 subject provenance、隔离 subject worktree、run-local spec rewrite 和 D601 Keil 编译证据。它不创建 HWLAB Code Agent session,不调用 DS/DeepSeek/provider,不向 Code Agent 下发任务 prompt,不等待 agent trace,也不验证 agent 在源码仓库内完成修改。因此,compile-only CaseRun 是 HWPOD runner 和 Keil 编译底座 smoke,不是 Code Agent 能力评测。
|
||||
|
||||
@@ -275,6 +275,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
|
||||
title: "D601-F103-V2 Keil compile-only CaseRun",
|
||||
hwpodSpec: "hwpod-spec.yaml",
|
||||
subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline" },
|
||||
agentTask: { workspace: "subjectWorktree", prompt: "Make a tiny source change through HWPOD.", timeoutMs: 234000, pollIntervalMs: 10 },
|
||||
runtime: { apiUrl: "http://api.test", webUrl: "http://web.test" }
|
||||
}, null, 2), "utf8");
|
||||
await writeFile(path.join(caseDir, "hwpod-spec.yaml"), hwpodSpecText(), "utf8");
|
||||
@@ -304,6 +305,8 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
|
||||
assert.equal(runState.agent.sessionId, "ses_case_run");
|
||||
assert.equal(runState.agent.conversationId, "cnv_case_d601-f103-v2-compile_run-agent-flow");
|
||||
assert.equal(runState.agent.threadId, "thread-case");
|
||||
assert.equal(runState.agent.timeoutMs, 234000);
|
||||
assert.equal(runState.agent.pollIntervalMs, 250);
|
||||
assert.match(runState.agent.traceId, /^trc_case_/u);
|
||||
assert.match(runState.agent.nextPollCommand, /client agent result trc_case_/u);
|
||||
assert.equal(runState._control.stage, "agent-running");
|
||||
@@ -324,11 +327,14 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
|
||||
assert.equal(result.payload.evidence.ok, undefined);
|
||||
assert.equal(result.payload.agent.stageStatus, "completed");
|
||||
assert.equal(result.payload.agent.traceId.startsWith("trc_case_"), true);
|
||||
assert.equal(result.payload.agent.timeoutMs, 234000);
|
||||
assert.ok(result.payload.agentTrace.hwpodBuildCommandCount >= 1);
|
||||
assert.ok(result.payload.evidence.agentTrace.commandCount > 0);
|
||||
assert.ok(result.payload.evidence.agentTrace.commandCount <= 30);
|
||||
assert.ok(result.payload.evidence.agentTrace.hwpodCommandCount >= 3);
|
||||
assert.equal(result.payload.evidence.agentTrace.keilJobCandidates.includes("20260605_203835_798515c0"), true);
|
||||
assert.equal(result.payload.evidence.agentTask.timeoutMs, 234000);
|
||||
assert.equal(result.payload.agent.polls <= 3, true);
|
||||
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod-ctl spec validate --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
|
||||
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod inspect --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
|
||||
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod build --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
|
||||
@@ -370,6 +376,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
|
||||
assert.equal(manifest.trace.conversationId, "cnv_case_d601-f103-v2-compile_run-agent-flow");
|
||||
assert.equal(manifest.trace.agentRun.runId, "run_agent_trace");
|
||||
assert.equal(manifest.prompt.sha256, result.payload.evidence.agent.promptSha256);
|
||||
assert.equal(manifest.subject.commitId, SUBJECT_COMMIT_ID);
|
||||
assert.equal(manifest.diff.sha256, result.payload.evidence.agentDiff.diffPatchSha256);
|
||||
assert.equal(manifest.readableAgent.renderer, "tools/src/hwlab-cli/trace-renderer:traceDisplayRows");
|
||||
assert.equal(manifest.readableAgent.messagesPath, "agent-messages.json");
|
||||
|
||||
@@ -12,8 +12,8 @@ const DEFAULT_WEB_URL = "http://74.48.78.17:19666";
|
||||
const DEFAULT_POLL_INTERVAL_MS = 1000;
|
||||
const DEFAULT_JOB_TIMEOUT_MS = 50000;
|
||||
const MAX_JOB_TIMEOUT_MS = 300000;
|
||||
const DEFAULT_AGENT_TIMEOUT_MS = 120000;
|
||||
const MAX_AGENT_TIMEOUT_MS = 600000;
|
||||
const DEFAULT_AGENT_TIMEOUT_MS = 600000;
|
||||
const MAX_AGENT_TIMEOUT_MS = 3600000;
|
||||
const MAX_AGENT_TRACE_COMMANDS = 30;
|
||||
const HWLAB_API_KEY_PREFIX = "hwl_live_";
|
||||
const STANDARD_CASE_REPO_PATH = "/root/hwlab-case-registry";
|
||||
@@ -80,6 +80,8 @@ type PreparedAgentTask = {
|
||||
constraints: string[];
|
||||
providerProfile: string;
|
||||
projectId: string;
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
};
|
||||
|
||||
type AgentRunStage = {
|
||||
@@ -97,6 +99,8 @@ type AgentRunStage = {
|
||||
result: Record<string, unknown> | null;
|
||||
polls: number;
|
||||
timedOut: boolean;
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
resultUrl?: string;
|
||||
lastPollAt?: string;
|
||||
lastHttpStatus?: number;
|
||||
@@ -614,15 +618,24 @@ function agentTaskFromDefinition(definition: Record<string, unknown>, caseId: st
|
||||
if (!prompt) throw cliError("agent_task_prompt_required", "case.json agentTask.prompt is required for Code Agent CaseRun", { field: "agentTask.prompt" });
|
||||
const workspace = text(source.workspace) || "subjectWorktree";
|
||||
if (workspace !== "subjectWorktree") throw cliError("unsupported_agent_task_workspace", "case.json agentTask.workspace must be subjectWorktree", { workspace });
|
||||
const timeouts = caseTimeoutsFromDefinition(definition);
|
||||
return {
|
||||
prompt,
|
||||
workspace,
|
||||
constraints: stringArray(source.constraints),
|
||||
providerProfile: text(source.providerProfile) || "deepseek",
|
||||
projectId: text(source.projectId) || "prj_hwpod_workbench"
|
||||
projectId: text(source.projectId) || "prj_hwpod_workbench",
|
||||
timeoutMs: numberOption(source.timeoutMs ?? source.agentTimeoutMs ?? timeouts.agentTimeoutMs),
|
||||
pollIntervalMs: numberOption(source.pollIntervalMs ?? source.agentPollIntervalMs ?? timeouts.agentPollIntervalMs)
|
||||
} as PreparedAgentTask;
|
||||
}
|
||||
|
||||
function caseTimeoutsFromDefinition(definition: Record<string, unknown>) {
|
||||
const value = definition.timeouts;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return {} as Record<string, unknown>;
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function defaultAgentTask(definition: Record<string, unknown>, caseId: string): PreparedAgentTask {
|
||||
const title = text(definition.title) || caseId;
|
||||
return {
|
||||
@@ -686,6 +699,8 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) {
|
||||
result: null,
|
||||
polls: 0,
|
||||
timedOut: false,
|
||||
timeoutMs: effectiveAgentTimeoutMs(context, run.agentTask),
|
||||
pollIntervalMs: effectiveAgentPollIntervalMs(context, run.agentTask),
|
||||
resultUrl,
|
||||
nextPollCommand: `hwlab-cli client agent result ${traceId} --base-url ${baseUrl}`,
|
||||
sessionResponse: compactObject(session.body)
|
||||
@@ -707,6 +722,8 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) {
|
||||
result: compactObject(result.body),
|
||||
polls: result.polls,
|
||||
timedOut: result.timedOut,
|
||||
timeoutMs: effectiveAgentTimeoutMs(context, run.agentTask),
|
||||
pollIntervalMs: effectiveAgentPollIntervalMs(context, run.agentTask),
|
||||
resultUrl,
|
||||
lastPollAt: result.lastPollAt,
|
||||
lastHttpStatus: result.lastHttpStatus,
|
||||
@@ -812,9 +829,9 @@ async function agentWorkspaceFilesForRun(run: PreparedCaseRun): Promise<AgentWor
|
||||
}
|
||||
|
||||
async function pollAgentResult(context: CaseContext, input: { baseUrl: string; traceId: string; acceptedBody: any; resultUrl?: string; run?: PreparedCaseRun; agent?: AgentRunStage }) {
|
||||
const requestedTimeoutMs = numberOption(context.parsed.agentTimeoutMs ?? context.parsed.timeoutMs) ?? DEFAULT_AGENT_TIMEOUT_MS;
|
||||
const requestedTimeoutMs = effectiveAgentTimeoutMs(context, input.run?.agentTask);
|
||||
const timeoutMs = Math.max(Math.min(requestedTimeoutMs, MAX_AGENT_TIMEOUT_MS), 1000);
|
||||
const pollIntervalMs = Math.max(numberOption(context.parsed.agentPollIntervalMs ?? context.parsed.pollIntervalMs) ?? DEFAULT_POLL_INTERVAL_MS, 250);
|
||||
const pollIntervalMs = effectiveAgentPollIntervalMs(context, input.run?.agentTask);
|
||||
const startedAt = Date.now();
|
||||
const resultPath = text(input.acceptedBody?.resultUrl) || `/v1/agent/chat/result/${encodeURIComponent(input.traceId)}`;
|
||||
const resultUrl = input.resultUrl || (resultPath.startsWith("http") ? resultPath : `${input.baseUrl}${resultPath}`);
|
||||
@@ -828,7 +845,7 @@ async function pollAgentResult(context: CaseContext, input: { baseUrl: string; t
|
||||
const response = await caseFetchJson(context, resultUrl, { method: "GET", headers: caseAgentHeaders(context) }, "agent_task_result");
|
||||
lastBody = response.body;
|
||||
lastStatus = response.status;
|
||||
if (input.run && input.agent) await updateRun(context, input.run, { stage: "agent-running", agent: { ...input.agent, stageStatus: "running", result: compactObject(lastBody), polls, resultUrl, lastPollAt, lastHttpStatus: lastStatus } });
|
||||
if (input.run && input.agent) await updateRun(context, input.run, { stage: "agent-running", agent: { ...input.agent, stageStatus: "running", result: compactObject(lastBody), polls, timeoutMs, pollIntervalMs, resultUrl, lastPollAt, lastHttpStatus: lastStatus } });
|
||||
if (!response.ok) return { stageStatus: "result_request_failed", body: lastBody, polls, timedOut: false, lastPollAt, lastHttpStatus: lastStatus, error: response.error };
|
||||
const status = text(lastBody?.status);
|
||||
if (status && status !== "running") {
|
||||
@@ -839,6 +856,14 @@ async function pollAgentResult(context: CaseContext, input: { baseUrl: string; t
|
||||
return { stageStatus: "timeout", body: lastBody, polls, timedOut: true, lastPollAt, lastHttpStatus: lastStatus, error: { code: "agent_task_timeout", traceId: input.traceId, timeoutMs, lastHttpStatus: lastStatus, lastBody: compactObject(lastBody) } };
|
||||
}
|
||||
|
||||
function effectiveAgentTimeoutMs(context: CaseContext, agentTask?: PreparedAgentTask | null) {
|
||||
return numberOption(context.parsed.agentTimeoutMs ?? context.parsed.timeoutMs) ?? agentTask?.timeoutMs ?? DEFAULT_AGENT_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
function effectiveAgentPollIntervalMs(context: CaseContext, agentTask?: PreparedAgentTask | null) {
|
||||
return Math.max(numberOption(context.parsed.agentPollIntervalMs ?? context.parsed.pollIntervalMs) ?? agentTask?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS, 250);
|
||||
}
|
||||
|
||||
async function collectAgentDiff(context: CaseContext, run: PreparedCaseRun) {
|
||||
const status = await requestSubjectCommand(context, run, ["-C", run.subject.worktreePath, "status", "--short"]);
|
||||
const diffStat = await requestSubjectCommand(context, run, ["-C", run.subject.worktreePath, "diff", "--stat"]);
|
||||
@@ -1135,11 +1160,11 @@ function requestSummary(label: string, value: any) {
|
||||
}
|
||||
|
||||
function agentTaskSummary(agentTask: PreparedAgentTask) {
|
||||
return { workspace: agentTask.workspace, providerProfile: agentTask.providerProfile, projectId: agentTask.projectId, promptSha256: sha256(agentTask.prompt), constraints: agentTask.constraints };
|
||||
return clean({ workspace: agentTask.workspace, providerProfile: agentTask.providerProfile, projectId: agentTask.projectId, timeoutMs: agentTask.timeoutMs, pollIntervalMs: agentTask.pollIntervalMs, promptSha256: sha256(agentTask.prompt), constraints: agentTask.constraints });
|
||||
}
|
||||
|
||||
function agentSummary(agent: AgentRunStage) {
|
||||
return clean({ stageStatus: agent.stageStatus, baseUrl: agent.baseUrl, projectId: agent.projectId, providerProfile: agent.providerProfile, conversationId: agent.conversationId, sessionId: agent.sessionId, threadId: agent.threadId, traceId: agent.traceId, promptPath: agent.promptPath, promptSha256: agent.promptSha256, polls: agent.polls, timedOut: agent.timedOut, lastHttpStatus: agent.lastHttpStatus, error: agent.error });
|
||||
return clean({ stageStatus: agent.stageStatus, baseUrl: agent.baseUrl, projectId: agent.projectId, providerProfile: agent.providerProfile, conversationId: agent.conversationId, sessionId: agent.sessionId, threadId: agent.threadId, traceId: agent.traceId, promptPath: agent.promptPath, promptSha256: agent.promptSha256, polls: agent.polls, timedOut: agent.timedOut, timeoutMs: agent.timeoutMs, pollIntervalMs: agent.pollIntervalMs, lastHttpStatus: agent.lastHttpStatus, error: agent.error });
|
||||
}
|
||||
|
||||
function traceSummary(trace: AgentTraceStage) {
|
||||
@@ -1313,7 +1338,8 @@ function rewriteHwpodSpecWorkspacePath(specText: string, workspacePath: string)
|
||||
}
|
||||
|
||||
async function pollKeilJobStatus(context: CaseContext, run: PreparedCaseRun, jobId: string) {
|
||||
const timeoutMs = Math.min(numberOption(context.parsed.jobTimeoutMs ?? context.parsed.timeoutMs) ?? DEFAULT_JOB_TIMEOUT_MS, MAX_JOB_TIMEOUT_MS);
|
||||
const timeouts = caseTimeoutsFromDefinition(run.definition);
|
||||
const timeoutMs = Math.min(numberOption(context.parsed.jobTimeoutMs ?? context.parsed.timeoutMs) ?? numberOption(timeouts.jobTimeoutMs ?? timeouts.keilJobTimeoutMs) ?? DEFAULT_JOB_TIMEOUT_MS, MAX_JOB_TIMEOUT_MS);
|
||||
const pollIntervalMs = numberOption(context.parsed.pollIntervalMs) ?? DEFAULT_POLL_INTERVAL_MS;
|
||||
const started = Date.now();
|
||||
const polls: any[] = [];
|
||||
|
||||
Reference in New Issue
Block a user