From 84dc6618e6a3007b9db0a5f681079deef564f47e Mon Sep 17 00:00:00 2001 From: root Date: Fri, 17 Jul 2026 04:19:31 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=98=B2=E6=AD=A2=20HWPOD=20operation?= =?UTF-8?q?=20=E9=87=8D=E5=A4=8D=E6=B4=BE=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/cloud/hwpod-node-ops.test.ts | 29 ++++++++++ internal/cloud/hwpod-node-ws-registry.ts | 48 +++++++++++++++- internal/cloud/server-hwpod-http.ts | 8 +++ internal/harnessrl/activities.ts | 25 +++++++- internal/harnessrl/harnessrl.test.ts | 34 ++++++++--- tools/src/hwlab-caserun-runtime.ts | 72 ++++++++++++++++++++++-- 6 files changed, 197 insertions(+), 19 deletions(-) diff --git a/internal/cloud/hwpod-node-ops.test.ts b/internal/cloud/hwpod-node-ops.test.ts index b156e4ec..722a40bc 100644 --- a/internal/cloud/hwpod-node-ops.test.ts +++ b/internal/cloud/hwpod-node-ops.test.ts @@ -426,6 +426,13 @@ test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by no assert.equal(payload.nodeId, "pc-host-1"); assert.equal(payload.results[0].op, "workspace.ls"); assert.equal(payload.results[0].output.entries.some((entry: any) => entry.name === "main.c"), true); + const recoveredResponse = await fetch(`${runtime.url}/v1/hwpod-node-ops?planId=${encodeURIComponent(plan.planId)}`); + const recovered = await recoveredResponse.json(); + assert.equal(recoveredResponse.status, 200); + assert.equal(recovered.status, "completed"); + assert.equal(recovered.result.planId, payload.planId); + assert.equal(recovered.result.status, payload.status); + assert.deepEqual(recovered.result.results, payload.results); } finally { connector.close(); runtime.stop(); @@ -433,6 +440,28 @@ test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by no } }); +test("WebSocket registry reuses one authoritative operation for concurrent duplicate admission", async () => { + const sent: any[] = []; + let dispatchCount = 0; + const socket = { send(value: string) { const message = JSON.parse(value); sent.push(message); if (message.type === "hwpod-node-ops") dispatchCount += 1; }, close() {} }; + const registry = createHwpodNodeWsRegistry({ now: () => "2026-07-17T02:00:00.000Z" }); + registry.openBunSocket(socket); + registry.handleBunMessage(socket, JSON.stringify({ type: "register", nodeId: "pc-host-1", name: "PC Host", capabilities: ["workspace.ls"], maxInFlight: 2 })); + const plan = samplePlan(); + plan.nodeId = "pc-host-1"; + plan.planId = "harnessrl-run:build:v1"; + const first = registry.dispatch(plan, { requestId: "req_first" }, { timeoutMs: 5000 }); + const retry = registry.dispatch(plan, { requestId: "req_retry" }, { timeoutMs: 5000 }); + assert.equal(dispatchCount, 1); + registry.handleBunMessage(socket, JSON.stringify({ type: "hwpod-node-ops-result", requestId: "req_first", nodeId: "pc-host-1", result: { ok: true, status: "completed", results: [{ opId: "op_1", ok: true }] } })); + assert.deepEqual(await retry, await first); + assert.equal(registry.lookup(plan.planId)?.result?.status, "completed"); + const recovered = await registry.dispatch(plan, { requestId: "req_after_completion" }, { timeoutMs: 5000 }); + assert.equal(recovered.status, "completed"); + assert.equal(dispatchCount, 1); + registry.closeBunSocket(socket); +}); + test("WebSocket registry keeps readiness probes separate and rejects dispatch above maxInFlight", async () => { const sent: any[] = []; const socket = { send(value: string) { sent.push(JSON.parse(value)); }, close() {} }; diff --git a/internal/cloud/hwpod-node-ws-registry.ts b/internal/cloud/hwpod-node-ws-registry.ts index 89c4ea8c..23818d75 100644 --- a/internal/cloud/hwpod-node-ws-registry.ts +++ b/internal/cloud/hwpod-node-ws-registry.ts @@ -37,12 +37,21 @@ type PendingDispatch = { resolve: (value: any) => void; }; +type AuthoritativeOperation = { + planId: string; + nodeId: string; + operation: Record; + promise: Promise; + result: any | null; +}; + export function createHwpodNodeWsRegistry(options: any = {}) { const now = options.now ?? (() => new Date().toISOString()); const clock = options.clock ?? (() => Date.now()); const logger = options.logger ?? console; const connections = new Map(); const pending = new Map(); + const operations = new Map(); const socketConnections = new WeakMap(); function openBunSocket(socket: any) { @@ -81,6 +90,15 @@ export function createHwpodNodeWsRegistry(options: any = {}) { function dispatch(plan: any, requestMeta: any = {}, dispatchOptions: any = {}) { const nodeId = safeNodeId(plan?.nodeId); + const operationKind = safeText(requestMeta.operationKind) === "readiness-probe" ? "readiness-probe" : "user-operation"; + const planId = operationKind === "user-operation" ? safeText(plan?.planId) : ""; + const authoritative = planId ? operations.get(planId) : null; + if (authoritative) { + if (authoritative.nodeId !== nodeId) { + return Promise.resolve(blockedDispatch(plan, requestMeta, `hwpod operation ${planId} is already owned by node ${authoritative.nodeId}`, "hwpod_operation_identity_conflict")); + } + return authoritative.result === null ? authoritative.promise : Promise.resolve(authoritative.result); + } const connection = nodeId ? connections.get(nodeId) : null; if (!nodeId || !connection) { return Promise.resolve(blockedDispatch(plan, requestMeta, `hwpod-node ${nodeId || ""} is not connected through outbound WebSocket`)); @@ -90,11 +108,10 @@ export function createHwpodNodeWsRegistry(options: any = {}) { } const requestId = safeText(requestMeta.requestId) || `req_hwpod_ws_${randomUUID()}`; const timeoutMs = positiveInteger(dispatchOptions.timeoutMs, DEFAULT_DISPATCH_TIMEOUT_MS); - const operationKind = safeText(requestMeta.operationKind) === "readiness-probe" ? "readiness-probe" : "user-operation"; const operation = operationStarted(plan, requestId, now()); if (operationKind === "readiness-probe") connection.latestReadinessProbe = operation; else connection.latestOperation = operation; - return new Promise((resolve) => { + const promise = new Promise((resolve) => { const timer = setTimeout(() => { pending.delete(requestId); finishPendingOperation({ nodeId, requestId, connection, operationKind, operation, timer, resolve }, "timed-out", now(), "hwpod_node_unavailable"); @@ -111,6 +128,31 @@ export function createHwpodNodeWsRegistry(options: any = {}) { resolve(blockedDispatch(plan, requestMeta, error instanceof Error ? error.message : String(error))); } }); + if (planId) operations.set(planId, { planId, nodeId, operation, promise, result: null }); + void promise.then((result) => { + const authoritativeOperation = planId ? operations.get(planId) : null; + if (authoritativeOperation) { + authoritativeOperation.result = result; + authoritativeOperation.operation = operationFinished( + authoritativeOperation.operation, + safeText(result?.status) || (result?.ok === false ? "failed" : "completed"), + now(), + safeText(result?.blocker?.code) || null + ); + } + }); + return promise; + } + + function lookup(planId: string) { + const authoritative = operations.get(safeText(planId)); + if (!authoritative) return null; + return { + planId: authoritative.planId, + nodeId: authoritative.nodeId, + operation: authoritative.operation, + result: authoritative.result + }; } function describe() { @@ -223,7 +265,7 @@ export function createHwpodNodeWsRegistry(options: any = {}) { } } - return { openBunSocket, handleBunMessage, closeBunSocket, dispatch, describe, hasNode }; + return { openBunSocket, handleBunMessage, closeBunSocket, dispatch, lookup, describe, hasNode }; } function sendJson(connection: HwpodNodeConnection, value: any) { diff --git a/internal/cloud/server-hwpod-http.ts b/internal/cloud/server-hwpod-http.ts index f140327f..be53cb35 100644 --- a/internal/cloud/server-hwpod-http.ts +++ b/internal/cloud/server-hwpod-http.ts @@ -32,6 +32,14 @@ const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024; export async function handleHwpodNodeOpsHttp(request, response, options) { if (request.method === "GET") { + const planId = new URL(request.url, "http://hwlab.local").searchParams.get("planId")?.trim() ?? ""; + if (planId) { + const operation = options.hwpodNodeWsRegistry.lookup(planId); + sendJson(response, operation ? 200 : 404, operation + ? { ok: true, status: operation.result ? "completed" : "running", contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, ...operation } + : { ok: false, status: "not-found", contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, planId, error: { code: "hwpod_operation_not_found", message: `hwpod operation ${planId} was not found` } }); + return; + } sendJson(response, 200, { ok: true, status: "ready", diff --git a/internal/harnessrl/activities.ts b/internal/harnessrl/activities.ts index 62200530..b2770bc9 100644 --- a/internal/harnessrl/activities.ts +++ b/internal/harnessrl/activities.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; -import { buildCaseRun, collectAgentDiff, collectAgentTraceEvidence, collectCaseRun, prepareCaseRun, runAgentTaskStage } from "../../tools/src/hwlab-caserun-lib.ts"; +import { buildCaseRun, collectAgentDiff, collectAgentTraceEvidence, collectCaseRun, prepareCaseRun, recoverAgentTaskStage, recoverBuildCaseRun, runAgentTaskStage } from "../../tools/src/hwlab-caserun-lib.ts"; import type { HarnessRLRegistry } from "./contracts.ts"; import { caseRunInternalFetch } from "../cloud/server-caserun-http.ts"; @@ -13,9 +13,11 @@ export function createHarnessRLActivities(options: { stages?: Partial<{ prepare(context: any, action: string): Promise; agent(context: any, run: any): Promise; + recoverAgent(context: any, run: any): Promise; trace(context: any, run: any): Promise; diff(context: any, run: any): Promise; build(context: any, run: any): Promise; + recoverBuild(context: any, run: any): Promise; collect(context: any, run: any, evidence: any): Promise; }>; }) { @@ -24,9 +26,11 @@ export function createHarnessRLActivities(options: { const stages = { prepare: options.stages?.prepare ?? prepareCaseRun, agent: options.stages?.agent ?? runAgentTaskStage, + recoverAgent: options.stages?.recoverAgent ?? recoverAgentTaskStage, trace: options.stages?.trace ?? collectAgentTraceEvidence, diff: options.stages?.diff ?? collectAgentDiff, build: options.stages?.build ?? buildCaseRun, + recoverBuild: options.stages?.recoverBuild ?? recoverBuildCaseRun, collect: options.stages?.collect ?? collectCaseRun }; return { @@ -59,6 +63,12 @@ export function createHarnessRLActivities(options: { const stage = await stages.agent(contextForRecord(record, env, input.identity), prepared.prepared.run); await transition(registry, input.runId, "running", "agent-completed", { traceId: stage.agent?.traceId ?? null, sessionId: stage.agent?.sessionId ?? null }); return { mode: prepared.mode, run: stage.run, agent: stage.agent }; + }, async () => { + const record = required(await registry.getRun(input.runId)); + const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "prepare", `${input.runId}:prepare:v1`))) as any; + const stage = await stages.recoverAgent(contextForRecord(record, env, input.identity), prepared.prepared.run); + await transition(registry, input.runId, "running", "agent-completed", { traceId: stage.agent?.traceId ?? null, sessionId: stage.agent?.sessionId ?? null, recovered: true }); + return { mode: prepared.mode, run: stage.run, agent: stage.agent }; }); }, async trace(input: { runId: string; identity: string }) { @@ -103,6 +113,12 @@ export function createHarnessRLActivities(options: { const build = await stages.build(context, prepared.run); await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null }); return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff }; + }, async () => { + const record = required(await registry.getRun(input.runId)); + const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "diff", `${input.runId}:diff:v1`))) as any; + const build = await stages.recoverBuild(contextForRecord(record, env, input.identity), prepared.run); + await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null, recovered: true }); + return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff }; }); }, async collect(input: { runId: string; identity: string }) { @@ -138,10 +154,15 @@ export function createHarnessRLActivities(options: { }; } -async function idempotent(registry: HarnessRLRegistry, input: { runId: string; identity: string }, execute: () => Promise>) { +async function idempotent(registry: HarnessRLRegistry, input: { runId: string; identity: string }, execute: () => Promise>, recover?: () => Promise>) { const activityName = input.identity.split(":").at(-2) ?? "activity"; const existing = await registry.getActivityResult(input.runId, activityName, input.identity); if ((existing?.output as any)?.state === "completed") return (existing!.output as any).result; + if ((existing?.output as any)?.state === "accepted" && recover) { + const output = await recover(); + const saved = await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "completed", authorityIdentity: input.identity, result: output } }); + return activityOutput(saved); + } if (!existing) await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "accepted", authorityIdentity: input.identity } }); const output = await execute(); const saved = await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "completed", authorityIdentity: input.identity, result: output } }); diff --git a/internal/harnessrl/harnessrl.test.ts b/internal/harnessrl/harnessrl.test.ts index 752ef890..38baa114 100644 --- a/internal/harnessrl/harnessrl.test.ts +++ b/internal/harnessrl/harnessrl.test.ts @@ -62,24 +62,34 @@ test("AgentRun and HWPOD crash gaps reuse stable authoritative operation identit const fixture = await hardwareFixture(); const registry = new MemoryRegistry(); await registry.createRun({ runId: "authority-gap", caseId: fixture.caseId, workflowId: "workflow-authority-gap", caseRepo: fixture.root, runDir: path.join(fixture.stateRoot, "runs", "authority-gap"), runtimeApiUrl: "http://127.0.0.1:6667" }); - const acceptedAgent = new Set(); - const acceptedHwpod = new Set(); + let agentDispatchCount = 0; + let hwpodDispatchCount = 0; + let agentLookupCount = 0; + let hwpodLookupCount = 0; let agentCrash = true; let hwpodCrash = true; const stages = { async prepare(context: any) { return { run: { runId: "authority-gap", caseId: fixture.caseId, runDir: context.parsed.runDir } }; }, async agent(context: any, run: any) { - acceptedAgent.add(context.parsed.traceId); + agentDispatchCount += 1; if (agentCrash) { agentCrash = false; throw new Error("worker exited after AgentRun accepted"); } return { run: { ...run, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }; }, + async recoverAgent(context: any, run: any) { + agentLookupCount += 1; + return { run: { ...run, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }; + }, async trace(_context: any, run: any) { return { run, trace: { traceId: run.agent.traceId, status: "completed" } }; }, async diff(_context: any, run: any) { return { run, diff: { diffPatchSha256: "d".repeat(64) } }; }, async build(context: any, run: any) { - acceptedHwpod.add(context.parsed.operationIdentity); + hwpodDispatchCount += 1; if (hwpodCrash) { hwpodCrash = false; throw new Error("worker exited after HWPOD accepted"); } return { run, summary: { jobId: "job-authoritative" }, evidence: { operation: { operationId: context.parsed.operationIdentity, jobId: "job-authoritative" } } }; }, + async recoverBuild(context: any, run: any) { + hwpodLookupCount += 1; + return { run, summary: { jobId: "job-authoritative" }, evidence: { operation: { operationId: context.parsed.operationIdentity, jobId: "job-authoritative" } } }; + }, async collect(_context: any, run: any, evidence: any) { const manifestPath = path.join(run.runDir, "artifact-manifest.json"); await mkdir(run.runDir, { recursive: true }); @@ -90,14 +100,22 @@ test("AgentRun and HWPOD crash gaps reuse stable authoritative operation identit const worker = createHarnessRLActivities({ registry, env: {}, stages }); await worker.prepare({ runId: "authority-gap", identity: "authority-gap:prepare:v1" }); await assert.rejects(worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }), /AgentRun accepted/); - await worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }); + await Promise.all([ + worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }), + worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }) + ]); await worker.trace({ runId: "authority-gap", identity: "authority-gap:trace:v1" }); await worker.diff({ runId: "authority-gap", identity: "authority-gap:diff:v1" }); await assert.rejects(worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }), /HWPOD accepted/); - await worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }); + await Promise.all([ + worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }), + worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }) + ]); const result = await worker.collect({ runId: "authority-gap", identity: "authority-gap:collect:v1" }); - assert.equal(acceptedAgent.size, 1); - assert.equal(acceptedHwpod.size, 1); + assert.equal(agentDispatchCount, 1); + assert.equal(hwpodDispatchCount, 1); + assert.equal(agentLookupCount, 2); + assert.equal(hwpodLookupCount, 2); assert.equal((result as any).authority.agentRun.sessionId, "ses-authoritative"); assert.equal((result as any).authority.hwpod.jobId, "job-authoritative"); assert.equal(typeof (result as any).artifactManifestSha256, "string"); diff --git a/tools/src/hwlab-caserun-runtime.ts b/tools/src/hwlab-caserun-runtime.ts index b04ea888..4b494d3e 100644 --- a/tools/src/hwlab-caserun-runtime.ts +++ b/tools/src/hwlab-caserun-runtime.ts @@ -445,16 +445,26 @@ export async function prepareCaseRun(context: CaseContext, action = "prepare") { } export async function buildCaseRun(context: CaseContext, prepared?: PreparedCaseRun) { + return buildCaseRunWithAuthorityMode(context, prepared, false); +} + +export async function recoverBuildCaseRun(context: CaseContext, prepared?: PreparedCaseRun) { + return buildCaseRunWithAuthorityMode(context, prepared, true); +} + +async function buildCaseRunWithAuthorityMode(context: CaseContext, prepared: PreparedCaseRun | undefined, recoverOnly: boolean) { const run = prepared ?? await loadOrPrepareCaseRun(context); const validationPlan = run.validationPlan ?? caseValidationPlanFromDefinition(run.definition); const buildStep = validationPlan.steps.find((step) => step.kind === "build")!; const command = hwpodCommandForValidationStep(buildStep, { executable: process.execPath, cliPath: path.join(context.cwd, "tools/hwpod-cli.ts"), specPath: run.specPath, reason: `case-run ${run.caseId} ${run.runId} ${validationPlan.mode}` }); - const invoked = await (context.runProcess ?? runProcess)(command, context.cwd, { - ...process.env, - ...context.env, - HWLAB_RUNTIME_API_URL: run.apiUrl, - HWLAB_RUNTIME_ENDPOINT_LOCKED: "1" - }); + const invoked = recoverOnly + ? await recoverHwpodInvocation(context, run) + : await (context.runProcess ?? runProcess)(command, context.cwd, { + ...process.env, + ...context.env, + HWLAB_RUNTIME_API_URL: run.apiUrl, + HWLAB_RUNTIME_ENDPOINT_LOCKED: "1" + }); const hwpodPayload = parseJsonMaybe(invoked.stdout); const hwpodDocument = await readHwpodSpec(run.specPath); const operation = summarizeHwpodOperationForTest(hwpodPayload, hwpodDocument, "debug.build"); @@ -794,6 +804,40 @@ export async function runAgentTaskStage(context: CaseContext, run: PreparedCaseR return { run: nextRun, agent }; } +export async function recoverAgentTaskStage(context: CaseContext, run: PreparedCaseRun) { + const baseUrl = webUrlFrom(context); + const providerProfile = text(context.parsed.providerProfile) || run.agentTask.providerProfile; + const projectId = text(context.parsed.projectId) || run.agentTask.projectId; + const conversationId = text(context.parsed.conversationId) || `cnv_case_${slug(run.caseId)}_${slug(run.runId)}`; + const traceId = text(context.parsed.traceId); + if (!traceId) throw Object.assign(new Error("AgentRun recovery requires the stable trace identity"), { code: "agentrun_recovery_identity_required" }); + const existing = run.agent?.traceId === traceId ? run.agent : null; + const resultUrl = text(existing?.resultUrl) || `${baseUrl}/v1/agent/chat/result/${encodeURIComponent(traceId)}`; + const recoveredAgent = existing ?? agentFailureStage({ + stageStatus: "recovering", + baseUrl, + projectId, + providerProfile, + conversationId, + traceId, + promptPath: path.join(run.runDir, "agent-prompt.md"), + promptSha256: "" + }); + const result = await pollAgentResult(context, { baseUrl, traceId, acceptedBody: { resultUrl }, resultUrl, run, agent: recoveredAgent }); + const agent = agentWithResultEvidence({ + ...recoveredAgent, + stageStatus: result.stageStatus, + resultUrl, + polls: result.polls, + timedOut: result.timedOut, + lastPollAt: result.lastPollAt, + lastHttpStatus: result.lastHttpStatus, + error: result.error + }, result.body, run.agentTask) as AgentRunStage; + const nextRun = await updateRun(context, run, { stage: "agent-recovered", agent }); + return { run: nextRun, agent }; +} + export async function collectAgentTraceEvidence(context: CaseContext, run: PreparedCaseRun) { const agent = run.agent; const traceId = text(agent?.traceId); @@ -907,6 +951,22 @@ async function pollAgentResult(context: CaseContext, input: { baseUrl: string; t return { stageStatus: "timeout", body: lastBody, polls, timedOut: true, lastPollAt, lastHttpStatus: lastStatus, error: { code: "agent_task_timeout", traceId: input.traceId, timeoutMs, lastHttpStatus: lastStatus, lastBody: compactObject(lastBody) } }; } +async function recoverHwpodInvocation(context: CaseContext, run: PreparedCaseRun) { + const planId = text(context.env.HWLAB_HWPOD_OPERATION_IDENTITY); + if (!planId) throw Object.assign(new Error("HWPOD recovery requires the stable operation identity"), { code: "hwpod_recovery_identity_required" }); + const timeoutMs = Math.max(numberOption(context.parsed.timeoutMs) ?? DEFAULT_KEIL_COMMAND_TIMEOUT_MS, 1000); + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const response = await context.fetchImpl(`${run.apiUrl}/v1/hwpod-node-ops?planId=${encodeURIComponent(planId)}`, { method: "GET" }); + const body = parseJsonMaybe(await response.text()); + if (response.status === 404) throw Object.assign(new Error(`HWPOD operation ${planId} was not found`), { code: "hwpod_operation_not_found", planId }); + if (!response.ok) throw Object.assign(new Error(`HWPOD operation lookup failed with HTTP ${response.status}`), { code: "hwpod_operation_lookup_failed", planId, status: response.status }); + if (body?.result) return { stdout: JSON.stringify(body.result), stderr: "", exitCode: body.result.ok === false ? 1 : 0 }; + await context.sleep(Math.max(numberOption(context.parsed.pollIntervalMs) ?? DEFAULT_POLL_INTERVAL_MS, 250)); + } + throw Object.assign(new Error(`HWPOD operation ${planId} did not complete within ${timeoutMs}ms`), { code: "hwpod_operation_recovery_timeout", planId, timeoutMs }); +} + function effectiveAgentTimeoutMs(context: CaseContext, agentTask?: PreparedAgentTask | null) { return numberOption(context.parsed.agentTimeoutMs ?? context.parsed.timeoutMs) ?? agentTask?.timeoutMs ?? DEFAULT_AGENT_TIMEOUT_MS; }