fix: 防止 HWPOD operation 重复派发

internal/cloud/hwpod-node-ops.test.ts

@@ -426,6 +426,13 @@ test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by no
     assert.equal(payload.nodeId, "pc-host-1");
     assert.equal(payload.results[0].op, "workspace.ls");
     assert.equal(payload.results[0].output.entries.some((entry: any) => entry.name === "main.c"), true);
+    const recoveredResponse = await fetch(`${runtime.url}/v1/hwpod-node-ops?planId=${encodeURIComponent(plan.planId)}`);
+    const recovered = await recoveredResponse.json();
+    assert.equal(recoveredResponse.status, 200);
+    assert.equal(recovered.status, "completed");
+    assert.equal(recovered.result.planId, payload.planId);
+    assert.equal(recovered.result.status, payload.status);
+    assert.deepEqual(recovered.result.results, payload.results);
   } finally {
     connector.close();
     runtime.stop();
@@ -433,6 +440,28 @@ test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by no
   }
 });
 
+test("WebSocket registry reuses one authoritative operation for concurrent duplicate admission", async () => {
+  const sent: any[] = [];
+  let dispatchCount = 0;
+  const socket = { send(value: string) { const message = JSON.parse(value); sent.push(message); if (message.type === "hwpod-node-ops") dispatchCount += 1; }, close() {} };
+  const registry = createHwpodNodeWsRegistry({ now: () => "2026-07-17T02:00:00.000Z" });
+  registry.openBunSocket(socket);
+  registry.handleBunMessage(socket, JSON.stringify({ type: "register", nodeId: "pc-host-1", name: "PC Host", capabilities: ["workspace.ls"], maxInFlight: 2 }));
+  const plan = samplePlan();
+  plan.nodeId = "pc-host-1";
+  plan.planId = "harnessrl-run:build:v1";
+  const first = registry.dispatch(plan, { requestId: "req_first" }, { timeoutMs: 5000 });
+  const retry = registry.dispatch(plan, { requestId: "req_retry" }, { timeoutMs: 5000 });
+  assert.equal(dispatchCount, 1);
+  registry.handleBunMessage(socket, JSON.stringify({ type: "hwpod-node-ops-result", requestId: "req_first", nodeId: "pc-host-1", result: { ok: true, status: "completed", results: [{ opId: "op_1", ok: true }] } }));
+  assert.deepEqual(await retry, await first);
+  assert.equal(registry.lookup(plan.planId)?.result?.status, "completed");
+  const recovered = await registry.dispatch(plan, { requestId: "req_after_completion" }, { timeoutMs: 5000 });
+  assert.equal(recovered.status, "completed");
+  assert.equal(dispatchCount, 1);
+  registry.closeBunSocket(socket);
+});
+
 test("WebSocket registry keeps readiness probes separate and rejects dispatch above maxInFlight", async () => {
   const sent: any[] = [];
   const socket = { send(value: string) { sent.push(JSON.parse(value)); }, close() {} };

internal/cloud/hwpod-node-ws-registry.ts

@@ -37,12 +37,21 @@ type PendingDispatch = {
   resolve: (value: any) => void;
 };
 
+type AuthoritativeOperation = {
+  planId: string;
+  nodeId: string;
+  operation: Record<string, unknown>;
+  promise: Promise<any>;
+  result: any | null;
+};
+
 export function createHwpodNodeWsRegistry(options: any = {}) {
   const now = options.now ?? (() => new Date().toISOString());
   const clock = options.clock ?? (() => Date.now());
   const logger = options.logger ?? console;
   const connections = new Map<string, HwpodNodeConnection>();
   const pending = new Map<string, PendingDispatch>();
+  const operations = new Map<string, AuthoritativeOperation>();
   const socketConnections = new WeakMap<object, HwpodNodeConnection>();
 
   function openBunSocket(socket: any) {
@@ -81,6 +90,15 @@ export function createHwpodNodeWsRegistry(options: any = {}) {
 
   function dispatch(plan: any, requestMeta: any = {}, dispatchOptions: any = {}) {
     const nodeId = safeNodeId(plan?.nodeId);
+    const operationKind = safeText(requestMeta.operationKind) === "readiness-probe" ? "readiness-probe" : "user-operation";
+    const planId = operationKind === "user-operation" ? safeText(plan?.planId) : "";
+    const authoritative = planId ? operations.get(planId) : null;
+    if (authoritative) {
+      if (authoritative.nodeId !== nodeId) {
+        return Promise.resolve(blockedDispatch(plan, requestMeta, `hwpod operation ${planId} is already owned by node ${authoritative.nodeId}`, "hwpod_operation_identity_conflict"));
+      }
+      return authoritative.result === null ? authoritative.promise : Promise.resolve(authoritative.result);
+    }
     const connection = nodeId ? connections.get(nodeId) : null;
     if (!nodeId || !connection) {
       return Promise.resolve(blockedDispatch(plan, requestMeta, `hwpod-node ${nodeId || "<missing>"} is not connected through outbound WebSocket`));
@@ -90,11 +108,10 @@ export function createHwpodNodeWsRegistry(options: any = {}) {
     }
     const requestId = safeText(requestMeta.requestId) || `req_hwpod_ws_${randomUUID()}`;
     const timeoutMs = positiveInteger(dispatchOptions.timeoutMs, DEFAULT_DISPATCH_TIMEOUT_MS);
-    const operationKind = safeText(requestMeta.operationKind) === "readiness-probe" ? "readiness-probe" : "user-operation";
     const operation = operationStarted(plan, requestId, now());
     if (operationKind === "readiness-probe") connection.latestReadinessProbe = operation;
     else connection.latestOperation = operation;
-    return new Promise((resolve) => {
+    const promise = new Promise((resolve) => {
       const timer = setTimeout(() => {
         pending.delete(requestId);
         finishPendingOperation({ nodeId, requestId, connection, operationKind, operation, timer, resolve }, "timed-out", now(), "hwpod_node_unavailable");
@@ -111,6 +128,31 @@ export function createHwpodNodeWsRegistry(options: any = {}) {
         resolve(blockedDispatch(plan, requestMeta, error instanceof Error ? error.message : String(error)));
       }
     });
+    if (planId) operations.set(planId, { planId, nodeId, operation, promise, result: null });
+    void promise.then((result) => {
+      const authoritativeOperation = planId ? operations.get(planId) : null;
+      if (authoritativeOperation) {
+        authoritativeOperation.result = result;
+        authoritativeOperation.operation = operationFinished(
+          authoritativeOperation.operation,
+          safeText(result?.status) || (result?.ok === false ? "failed" : "completed"),
+          now(),
+          safeText(result?.blocker?.code) || null
+        );
+      }
+    });
+    return promise;
+  }
+
+  function lookup(planId: string) {
+    const authoritative = operations.get(safeText(planId));
+    if (!authoritative) return null;
+    return {
+      planId: authoritative.planId,
+      nodeId: authoritative.nodeId,
+      operation: authoritative.operation,
+      result: authoritative.result
+    };
   }
 
   function describe() {
@@ -223,7 +265,7 @@ export function createHwpodNodeWsRegistry(options: any = {}) {
     }
   }
 
-  return { openBunSocket, handleBunMessage, closeBunSocket, dispatch, describe, hasNode };
+  return { openBunSocket, handleBunMessage, closeBunSocket, dispatch, lookup, describe, hasNode };
 }
 
 function sendJson(connection: HwpodNodeConnection, value: any) {

internal/cloud/server-hwpod-http.ts

@@ -32,6 +32,14 @@ const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024;
 
 export async function handleHwpodNodeOpsHttp(request, response, options) {
   if (request.method === "GET") {
+    const planId = new URL(request.url, "http://hwlab.local").searchParams.get("planId")?.trim() ?? "";
+    if (planId) {
+      const operation = options.hwpodNodeWsRegistry.lookup(planId);
+      sendJson(response, operation ? 200 : 404, operation
+        ? { ok: true, status: operation.result ? "completed" : "running", contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, ...operation }
+        : { ok: false, status: "not-found", contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, planId, error: { code: "hwpod_operation_not_found", message: `hwpod operation ${planId} was not found` } });
+      return;
+    }
     sendJson(response, 200, {
       ok: true,
       status: "ready",

internal/harnessrl/activities.ts

@@ -3,7 +3,7 @@ import { createHash } from "node:crypto";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 
-import { buildCaseRun, collectAgentDiff, collectAgentTraceEvidence, collectCaseRun, prepareCaseRun, runAgentTaskStage } from "../../tools/src/hwlab-caserun-lib.ts";
+import { buildCaseRun, collectAgentDiff, collectAgentTraceEvidence, collectCaseRun, prepareCaseRun, recoverAgentTaskStage, recoverBuildCaseRun, runAgentTaskStage } from "../../tools/src/hwlab-caserun-lib.ts";
 import type { HarnessRLRegistry } from "./contracts.ts";
 import { caseRunInternalFetch } from "../cloud/server-caserun-http.ts";
 
@@ -13,9 +13,11 @@ export function createHarnessRLActivities(options: {
   stages?: Partial<{
     prepare(context: any, action: string): Promise<any>;
     agent(context: any, run: any): Promise<any>;
+    recoverAgent(context: any, run: any): Promise<any>;
     trace(context: any, run: any): Promise<any>;
     diff(context: any, run: any): Promise<any>;
     build(context: any, run: any): Promise<any>;
+    recoverBuild(context: any, run: any): Promise<any>;
     collect(context: any, run: any, evidence: any): Promise<any>;
   }>;
 }) {
@@ -24,9 +26,11 @@ export function createHarnessRLActivities(options: {
   const stages = {
     prepare: options.stages?.prepare ?? prepareCaseRun,
     agent: options.stages?.agent ?? runAgentTaskStage,
+    recoverAgent: options.stages?.recoverAgent ?? recoverAgentTaskStage,
     trace: options.stages?.trace ?? collectAgentTraceEvidence,
     diff: options.stages?.diff ?? collectAgentDiff,
     build: options.stages?.build ?? buildCaseRun,
+    recoverBuild: options.stages?.recoverBuild ?? recoverBuildCaseRun,
     collect: options.stages?.collect ?? collectCaseRun
   };
   return {
@@ -59,6 +63,12 @@ export function createHarnessRLActivities(options: {
         const stage = await stages.agent(contextForRecord(record, env, input.identity), prepared.prepared.run);
         await transition(registry, input.runId, "running", "agent-completed", { traceId: stage.agent?.traceId ?? null, sessionId: stage.agent?.sessionId ?? null });
         return { mode: prepared.mode, run: stage.run, agent: stage.agent };
+      }, async () => {
+        const record = required(await registry.getRun(input.runId));
+        const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "prepare", `${input.runId}:prepare:v1`))) as any;
+        const stage = await stages.recoverAgent(contextForRecord(record, env, input.identity), prepared.prepared.run);
+        await transition(registry, input.runId, "running", "agent-completed", { traceId: stage.agent?.traceId ?? null, sessionId: stage.agent?.sessionId ?? null, recovered: true });
+        return { mode: prepared.mode, run: stage.run, agent: stage.agent };
       });
     },
     async trace(input: { runId: string; identity: string }) {
@@ -103,6 +113,12 @@ export function createHarnessRLActivities(options: {
         const build = await stages.build(context, prepared.run);
         await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null });
         return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff };
+      }, async () => {
+        const record = required(await registry.getRun(input.runId));
+        const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "diff", `${input.runId}:diff:v1`))) as any;
+        const build = await stages.recoverBuild(contextForRecord(record, env, input.identity), prepared.run);
+        await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null, recovered: true });
+        return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff };
       });
     },
     async collect(input: { runId: string; identity: string }) {
@@ -138,10 +154,15 @@ export function createHarnessRLActivities(options: {
   };
 }
 
-async function idempotent(registry: HarnessRLRegistry, input: { runId: string; identity: string }, execute: () => Promise<Record<string, unknown>>) {
+async function idempotent(registry: HarnessRLRegistry, input: { runId: string; identity: string }, execute: () => Promise<Record<string, unknown>>, recover?: () => Promise<Record<string, unknown>>) {
   const activityName = input.identity.split(":").at(-2) ?? "activity";
   const existing = await registry.getActivityResult(input.runId, activityName, input.identity);
   if ((existing?.output as any)?.state === "completed") return (existing!.output as any).result;
+  if ((existing?.output as any)?.state === "accepted" && recover) {
+    const output = await recover();
+    const saved = await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "completed", authorityIdentity: input.identity, result: output } });
+    return activityOutput(saved);
+  }
   if (!existing) await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "accepted", authorityIdentity: input.identity } });
   const output = await execute();
   const saved = await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "completed", authorityIdentity: input.identity, result: output } });

internal/harnessrl/harnessrl.test.ts

@@ -62,24 +62,34 @@ test("AgentRun and HWPOD crash gaps reuse stable authoritative operation identit
   const fixture = await hardwareFixture();
   const registry = new MemoryRegistry();
   await registry.createRun({ runId: "authority-gap", caseId: fixture.caseId, workflowId: "workflow-authority-gap", caseRepo: fixture.root, runDir: path.join(fixture.stateRoot, "runs", "authority-gap"), runtimeApiUrl: "http://127.0.0.1:6667" });
-  const acceptedAgent = new Set<string>();
-  const acceptedHwpod = new Set<string>();
+  let agentDispatchCount = 0;
+  let hwpodDispatchCount = 0;
+  let agentLookupCount = 0;
+  let hwpodLookupCount = 0;
   let agentCrash = true;
   let hwpodCrash = true;
   const stages = {
     async prepare(context: any) { return { run: { runId: "authority-gap", caseId: fixture.caseId, runDir: context.parsed.runDir } }; },
     async agent(context: any, run: any) {
-      acceptedAgent.add(context.parsed.traceId);
+      agentDispatchCount += 1;
       if (agentCrash) { agentCrash = false; throw new Error("worker exited after AgentRun accepted"); }
       return { run: { ...run, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } };
     },
+    async recoverAgent(context: any, run: any) {
+      agentLookupCount += 1;
+      return { run: { ...run, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } };
+    },
     async trace(_context: any, run: any) { return { run, trace: { traceId: run.agent.traceId, status: "completed" } }; },
     async diff(_context: any, run: any) { return { run, diff: { diffPatchSha256: "d".repeat(64) } }; },
     async build(context: any, run: any) {
-      acceptedHwpod.add(context.parsed.operationIdentity);
+      hwpodDispatchCount += 1;
       if (hwpodCrash) { hwpodCrash = false; throw new Error("worker exited after HWPOD accepted"); }
       return { run, summary: { jobId: "job-authoritative" }, evidence: { operation: { operationId: context.parsed.operationIdentity, jobId: "job-authoritative" } } };
     },
+    async recoverBuild(context: any, run: any) {
+      hwpodLookupCount += 1;
+      return { run, summary: { jobId: "job-authoritative" }, evidence: { operation: { operationId: context.parsed.operationIdentity, jobId: "job-authoritative" } } };
+    },
     async collect(_context: any, run: any, evidence: any) {
       const manifestPath = path.join(run.runDir, "artifact-manifest.json");
       await mkdir(run.runDir, { recursive: true });
@@ -90,14 +100,22 @@ test("AgentRun and HWPOD crash gaps reuse stable authoritative operation identit
   const worker = createHarnessRLActivities({ registry, env: {}, stages });
   await worker.prepare({ runId: "authority-gap", identity: "authority-gap:prepare:v1" });
   await assert.rejects(worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }), /AgentRun accepted/);
-  await worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" });
+  await Promise.all([
+    worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }),
+    worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" })
+  ]);
   await worker.trace({ runId: "authority-gap", identity: "authority-gap:trace:v1" });
   await worker.diff({ runId: "authority-gap", identity: "authority-gap:diff:v1" });
   await assert.rejects(worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }), /HWPOD accepted/);
-  await worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" });
+  await Promise.all([
+    worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }),
+    worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" })
+  ]);
   const result = await worker.collect({ runId: "authority-gap", identity: "authority-gap:collect:v1" });
-  assert.equal(acceptedAgent.size, 1);
-  assert.equal(acceptedHwpod.size, 1);
+  assert.equal(agentDispatchCount, 1);
+  assert.equal(hwpodDispatchCount, 1);
+  assert.equal(agentLookupCount, 2);
+  assert.equal(hwpodLookupCount, 2);
   assert.equal((result as any).authority.agentRun.sessionId, "ses-authoritative");
   assert.equal((result as any).authority.hwpod.jobId, "job-authoritative");
   assert.equal(typeof (result as any).artifactManifestSha256, "string");

tools/src/hwlab-caserun-runtime.ts

@@ -445,16 +445,26 @@ export async function prepareCaseRun(context: CaseContext, action = "prepare") {
 }
 
 export async function buildCaseRun(context: CaseContext, prepared?: PreparedCaseRun) {
+  return buildCaseRunWithAuthorityMode(context, prepared, false);
+}
+
+export async function recoverBuildCaseRun(context: CaseContext, prepared?: PreparedCaseRun) {
+  return buildCaseRunWithAuthorityMode(context, prepared, true);
+}
+
+async function buildCaseRunWithAuthorityMode(context: CaseContext, prepared: PreparedCaseRun | undefined, recoverOnly: boolean) {
   const run = prepared ?? await loadOrPrepareCaseRun(context);
   const validationPlan = run.validationPlan ?? caseValidationPlanFromDefinition(run.definition);
   const buildStep = validationPlan.steps.find((step) => step.kind === "build")!;
   const command = hwpodCommandForValidationStep(buildStep, { executable: process.execPath, cliPath: path.join(context.cwd, "tools/hwpod-cli.ts"), specPath: run.specPath, reason: `case-run ${run.caseId} ${run.runId} ${validationPlan.mode}` });
-  const invoked = await (context.runProcess ?? runProcess)(command, context.cwd, {
-    ...process.env,
-    ...context.env,
-    HWLAB_RUNTIME_API_URL: run.apiUrl,
-    HWLAB_RUNTIME_ENDPOINT_LOCKED: "1"
-  });
+  const invoked = recoverOnly
+    ? await recoverHwpodInvocation(context, run)
+    : await (context.runProcess ?? runProcess)(command, context.cwd, {
+        ...process.env,
+        ...context.env,
+        HWLAB_RUNTIME_API_URL: run.apiUrl,
+        HWLAB_RUNTIME_ENDPOINT_LOCKED: "1"
+      });
   const hwpodPayload = parseJsonMaybe(invoked.stdout);
   const hwpodDocument = await readHwpodSpec(run.specPath);
   const operation = summarizeHwpodOperationForTest(hwpodPayload, hwpodDocument, "debug.build");
@@ -794,6 +804,40 @@ export async function runAgentTaskStage(context: CaseContext, run: PreparedCaseR
   return { run: nextRun, agent };
 }
 
+export async function recoverAgentTaskStage(context: CaseContext, run: PreparedCaseRun) {
+  const baseUrl = webUrlFrom(context);
+  const providerProfile = text(context.parsed.providerProfile) || run.agentTask.providerProfile;
+  const projectId = text(context.parsed.projectId) || run.agentTask.projectId;
+  const conversationId = text(context.parsed.conversationId) || `cnv_case_${slug(run.caseId)}_${slug(run.runId)}`;
+  const traceId = text(context.parsed.traceId);
+  if (!traceId) throw Object.assign(new Error("AgentRun recovery requires the stable trace identity"), { code: "agentrun_recovery_identity_required" });
+  const existing = run.agent?.traceId === traceId ? run.agent : null;
+  const resultUrl = text(existing?.resultUrl) || `${baseUrl}/v1/agent/chat/result/${encodeURIComponent(traceId)}`;
+  const recoveredAgent = existing ?? agentFailureStage({
+    stageStatus: "recovering",
+    baseUrl,
+    projectId,
+    providerProfile,
+    conversationId,
+    traceId,
+    promptPath: path.join(run.runDir, "agent-prompt.md"),
+    promptSha256: ""
+  });
+  const result = await pollAgentResult(context, { baseUrl, traceId, acceptedBody: { resultUrl }, resultUrl, run, agent: recoveredAgent });
+  const agent = agentWithResultEvidence({
+    ...recoveredAgent,
+    stageStatus: result.stageStatus,
+    resultUrl,
+    polls: result.polls,
+    timedOut: result.timedOut,
+    lastPollAt: result.lastPollAt,
+    lastHttpStatus: result.lastHttpStatus,
+    error: result.error
+  }, result.body, run.agentTask) as AgentRunStage;
+  const nextRun = await updateRun(context, run, { stage: "agent-recovered", agent });
+  return { run: nextRun, agent };
+}
+
 export async function collectAgentTraceEvidence(context: CaseContext, run: PreparedCaseRun) {
   const agent = run.agent;
   const traceId = text(agent?.traceId);
@@ -907,6 +951,22 @@ async function pollAgentResult(context: CaseContext, input: { baseUrl: string; t
   return { stageStatus: "timeout", body: lastBody, polls, timedOut: true, lastPollAt, lastHttpStatus: lastStatus, error: { code: "agent_task_timeout", traceId: input.traceId, timeoutMs, lastHttpStatus: lastStatus, lastBody: compactObject(lastBody) } };
 }
 
+async function recoverHwpodInvocation(context: CaseContext, run: PreparedCaseRun) {
+  const planId = text(context.env.HWLAB_HWPOD_OPERATION_IDENTITY);
+  if (!planId) throw Object.assign(new Error("HWPOD recovery requires the stable operation identity"), { code: "hwpod_recovery_identity_required" });
+  const timeoutMs = Math.max(numberOption(context.parsed.timeoutMs) ?? DEFAULT_KEIL_COMMAND_TIMEOUT_MS, 1000);
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const response = await context.fetchImpl(`${run.apiUrl}/v1/hwpod-node-ops?planId=${encodeURIComponent(planId)}`, { method: "GET" });
+    const body = parseJsonMaybe(await response.text());
+    if (response.status === 404) throw Object.assign(new Error(`HWPOD operation ${planId} was not found`), { code: "hwpod_operation_not_found", planId });
+    if (!response.ok) throw Object.assign(new Error(`HWPOD operation lookup failed with HTTP ${response.status}`), { code: "hwpod_operation_lookup_failed", planId, status: response.status });
+    if (body?.result) return { stdout: JSON.stringify(body.result), stderr: "", exitCode: body.result.ok === false ? 1 : 0 };
+    await context.sleep(Math.max(numberOption(context.parsed.pollIntervalMs) ?? DEFAULT_POLL_INTERVAL_MS, 250));
+  }
+  throw Object.assign(new Error(`HWPOD operation ${planId} did not complete within ${timeoutMs}ms`), { code: "hwpod_operation_recovery_timeout", planId, timeoutMs });
+}
+
 function effectiveAgentTimeoutMs(context: CaseContext, agentTask?: PreparedAgentTask | null) {
   return numberOption(context.parsed.agentTimeoutMs ?? context.parsed.timeoutMs) ?? agentTask?.timeoutMs ?? DEFAULT_AGENT_TIMEOUT_MS;
 }