fix: 恢复 HarnessRL HWPOD 重启幂等

tools/hwlab-node.py

@@ -11,6 +11,7 @@
 from __future__ import annotations
 
 import base64
+import calendar
 import ctypes
 import ctypes.wintypes as wt
 import hashlib
@@ -61,6 +62,7 @@ LOG_DIR = CONFIG_DIR / "logs"
 UPDATE_DIR = CONFIG_DIR / "update"
 CONFIG_PATH = CONFIG_DIR / "config.json"
 STATE_PATH = CONFIG_DIR / "state.json"
+OPERATION_LEDGER_PATH = CONFIG_DIR / "hwpod-operations.json"
 LOG_PATH = LOG_DIR / "hwlab-node.log"
 
 DEFAULT_CONFIG = {
@@ -832,6 +834,7 @@ class WebSocketNodeClient:
         self.manual_stop = False
         self.connected = False
         self.diagnostic_cursor = 0
+        self.operation_retention_seconds = 0
 
     def start(self) -> None:
         if self.thread and self.thread.is_alive():
@@ -972,18 +975,52 @@ class WebSocketNodeClient:
         message = json.loads(text)
         if message.get("type") == "ack" and message.get("requestId") == "register":
             self.connected = message.get("ok") is True
+            self.operation_retention_seconds = safe_int(message.get("operationRetentionSeconds"), 0)
             self.bus.emit("connection", "connected" if self.connected else "error")
             self.bus.emit("log", f"注册结果: ok={message.get('ok')} message={message.get('message')}")
             return
         if message.get("type") != "hwpod-node-ops":
             return
         request_id = str(message.get("requestId") or "req_unknown")
+        plan = message.get("plan") or {}
+        plan_id = str(plan.get("planId") or request_id)
+        ledger = self._operation_ledger()
+        existing = ledger.get(plan_id)
+        self._send_json({"type": "hwpod-node-ops-accepted", "nodeId": self.executor.node_id, "requestId": request_id, "planId": plan_id, "observedAt": utc_now()})
+        if isinstance(existing, dict) and existing.get("status") == "completed" and isinstance(existing.get("result"), dict):
+            self._send_json({"type": "hwpod-node-ops-result", "nodeId": self.executor.node_id, "requestId": request_id, "result": existing["result"], "observedAt": utc_now()})
+            return
+        ledger[plan_id] = {"status": "accepted", "acceptedAt": utc_now(), "result": None}
+        self._save_operation_ledger(ledger)
         try:
-            result = self.executor.execute(message.get("plan") or {})
+            result = self.executor.execute(plan)
         except Exception as exc:
             result = {"ok": False, "status": "failed", "results": [], "blocker": {"code": "hwpod_node_ws_run_failed", "layer": "hwpod-node", "retryable": True, "summary": str(exc)}}
+        ledger[plan_id] = {"status": "completed", "acceptedAt": ledger[plan_id]["acceptedAt"], "completedAt": utc_now(), "result": result}
+        self._save_operation_ledger(ledger)
         self._send_json({"type": "hwpod-node-ops-result", "nodeId": self.executor.node_id, "requestId": request_id, "result": result, "observedAt": utc_now()})
 
+    def _operation_ledger(self) -> dict:
+        try:
+            value = json.loads(OPERATION_LEDGER_PATH.read_text(encoding="utf-8"))
+            ledger = value if isinstance(value, dict) else {}
+        except Exception:
+            ledger = {}
+        if self.operation_retention_seconds > 0:
+            cutoff = time.time() - self.operation_retention_seconds
+            ledger = {key: value for key, value in ledger.items() if not isinstance(value, dict) or self._operation_timestamp(value) >= cutoff}
+        return ledger
+
+    def _save_operation_ledger(self, ledger: dict) -> None:
+        atomic_write_json(OPERATION_LEDGER_PATH, ledger)
+
+    def _operation_timestamp(self, value: dict) -> float:
+        timestamp = str(value.get("completedAt") or value.get("acceptedAt") or "")
+        try:
+            return calendar.timegm(time.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ"))
+        except Exception:
+            return time.time()
+
     def _flush_diagnostics(self) -> None:
         for event in DIAGNOSTICS.since(self.diagnostic_cursor, 20):
             self._send_json({"type": "hwpod-node-diagnostic", "nodeId": self.executor.node_id, "diagnostic": event, "observedAt": utc_now()})

tools/src/hwpod-node-lib.ts

@@ -86,6 +86,9 @@ export function connectHwpodNodeWs(config: any, options: any = {}) {
   let closed = false;
   let attempt = 0;
   let readyResolve: ((value: unknown) => void) | null = null;
+  const operations = new Map<string, Promise<any>>();
+  const operationCompletedAt = new Map<string, number>();
+  let operationRetentionMs = 0;
   const ready = new Promise((resolve) => { readyResolve = resolve; });
 
   function connect() {
@@ -129,19 +132,24 @@ export function connectHwpodNodeWs(config: any, options: any = {}) {
     const textValue = typeof raw === "string" ? raw : await new Response(raw).text();
     const message = JSON.parse(textValue);
     if (message?.type === "ack" && message.requestId === "register" && readyResolve) {
+      operationRetentionMs = Math.max(0, (numberValue(message.operationRetentionSeconds) ?? 0) * 1000);
       logWsEvent("registered", { ok: message.ok === true, message: text(message.message) || null });
       readyResolve(message);
       readyResolve = null;
       return;
     }
     if (message?.type !== "hwpod-node-ops") return;
+    pruneOperations();
     const requestId = text(message.requestId) || "req_unknown";
-    let resultPayload;
-    try {
-      resultPayload = await executePlan(message.plan);
-    } catch (error) {
-      resultPayload = failure("hwpod-node.ws.run", error);
+    const planId = text(message.plan?.planId) || requestId;
+    sendJson({ type: "hwpod-node-ops-accepted", nodeId, requestId, planId, observedAt: now() });
+    let operation = operations.get(planId);
+    if (!operation) {
+      operation = Promise.resolve().then(() => executePlan(message.plan)).catch((error) => failure("hwpod-node.ws.run", error));
+      operations.set(planId, operation);
+      void operation.finally(() => operationCompletedAt.set(planId, Date.now()));
     }
+    const resultPayload = await operation;
     sendJson({ type: "hwpod-node-ops-result", nodeId, requestId, result: resultPayload, observedAt: now() });
   }
 
@@ -149,6 +157,16 @@ export function connectHwpodNodeWs(config: any, options: any = {}) {
     sendJson({ type: "heartbeat", nodeId, at: now() });
   }
 
+  function pruneOperations() {
+    if (!operationRetentionMs) return;
+    const cutoff = Date.now() - operationRetentionMs;
+    for (const [planId, completedAt] of operationCompletedAt) {
+      if (completedAt >= cutoff) continue;
+      operationCompletedAt.delete(planId);
+      operations.delete(planId);
+    }
+  }
+
   function sendJson(value: any) {
     if (!socket || socket.readyState !== WebSocket.OPEN) return false;
     socket.send(JSON.stringify(value));