fix: 恢复 HarnessRL HWPOD 重启幂等
This commit is contained in:
@@ -11,6 +11,7 @@ import { test } from "bun:test";
|
||||
|
||||
import { createCloudApiBunServer } from "./bun-server.ts";
|
||||
import { createHwpodNodeWsRegistry } from "./hwpod-node-ws-registry.ts";
|
||||
import { createMemoryHwpodOperationLedger } from "./hwpod-operation-ledger.ts";
|
||||
import { createCloudApiServer } from "./server.ts";
|
||||
import { connectHwpodNodeWs, createHwpodNodeServer } from "../../tools/src/hwpod-node-lib.ts";
|
||||
|
||||
@@ -452,16 +453,57 @@ test("WebSocket registry reuses one authoritative operation for concurrent dupli
|
||||
plan.planId = "harnessrl-run:build:v1";
|
||||
const first = registry.dispatch(plan, { requestId: "req_first" }, { timeoutMs: 5000 });
|
||||
const retry = registry.dispatch(plan, { requestId: "req_retry" }, { timeoutMs: 5000 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
assert.equal(dispatchCount, 1);
|
||||
registry.handleBunMessage(socket, JSON.stringify({ type: "hwpod-node-ops-result", requestId: "req_first", nodeId: "pc-host-1", result: { ok: true, status: "completed", results: [{ opId: "op_1", ok: true }] } }));
|
||||
assert.deepEqual(await retry, await first);
|
||||
assert.equal(registry.lookup(plan.planId)?.result?.status, "completed");
|
||||
assert.equal((await registry.lookup(plan.planId))?.result?.status, "completed");
|
||||
const recovered = await registry.dispatch(plan, { requestId: "req_after_completion" }, { timeoutMs: 5000 });
|
||||
assert.equal(recovered.status, "completed");
|
||||
assert.equal(dispatchCount, 1);
|
||||
registry.closeBunSocket(socket);
|
||||
});
|
||||
|
||||
test("WebSocket registry rebuild recovers one durable plan result without redispatch", async () => {
|
||||
const records = new Map();
|
||||
let dispatchCount = 0;
|
||||
const firstLedger = createMemoryHwpodOperationLedger({ records, now: () => "2026-07-17T02:40:00.000Z" });
|
||||
const firstRegistry = createHwpodNodeWsRegistry({ operationLedger: firstLedger, now: () => "2026-07-17T02:40:00.000Z" });
|
||||
const firstSocket = { send(value: string) { const message = JSON.parse(value); if (message.type !== "hwpod-node-ops") return; dispatchCount += 1; queueMicrotask(() => { firstRegistry.handleBunMessage(firstSocket, JSON.stringify({ type: "hwpod-node-ops-accepted", requestId: message.requestId, nodeId: "pc-host-1" })); firstRegistry.handleBunMessage(firstSocket, JSON.stringify({ type: "hwpod-node-ops-result", requestId: message.requestId, nodeId: "pc-host-1", result: { ok: true, status: "completed", planId: message.plan.planId, results: [] } })); }); }, close() {} };
|
||||
firstRegistry.openBunSocket(firstSocket);
|
||||
firstRegistry.handleBunMessage(firstSocket, JSON.stringify({ type: "register", nodeId: "pc-host-1", maxInFlight: 1 }));
|
||||
const plan = { ...samplePlan(), nodeId: "pc-host-1", planId: "restart-safe-plan" };
|
||||
const firstResult = await firstRegistry.dispatch(plan, { requestId: "req_first" }, { timeoutMs: 1000 });
|
||||
firstRegistry.closeBunSocket(firstSocket);
|
||||
|
||||
const rebuiltLedger = createMemoryHwpodOperationLedger({ records, now: () => "2026-07-17T02:41:00.000Z" });
|
||||
const rebuiltRegistry = createHwpodNodeWsRegistry({ operationLedger: rebuiltLedger, now: () => "2026-07-17T02:41:00.000Z" });
|
||||
const recovered = await rebuiltRegistry.dispatch(plan, { requestId: "req_rebuilt" }, { timeoutMs: 1000 });
|
||||
assert.deepEqual(recovered, firstResult);
|
||||
assert.equal((await rebuiltRegistry.lookup(plan.planId))?.result?.status, "completed");
|
||||
assert.equal(dispatchCount, 1);
|
||||
});
|
||||
|
||||
test("WebSocket registry retries the same plan identity after pre-accept send failure", async () => {
|
||||
const records = new Map();
|
||||
const plan = { ...samplePlan(), nodeId: "pc-host-1", planId: "retry-same-plan" };
|
||||
const firstRegistry = createHwpodNodeWsRegistry({ operationLedger: createMemoryHwpodOperationLedger({ records }) });
|
||||
const failedSocket = { send(value: string) { if (JSON.parse(value).type === "hwpod-node-ops") throw new Error("connection failed before acceptance"); }, close() {} };
|
||||
firstRegistry.openBunSocket(failedSocket);
|
||||
firstRegistry.handleBunMessage(failedSocket, JSON.stringify({ type: "register", nodeId: "pc-host-1", maxInFlight: 1 }));
|
||||
assert.equal((await firstRegistry.dispatch(plan, { requestId: "req_failed" }, { timeoutMs: 1000 })).status, "blocked");
|
||||
|
||||
let acceptanceCount = 0;
|
||||
const retryRegistry = createHwpodNodeWsRegistry({ operationLedger: createMemoryHwpodOperationLedger({ records }) });
|
||||
const retrySocket = { send(value: string) { const message = JSON.parse(value); if (message.type !== "hwpod-node-ops") return; acceptanceCount += 1; queueMicrotask(() => { retryRegistry.handleBunMessage(retrySocket, JSON.stringify({ type: "hwpod-node-ops-accepted", requestId: message.requestId, nodeId: "pc-host-1" })); retryRegistry.handleBunMessage(retrySocket, JSON.stringify({ type: "hwpod-node-ops-result", requestId: message.requestId, nodeId: "pc-host-1", result: { ok: true, status: "completed", planId: message.plan.planId, results: [] } })); }); }, close() {} };
|
||||
retryRegistry.openBunSocket(retrySocket);
|
||||
retryRegistry.handleBunMessage(retrySocket, JSON.stringify({ type: "register", nodeId: "pc-host-1", maxInFlight: 1 }));
|
||||
const result = await retryRegistry.dispatch(plan, { requestId: "req_retry" }, { timeoutMs: 1000 });
|
||||
assert.equal(result.status, "completed");
|
||||
assert.equal(result.planId, plan.planId);
|
||||
assert.equal(acceptanceCount, 1);
|
||||
});
|
||||
|
||||
test("WebSocket registry keeps readiness probes separate and rejects dispatch above maxInFlight", async () => {
|
||||
const sent: any[] = [];
|
||||
const socket = { send(value: string) { sent.push(JSON.parse(value)); }, close() {} };
|
||||
|
||||
Reference in New Issue
Block a user