fix(workbench): classify runtime dependency outages as retryable (#2021)
This commit is contained in:
@@ -7,6 +7,7 @@ import { test } from "bun:test";
|
|||||||
import { createCloudApiServer } from "./server.ts";
|
import { createCloudApiServer } from "./server.ts";
|
||||||
import { createBackendPerformanceStore } from "./backend-performance.ts";
|
import { createBackendPerformanceStore } from "./backend-performance.ts";
|
||||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||||
|
import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts";
|
||||||
import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
|
import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
|
||||||
import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
||||||
|
|
||||||
@@ -58,6 +59,24 @@ test("workbench projection diagnostics keeps projecting health distinct from cau
|
|||||||
assert.equal(diagnostics.projectionHealth, "projecting");
|
assert.equal(diagnostics.projectionHealth, "projecting");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("workbench runtime retryable dependency failures classify as 503 (#2020)", () => {
|
||||||
|
const error = new Error("fetch failed");
|
||||||
|
error.name = "WorkbenchRuntimeDependencyError";
|
||||||
|
error.code = "workbench_runtime_unreachable";
|
||||||
|
error.data = {
|
||||||
|
serviceId: "hwlab-workbench-runtime",
|
||||||
|
retryable: true,
|
||||||
|
transient: true,
|
||||||
|
valuesRedacted: true
|
||||||
|
};
|
||||||
|
const classified = classifyWorkbenchReadModelFailure(error);
|
||||||
|
assert.equal(classified.statusCode, 503);
|
||||||
|
assert.equal(classified.code, "workbench_runtime_unreachable");
|
||||||
|
assert.equal(classified.retryable, true);
|
||||||
|
assert.equal(classified.transient, true);
|
||||||
|
assert.equal(classified.serviceId, "hwlab-workbench-runtime");
|
||||||
|
});
|
||||||
|
|
||||||
test("workbench turn projection keeps lastEventAt on the latest event instead of stale admission timing (#1889)", () => {
|
test("workbench turn projection keeps lastEventAt on the latest event instead of stale admission timing (#1889)", () => {
|
||||||
const traceId = "trc_turn_timing_latest_event";
|
const traceId = "trc_turn_timing_latest_event";
|
||||||
const result = {
|
const result = {
|
||||||
|
|||||||
@@ -2295,19 +2295,31 @@ function handleWorkbenchReadModelFailure(response, url, options = {}, error) {
|
|||||||
valuesRedacted: true
|
valuesRedacted: true
|
||||||
});
|
});
|
||||||
if (response.headersSent || response.destroyed) return;
|
if (response.headersSent || response.destroyed) return;
|
||||||
sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { route, errorName: error?.name ?? "Error" }));
|
sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, {
|
||||||
|
route,
|
||||||
|
errorName: error?.name ?? "Error",
|
||||||
|
retryable: classified.retryable === true,
|
||||||
|
transient: classified.transient === true,
|
||||||
|
serviceId: classified.serviceId ?? null,
|
||||||
|
runtimeStatus: classified.runtimeStatus ?? null
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
function classifyWorkbenchReadModelFailure(error) {
|
export function classifyWorkbenchReadModelFailure(error) {
|
||||||
const message = String(error?.message ?? error ?? "");
|
const message = String(error?.message ?? error ?? "");
|
||||||
const code = String(error?.code ?? "").toLowerCase();
|
const code = String(error?.code ?? "").toLowerCase();
|
||||||
const runtimeStatus = Number(error?.data?.status ?? 0);
|
const runtimeStatus = Number(error?.data?.status ?? 0);
|
||||||
const runtimePayloadError = error?.data?.payload?.error ?? null;
|
const runtimePayloadError = error?.data?.payload?.error ?? null;
|
||||||
if (error?.name === "WorkbenchRuntimeDependencyError" || code.startsWith("workbench_runtime_")) {
|
if (error?.name === "WorkbenchRuntimeDependencyError" || code.startsWith("workbench_runtime_")) {
|
||||||
|
const retryable = error?.data?.retryable === true || error?.data?.transient === true || runtimePayloadError?.retryable === true;
|
||||||
return {
|
return {
|
||||||
statusCode: Number.isFinite(runtimeStatus) && runtimeStatus >= 500 ? 503 : 500,
|
statusCode: retryable || (Number.isFinite(runtimeStatus) && runtimeStatus >= 500) ? 503 : 500,
|
||||||
code: String(runtimePayloadError?.code ?? error?.code ?? "workbench_runtime_dependency_failed"),
|
code: String(runtimePayloadError?.code ?? error?.code ?? "workbench_runtime_dependency_failed"),
|
||||||
message: String(runtimePayloadError?.message ?? error?.message ?? "Workbench runtime dependency failed.")
|
message: String(runtimePayloadError?.message ?? error?.message ?? "Workbench runtime dependency failed."),
|
||||||
|
retryable,
|
||||||
|
transient: retryable,
|
||||||
|
serviceId: String(error?.data?.serviceId ?? "hwlab-workbench-runtime"),
|
||||||
|
runtimeStatus: Number.isFinite(runtimeStatus) && runtimeStatus > 0 ? runtimeStatus : null
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (/timeout|connection terminated|terminating connection|econnreset|econnrefused|etimedout/iu.test(message) || /timeout|econnreset|econnrefused|etimedout/u.test(code)) {
|
if (/timeout|connection terminated|terminating connection|econnreset|econnrefused|etimedout/iu.test(message) || /timeout|econnreset|econnrefused|etimedout/u.test(code)) {
|
||||||
|
|||||||
Reference in New Issue
Block a user