fix(workbench): classify runtime dependency outages as retryable (#2021)
This commit is contained in:
@@ -7,6 +7,7 @@ import { test } from "bun:test";
|
||||
import { createCloudApiServer } from "./server.ts";
|
||||
import { createBackendPerformanceStore } from "./backend-performance.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts";
|
||||
import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
|
||||
import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
||||
|
||||
@@ -58,6 +59,24 @@ test("workbench projection diagnostics keeps projecting health distinct from cau
|
||||
assert.equal(diagnostics.projectionHealth, "projecting");
|
||||
});
|
||||
|
||||
test("workbench runtime retryable dependency failures classify as 503 (#2020)", () => {
|
||||
const error = new Error("fetch failed");
|
||||
error.name = "WorkbenchRuntimeDependencyError";
|
||||
error.code = "workbench_runtime_unreachable";
|
||||
error.data = {
|
||||
serviceId: "hwlab-workbench-runtime",
|
||||
retryable: true,
|
||||
transient: true,
|
||||
valuesRedacted: true
|
||||
};
|
||||
const classified = classifyWorkbenchReadModelFailure(error);
|
||||
assert.equal(classified.statusCode, 503);
|
||||
assert.equal(classified.code, "workbench_runtime_unreachable");
|
||||
assert.equal(classified.retryable, true);
|
||||
assert.equal(classified.transient, true);
|
||||
assert.equal(classified.serviceId, "hwlab-workbench-runtime");
|
||||
});
|
||||
|
||||
test("workbench turn projection keeps lastEventAt on the latest event instead of stale admission timing (#1889)", () => {
|
||||
const traceId = "trc_turn_timing_latest_event";
|
||||
const result = {
|
||||
|
||||
@@ -2295,19 +2295,31 @@ function handleWorkbenchReadModelFailure(response, url, options = {}, error) {
|
||||
valuesRedacted: true
|
||||
});
|
||||
if (response.headersSent || response.destroyed) return;
|
||||
sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { route, errorName: error?.name ?? "Error" }));
|
||||
sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, {
|
||||
route,
|
||||
errorName: error?.name ?? "Error",
|
||||
retryable: classified.retryable === true,
|
||||
transient: classified.transient === true,
|
||||
serviceId: classified.serviceId ?? null,
|
||||
runtimeStatus: classified.runtimeStatus ?? null
|
||||
}));
|
||||
}
|
||||
|
||||
function classifyWorkbenchReadModelFailure(error) {
|
||||
export function classifyWorkbenchReadModelFailure(error) {
|
||||
const message = String(error?.message ?? error ?? "");
|
||||
const code = String(error?.code ?? "").toLowerCase();
|
||||
const runtimeStatus = Number(error?.data?.status ?? 0);
|
||||
const runtimePayloadError = error?.data?.payload?.error ?? null;
|
||||
if (error?.name === "WorkbenchRuntimeDependencyError" || code.startsWith("workbench_runtime_")) {
|
||||
const retryable = error?.data?.retryable === true || error?.data?.transient === true || runtimePayloadError?.retryable === true;
|
||||
return {
|
||||
statusCode: Number.isFinite(runtimeStatus) && runtimeStatus >= 500 ? 503 : 500,
|
||||
statusCode: retryable || (Number.isFinite(runtimeStatus) && runtimeStatus >= 500) ? 503 : 500,
|
||||
code: String(runtimePayloadError?.code ?? error?.code ?? "workbench_runtime_dependency_failed"),
|
||||
message: String(runtimePayloadError?.message ?? error?.message ?? "Workbench runtime dependency failed.")
|
||||
message: String(runtimePayloadError?.message ?? error?.message ?? "Workbench runtime dependency failed."),
|
||||
retryable,
|
||||
transient: retryable,
|
||||
serviceId: String(error?.data?.serviceId ?? "hwlab-workbench-runtime"),
|
||||
runtimeStatus: Number.isFinite(runtimeStatus) && runtimeStatus > 0 ? runtimeStatus : null
|
||||
};
|
||||
}
|
||||
if (/timeout|connection terminated|terminating connection|econnreset|econnrefused|etimedout/iu.test(message) || /timeout|econnreset|econnrefused|etimedout/u.test(code)) {
|
||||
|
||||
Reference in New Issue
Block a user