fix: keep long Workbench retries visible

This commit is contained in:
root
2026-07-20 14:05:31 +02:00
parent 68f57e0eb4
commit 859c771bcd
5 changed files with 228 additions and 68 deletions
+58
View File
@@ -216,6 +216,64 @@ test("projects AgentRun semantic infrastructure retry without reclassification",
assert.equal(projected.event.runnerJobId, "rjob_retry");
});
test("normalizes sparse AgentRun provider retry metadata for Workbench visibility", () => {
const projected = projectAgentRunKafkaEventToHwlabEvent({
schema: "agentrun.event.v1",
eventType: "agentrun.run.event",
producedAt: "2026-07-20T19:32:07.000Z",
run: { runId: "run_provider_retry", sessionId: "ses_provider_retry", status: "running" },
command: { commandId: "cmd_provider_retry", state: "running" },
event: {
id: "evt_provider_retry",
seq: 52,
type: "error",
createdAt: "2026-07-20T19:32:07.000Z",
payload: {
failureKind: "provider-stream-disconnected",
message: "Provider stream disconnected",
willRetry: true,
retryAttempt: 1,
retryMax: 5,
retryDelayMs: 240_000
}
}
}, { source: "hwlab-test" });
assert.equal(projected.event.status, "retrying");
assert.equal(projected.event.failureDomain, "upstream");
assert.equal(projected.event.retryPhase, "retryScheduled");
assert.equal(projected.event.retryAttempt, 1);
assert.equal(projected.event.retryMaxAttempts, 5);
assert.equal(projected.event.retryBackoffMs, 240_000);
assert.equal(projected.event.nextRetryAt, "2026-07-20T19:36:07.000Z");
assert.equal(projected.event.terminal, false);
});
test("keeps sparse non-retryable provider errors failed", () => {
const projected = projectAgentRunKafkaEventToHwlabEvent({
schema: "agentrun.event.v1",
eventType: "agentrun.run.event",
producedAt: "2026-07-20T19:32:07.000Z",
run: { runId: "run_provider_failed", sessionId: "ses_provider_failed", status: "failed" },
command: { commandId: "cmd_provider_failed", state: "failed" },
event: {
id: "evt_provider_failed",
seq: 53,
type: "error",
createdAt: "2026-07-20T19:32:07.000Z",
payload: {
failureKind: "provider-unavailable",
message: "Provider unavailable",
willRetry: false
}
}
}, { source: "hwlab-test" });
assert.equal(projected.event.status, "failed");
assert.equal(projected.event.failureDomain, "upstream");
assert.equal(projected.event.retryPhase, null);
});
test("projects AgentRun Git mirror fetch as running preparation", () => {
const projected = projectAgentRunKafkaEventToHwlabEvent({
schema: "agentrun.event.v1",
+37 -16
View File
@@ -1610,7 +1610,7 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run,
}
if (type === "backend_status") {
const phase = firstText(payload.phase) || "status";
const semanticRetry = semanticFailureRetryProjection(payload);
const semanticRetry = semanticFailureRetryProjection(payload, sourceEvent.createdAt);
return {
...base,
type: "backend",
@@ -1678,26 +1678,31 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run,
}
if (type === "error") {
const failureKind = firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode) || "backend";
const semanticRetry = semanticFailureRetryProjection(payload);
const semanticRetry = semanticFailureRetryProjection(payload, sourceEvent.createdAt);
return { ...base, type: "error", eventType: "error", status: semanticRetry.status ?? "failed", label: `agentrun:error:${failureKind}`, errorCode: firstText(payload.code, failureKind), failureKind, message: textPayload(payload, "AgentRun error"), ...semanticRetry.fields, terminal: false };
}
return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${type}`, message: textPayload(payload, firstText(input.eventType, type) || "AgentRun event") };
}
function semanticFailureRetryProjection(payload) {
const retryPhase = firstText(payload.retryPhase);
const failureDomain = firstText(payload.failureDomain);
function semanticFailureRetryProjection(payload, eventCreatedAt = null) {
const failureKind = firstText(payload.failureKind, payload.errorCode);
const willRetry = payload.willRetry === true;
const retryPhase = firstText(payload.retryPhase)
|| (payload.retryExhausted === true ? "retryExhausted" : willRetry ? "retryScheduled" : null);
const failureDomain = firstText(payload.failureDomain) || inferredFailureDomain(failureKind);
const component = firstText(payload.component);
const code = firstText(payload.code);
const code = firstText(payload.code, failureKind);
if (!retryPhase && !failureDomain && !component && !code) return { status: null, fields: {} };
const normalized = String(retryPhase ?? "").toLowerCase();
const status = normalized.includes("exhausted")
? "failed"
: normalized.includes("recovered")
? "running"
: normalized.includes("scheduled") || (normalized.includes("retry") && normalized.includes("started"))
? "retrying"
: "running";
const status = !normalized
? null
: normalized.includes("exhausted")
? "failed"
: normalized.includes("recovered")
? "running"
: normalized.includes("scheduled") || (normalized.includes("retry") && normalized.includes("started"))
? "retrying"
: "running";
return {
status,
fields: {
@@ -1708,11 +1713,11 @@ function semanticFailureRetryProjection(payload) {
failureCode: code,
code,
summary: firstText(payload.summary, payload.message),
retryable: explicitBoolean(payload.retryable),
retryable: explicitBoolean(payload.retryable) ?? (willRetry ? true : null),
retryAttempt: integerValue(payload.attempt ?? payload.retryAttempt),
retryMaxAttempts: integerValue(payload.maxAttempts ?? payload.retryMaxAttempts),
retryMaxAttempts: integerValue(payload.maxAttempts ?? payload.retryMaxAttempts ?? payload.retryMax),
retryBackoffMs: integerValue(payload.backoffMs ?? payload.retryDelayMs),
nextRetryAt: timestampValue(payload.nextRetryAt),
nextRetryAt: retryTimestamp(payload.nextRetryAt, eventCreatedAt, payload.backoffMs ?? payload.retryDelayMs),
firstObservedAt: timestampValue(payload.firstObservedAt),
observedAt: timestampValue(payload.observedAt),
runnerJobId: firstText(payload.runnerJobId),
@@ -1721,6 +1726,22 @@ function semanticFailureRetryProjection(payload) {
};
}
function inferredFailureDomain(failureKind) {
const normalized = String(failureKind ?? "").trim().toLowerCase().replace(/_/gu, "-");
if (!normalized) return null;
if (normalized.startsWith("provider-") || normalized.includes("upstream")) return "upstream";
return "infrastructure";
}
function retryTimestamp(explicitValue, eventCreatedAt, retryDelayMs) {
const explicit = timestampValue(explicitValue);
if (explicit) return explicit;
const createdAt = timestampValue(eventCreatedAt);
const delayMs = integerValue(retryDelayMs);
if (!createdAt || delayMs === null || delayMs < 0) return null;
return new Date(Date.parse(createdAt) + delayMs).toISOString();
}
function eventMatchesFilters(value, filters) {
return Object.values(eventFilterResults(value, filters)).every(Boolean);
}