diff --git a/internal/cloud/kafka-event-bridge.test.ts b/internal/cloud/kafka-event-bridge.test.ts index e11be694..722079ae 100644 --- a/internal/cloud/kafka-event-bridge.test.ts +++ b/internal/cloud/kafka-event-bridge.test.ts @@ -174,6 +174,48 @@ test("projects only explicit AgentRun lifecycle timestamps", () => { assert.equal(withoutFacts.event.blocking, false); }); +test("projects AgentRun semantic infrastructure retry without reclassification", () => { + const projected = projectAgentRunKafkaEventToHwlabEvent({ + schema: "agentrun.event.v1", + eventType: "agentrun.run.event", + producedAt: "2026-07-20T10:00:01.000Z", + run: { runId: "run_retry", sessionId: "ses_retry", status: "running" }, + command: { commandId: "cmd_retry", state: "pending" }, + event: { + id: "evt_retry", + seq: 5, + type: "backend_status", + createdAt: "2026-07-20T10:00:01.000Z", + payload: { + phase: "runner-startup-retry-scheduled", + retryPhase: "retryScheduled", + failureDomain: "infrastructure", + component: "runner-image-pull", + code: "runner-image-registry-unreachable", + summary: "Runner 镜像仓库暂时不可达", + retryable: true, + attempt: 2, + maxAttempts: 5, + backoffMs: 8_000, + nextRetryAt: "2026-07-20T10:00:09.000Z", + firstObservedAt: "2026-07-20T09:59:55.000Z", + observedAt: "2026-07-20T10:00:01.000Z", + runnerJobId: "rjob_retry" + } + } + }, { source: "hwlab-test" }); + assert.equal(projected.event.status, "retrying"); + assert.equal(projected.event.failureDomain, "infrastructure"); + assert.equal(projected.event.failureComponent, "runner-image-pull"); + assert.equal(projected.event.failureCode, "runner-image-registry-unreachable"); + assert.equal(projected.event.retryPhase, "retryScheduled"); + assert.equal(projected.event.retryAttempt, 2); + assert.equal(projected.event.retryMaxAttempts, 5); + assert.equal(projected.event.retryBackoffMs, 8_000); + assert.equal(projected.event.nextRetryAt, "2026-07-20T10:00:09.000Z"); + assert.equal(projected.event.runnerJobId, "rjob_retry"); +}); + test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", diff --git a/internal/cloud/kafka-event-bridge.ts b/internal/cloud/kafka-event-bridge.ts index f9ee3a1c..9876913c 100644 --- a/internal/cloud/kafka-event-bridge.ts +++ b/internal/cloud/kafka-event-bridge.ts @@ -1594,13 +1594,15 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, } if (type === "backend_status") { const phase = firstText(payload.phase) || "status"; + const semanticRetry = semanticFailureRetryProjection(payload); return { ...base, type: "backend", eventType: "backend", - status: "running", + status: semanticRetry.status ?? "running", label: `agentrun:backend:${phase}`, message: textPayload(payload, phase), + ...semanticRetry.fields, itemId: firstText(payload.itemId), finalSeal: payload.finalSeal === true, replyRef: payload.replyRef && typeof payload.replyRef === "object" && !Array.isArray(payload.replyRef) ? payload.replyRef : null, @@ -1660,11 +1662,47 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, } if (type === "error") { const failureKind = firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode) || "backend"; - return { ...base, type: "error", eventType: "error", status: "failed", label: `agentrun:error:${failureKind}`, errorCode: failureKind, failureKind, message: textPayload(payload, "AgentRun error"), terminal: false }; + const semanticRetry = semanticFailureRetryProjection(payload); + return { ...base, type: "error", eventType: "error", status: semanticRetry.status ?? "failed", label: `agentrun:error:${failureKind}`, errorCode: firstText(payload.code, failureKind), failureKind, message: textPayload(payload, "AgentRun error"), ...semanticRetry.fields, terminal: false }; } return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${type}`, message: textPayload(payload, firstText(input.eventType, type) || "AgentRun event") }; } +function semanticFailureRetryProjection(payload) { + const retryPhase = firstText(payload.retryPhase); + const failureDomain = firstText(payload.failureDomain); + const component = firstText(payload.component); + const code = firstText(payload.code); + if (!retryPhase && !failureDomain && !component && !code) return { status: null, fields: {} }; + const normalized = String(retryPhase ?? "").toLowerCase(); + const status = normalized.includes("exhausted") + ? "failed" + : normalized.includes("recovered") + ? "running" + : "retrying"; + return { + status, + fields: { + retryPhase, + failureDomain, + failureComponent: component, + component, + failureCode: code, + code, + summary: firstText(payload.summary, payload.message), + retryable: explicitBoolean(payload.retryable), + retryAttempt: integerValue(payload.attempt ?? payload.retryAttempt), + retryMaxAttempts: integerValue(payload.maxAttempts ?? payload.retryMaxAttempts), + retryBackoffMs: integerValue(payload.backoffMs ?? payload.retryDelayMs), + nextRetryAt: timestampValue(payload.nextRetryAt), + firstObservedAt: timestampValue(payload.firstObservedAt), + observedAt: timestampValue(payload.observedAt), + runnerJobId: firstText(payload.runnerJobId), + transitionKey: firstText(payload.transitionKey) + } + }; +} + function eventMatchesFilters(value, filters) { return Object.values(eventFilterResults(value, filters)).every(Boolean); } diff --git a/tools/src/hwlab-cli/trace-renderer.ts b/tools/src/hwlab-cli/trace-renderer.ts index 38548895..d2c65564 100644 --- a/tools/src/hwlab-cli/trace-renderer.ts +++ b/tools/src/hwlab-cli/trace-renderer.ts @@ -57,7 +57,12 @@ export function traceDisplayRows(trace: Record = {}, events: Tr if (renderedSourceEvents.has(sourceEventKey)) continue; renderedSourceEvents.add(sourceEventKey); } - if (!event || isNoisyTraceEvent(event)) continue; + if (!event) continue; + if (isSemanticFailureRetryEvent(event)) { + rows.push(traceSemanticFailureRetryRow(event, displayOptions)); + continue; + } + if (isNoisyTraceEvent(event)) continue; if (isSupersededTraceItemLifecycleEvent(event, orderedEvents, index)) continue; if (isToolTraceEvent(event)) { const identity = toolIdentity(event); @@ -111,6 +116,56 @@ export function traceDisplayRows(trace: Record = {}, events: Tr .map((event) => traceDisplayRow(event, displayOptions)); } +function isSemanticFailureRetryEvent(event: TraceEvent): boolean { + return Boolean(nonEmptyString(event.failureDomain) && nonEmptyString(event.retryPhase)); +} + +function traceSemanticFailureRetryRow(event: TraceEvent, options: ResolvedTraceDisplayRowsOptions): TraceEventRow { + const eventKey = traceEventIdentityToken(event, options.sequenceAuthority); + const domain = nonEmptyString(event.failureDomain) === "upstream" ? "上游故障" : "基础设施故障"; + const phase = normalizedRetryPhase(event.retryPhase); + const attempt = numberOrNull(event.retryAttempt ?? event.attempt); + const maxAttempts = numberOrNull(event.retryMaxAttempts ?? event.maxAttempts); + const backoffMs = numberOrNull(event.retryBackoffMs ?? event.backoffMs); + const summary = nonEmptyString(event.summary ?? event.message) ?? "执行依赖暂时不可用"; + const retryProgress = attempt !== null && maxAttempts !== null ? `第 ${attempt}/${maxAttempts} 次` : null; + const phaseText = phase.includes("exhausted") + ? "有限重试已耗尽" + : phase.includes("recovered") + ? "故障已恢复" + : phase.includes("started") + ? `${retryProgress ? `${retryProgress}重试` : "重试"}已开始` + : phase.includes("scheduled") + ? `${retryProgress ? `${retryProgress}重试` : "重试"}已安排${backoffMs !== null ? `，退让 ${formatRetryDelay(backoffMs)}` : ""}` + : "已观测"; + const details = [ + summary, + nonEmptyString(event.failureComponent ?? event.component) ? `组件：${nonEmptyString(event.failureComponent ?? event.component)}` : null, + nonEmptyString(event.failureCode ?? event.code) ? `代码：${nonEmptyString(event.failureCode ?? event.code)}` : null, + nonEmptyString(event.nextRetryAt) ? `下次重试：${nonEmptyString(event.nextRetryAt)}` : null + ].filter((value): value is string => Boolean(value)); + return { + rowId: `event:semantic-retry:${eventKey ?? `${event.failureCode ?? event.code ?? "failure"}:${attempt ?? 0}:${phase}`}`, + seq: traceEventDisplaySeq(event, options.sequenceAuthority), + sequenceAuthority: options.sequenceAuthority, + tone: phase.includes("exhausted") ? "blocked" : phase.includes("recovered") ? "ok" : "warn", + header: `${traceEventClock(event, options)} ${domain} · ${phaseText}`.trim(), + body: details.join("\n"), + bodyFormat: "text", + terminal: phase.includes("exhausted") ? true : undefined + }; +} + +function normalizedRetryPhase(value: unknown): string { + return String(value ?? "").trim().toLowerCase().replace(/[^a-z]/gu, ""); +} + +function formatRetryDelay(milliseconds: number): string { + if (milliseconds < 1_000) return `${milliseconds} 毫秒`; + const seconds = Math.ceil(milliseconds / 1_000); + return seconds < 60 ? `${seconds} 秒` : `${Math.floor(seconds / 60)} 分 ${seconds % 60} 秒`; +} + function isOuterFinalResponseTraceEvent( event: TraceEvent, options: ResolvedTraceDisplayRowsOptions, diff --git a/web/hwlab-cloud-web/src/components/agent/TraceTimeline.test.ts b/web/hwlab-cloud-web/src/components/agent/TraceTimeline.test.ts index c312f61a..ad3c6a08 100644 --- a/web/hwlab-cloud-web/src/components/agent/TraceTimeline.test.ts +++ b/web/hwlab-cloud-web/src/components/agent/TraceTimeline.test.ts @@ -134,6 +134,51 @@ describe("TraceTimeline sequence authority", () => { wrapper.unmount(); }); + it("renders semantic infrastructure retry as a readable row", async () => { + const wrapper = mount(TraceTimeline, { + props: { + autoExpanded: true, + trace: { + traceId: "trc_retry_timeline", + eventSource: "hwlab-kafka-sse", + status: "running", + eventCount: 1, + events: [{ + sourceEventId: "evt_retry_timeline", + runId: "run_retry", + sourceSeq: 5, + source: "agentrun.kafka", + type: "backend", + status: "retrying", + label: "agentrun:backend:runner-startup-retry-scheduled", + failureDomain: "infrastructure", + failureComponent: "runner-image-pull", + failureCode: "runner-image-registry-unreachable", + summary: "Runner 镜像仓库暂时不可达", + retryPhase: "retryScheduled", + retryAttempt: 2, + retryMaxAttempts: 5, + retryBackoffMs: 8_000, + nextRetryAt: "2026-07-20T10:00:09.000Z", + createdAt: "2026-07-20T10:00:01.000Z" + }] + } + } + }); + await nextTick(); + + expect(wrapper.text()).toContain("基础设施故障"); + const row = wrapper.findAll('[data-testid="trace-render-row"]').find((item) => item.text().includes("基础设施故障")); + expect(row).toBeDefined(); + if (!row) throw new Error("semantic retry row was not rendered"); + expect(row.attributes("data-event-status")).toBe("warn"); + expect(row.text()).toContain("基础设施故障"); + expect(row.text()).toContain("第 2/5 次重试已安排"); + expect(row.text()).toContain("Runner 镜像仓库暂时不可达"); + expect(row.text()).toContain("runner-image-registry-unreachable"); + wrapper.unmount(); + }); + it("keeps a tool row stable while projecting visual and accessibility state", async () => { const wrapper = mount(TraceTimeline, { props: { diff --git a/web/hwlab-cloud-web/src/components/workbench/WorkbenchMessageCard.test.ts b/web/hwlab-cloud-web/src/components/workbench/WorkbenchMessageCard.test.ts new file mode 100644 index 00000000..6baaaca3 --- /dev/null +++ b/web/hwlab-cloud-web/src/components/workbench/WorkbenchMessageCard.test.ts @@ -0,0 +1,63 @@ +// @vitest-environment jsdom + +import { mount } from "@vue/test-utils"; +import { describe, expect, it } from "vitest"; + +import WorkbenchMessageCard from "./WorkbenchMessageCard.vue"; + +describe("WorkbenchMessageCard semantic retry visibility", () => { + it("shows an infrastructure retry banner without opening Trace details", () => { + const wrapper = mount(WorkbenchMessageCard, { + props: { + detailEnabled: false, + message: { + id: "msg_retry_agent", + messageId: "msg_retry_agent", + role: "agent", + title: "Code Agent", + text: "", + status: "running", + traceId: "trc_retry_banner", + sessionId: "ses_retry_banner", + createdAt: "2026-07-20T10:00:00.000Z", + runnerTrace: { + traceId: "trc_retry_banner", + eventSource: "hwlab-kafka-sse", + status: "running", + eventCount: 1, + events: [{ + sourceEventId: "evt_retry_banner", + sourceSeq: 5, + type: "backend", + status: "retrying", + failureDomain: "infrastructure", + failureCode: "runner-image-registry-unreachable", + summary: "Runner 镜像仓库暂时不可达", + retryPhase: "retryScheduled", + retryAttempt: 2, + retryMaxAttempts: 5, + nextRetryAt: new Date(Date.now() + 12_000).toISOString(), + createdAt: new Date().toISOString() + }] + } + } + }, + global: { + stubs: { + TraceTimeline: true, + LoadingState: true, + MessageMarkdown: true, + ApiErrorDiagnostic: true, + StatusBadge: true + } + } + }); + + const banner = wrapper.get(".semantic-failure-banner"); + expect(banner.text()).toContain("基础设施故障：Runner 镜像仓库暂时不可达"); + expect(banner.text()).toContain("第 2/5 次重试已安排"); + expect(banner.text()).toMatch(/预计 \d+ 秒后开始/u); + expect(banner.text()).toContain("runner-image-registry-unreachable"); + wrapper.unmount(); + }); +}); diff --git a/web/hwlab-cloud-web/src/components/workbench/WorkbenchMessageCard.vue b/web/hwlab-cloud-web/src/components/workbench/WorkbenchMessageCard.vue index 19c39aee..0d858d7f 100644 --- a/web/hwlab-cloud-web/src/components/workbench/WorkbenchMessageCard.vue +++ b/web/hwlab-cloud-web/src/components/workbench/WorkbenchMessageCard.vue @@ -36,6 +36,7 @@ const diagnostic = computed(() => messageDiagnosticView(props.message)); const visibleText = computed(() => visibleMessageText(props.message)); const durationMeta = computed(() => messageDurationMeta(props.message)); const activityMeta = computed(() => messageActivityMeta(props.message)); +const semanticFailure = computed(() => semanticFailureView(props.message, nowMs.value)); const autoExpanded = computed(() => props.traceAutoExpanded ?? traceLifecycleExpanded(props.message, traceTimelinePolicy.value)); function isRunningMessage(message: ChatMessage): boolean { @@ -121,6 +122,51 @@ function formatDuration(ms: number): string { function traceStorageKey(message: ChatMessage): string { return `${props.storageKeyPrefix}.${message.traceId ?? message.runnerTrace?.traceId ?? message.id}`; } + +function semanticFailureView(message: ChatMessage, currentTimeMs: number): { title: string; detail: string; tone: string; code: string | null } | null { + const events = Array.isArray(message.runnerTrace?.events) ? message.runnerTrace.events : []; + const event = [...events].reverse().find((item) => typeof item.failureDomain === "string" && typeof item.retryPhase === "string"); + if (!event) return null; + const phase = String(event.retryPhase).toLowerCase().replace(/[^a-z]/gu, ""); + const domain = event.failureDomain === "upstream" ? "上游故障" : "基础设施故障"; + const summary = firstText(event.summary, event.message, event.failureCode, event.code, "执行依赖暂时不可用"); + const attempt = finiteInteger(event.retryAttempt ?? event.attempt); + const maxAttempts = finiteInteger(event.retryMaxAttempts ?? event.maxAttempts); + const progress = attempt !== null && maxAttempts !== null ? `第 ${attempt}/${maxAttempts} 次` : "本次"; + let detail = "正在判定有限重试"; + let tone = "warning"; + if (phase.includes("scheduled")) { + const nextRetryMs = timestampMs(event.nextRetryAt); + const remainingSeconds = nextRetryMs === null ? null : Math.max(0, Math.ceil((nextRetryMs - currentTimeMs) / 1_000)); + detail = `${progress}重试已安排${remainingSeconds === null ? "" : `，预计 ${remainingSeconds} 秒后开始`}`; + } else if (phase.includes("started")) { + detail = `${progress}重试已开始`; + } else if (phase.includes("recovered")) { + detail = "故障已恢复，继续执行"; + tone = "recovered"; + } else if (phase.includes("exhausted")) { + detail = "有限重试已耗尽，执行已失败"; + tone = "failed"; + } + return { + title: `${domain}：${summary}`, + detail, + tone, + code: firstText(event.failureCode, event.code) + }; +} + +function firstText(...values: unknown[]): string { + for (const value of values) { + if (typeof value === "string" && value.trim()) return value.trim(); + } + return ""; +} + +function finiteInteger(value: unknown): number | null { + const number = Number(value); + return Number.isSafeInteger(number) && number >= 0 ? number : null; +} + +