fix: surface semantic retry progress in workbench
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success

This commit is contained in:
root
2026-07-20 06:24:08 +02:00
parent ab17051d4c
commit 32e239f610
6 changed files with 327 additions and 3 deletions
+42
View File
@@ -174,6 +174,48 @@ test("projects only explicit AgentRun lifecycle timestamps", () => {
assert.equal(withoutFacts.event.blocking, false);
});
test("projects AgentRun semantic infrastructure retry without reclassification", () => {
const projected = projectAgentRunKafkaEventToHwlabEvent({
schema: "agentrun.event.v1",
eventType: "agentrun.run.event",
producedAt: "2026-07-20T10:00:01.000Z",
run: { runId: "run_retry", sessionId: "ses_retry", status: "running" },
command: { commandId: "cmd_retry", state: "pending" },
event: {
id: "evt_retry",
seq: 5,
type: "backend_status",
createdAt: "2026-07-20T10:00:01.000Z",
payload: {
phase: "runner-startup-retry-scheduled",
retryPhase: "retryScheduled",
failureDomain: "infrastructure",
component: "runner-image-pull",
code: "runner-image-registry-unreachable",
summary: "Runner 镜像仓库暂时不可达",
retryable: true,
attempt: 2,
maxAttempts: 5,
backoffMs: 8_000,
nextRetryAt: "2026-07-20T10:00:09.000Z",
firstObservedAt: "2026-07-20T09:59:55.000Z",
observedAt: "2026-07-20T10:00:01.000Z",
runnerJobId: "rjob_retry"
}
}
}, { source: "hwlab-test" });
assert.equal(projected.event.status, "retrying");
assert.equal(projected.event.failureDomain, "infrastructure");
assert.equal(projected.event.failureComponent, "runner-image-pull");
assert.equal(projected.event.failureCode, "runner-image-registry-unreachable");
assert.equal(projected.event.retryPhase, "retryScheduled");
assert.equal(projected.event.retryAttempt, 2);
assert.equal(projected.event.retryMaxAttempts, 5);
assert.equal(projected.event.retryBackoffMs, 8_000);
assert.equal(projected.event.nextRetryAt, "2026-07-20T10:00:09.000Z");
assert.equal(projected.event.runnerJobId, "rjob_retry");
});
test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => {
const projected = projectAgentRunKafkaEventToHwlabEvent({
schema: "agentrun.event.v1",
+40 -2
View File
@@ -1594,13 +1594,15 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run,
}
if (type === "backend_status") {
const phase = firstText(payload.phase) || "status";
const semanticRetry = semanticFailureRetryProjection(payload);
return {
...base,
type: "backend",
eventType: "backend",
status: "running",
status: semanticRetry.status ?? "running",
label: `agentrun:backend:${phase}`,
message: textPayload(payload, phase),
...semanticRetry.fields,
itemId: firstText(payload.itemId),
finalSeal: payload.finalSeal === true,
replyRef: payload.replyRef && typeof payload.replyRef === "object" && !Array.isArray(payload.replyRef) ? payload.replyRef : null,
@@ -1660,11 +1662,47 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run,
}
if (type === "error") {
const failureKind = firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode) || "backend";
return { ...base, type: "error", eventType: "error", status: "failed", label: `agentrun:error:${failureKind}`, errorCode: failureKind, failureKind, message: textPayload(payload, "AgentRun error"), terminal: false };
const semanticRetry = semanticFailureRetryProjection(payload);
return { ...base, type: "error", eventType: "error", status: semanticRetry.status ?? "failed", label: `agentrun:error:${failureKind}`, errorCode: firstText(payload.code, failureKind), failureKind, message: textPayload(payload, "AgentRun error"), ...semanticRetry.fields, terminal: false };
}
return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${type}`, message: textPayload(payload, firstText(input.eventType, type) || "AgentRun event") };
}
function semanticFailureRetryProjection(payload) {
const retryPhase = firstText(payload.retryPhase);
const failureDomain = firstText(payload.failureDomain);
const component = firstText(payload.component);
const code = firstText(payload.code);
if (!retryPhase && !failureDomain && !component && !code) return { status: null, fields: {} };
const normalized = String(retryPhase ?? "").toLowerCase();
const status = normalized.includes("exhausted")
? "failed"
: normalized.includes("recovered")
? "running"
: "retrying";
return {
status,
fields: {
retryPhase,
failureDomain,
failureComponent: component,
component,
failureCode: code,
code,
summary: firstText(payload.summary, payload.message),
retryable: explicitBoolean(payload.retryable),
retryAttempt: integerValue(payload.attempt ?? payload.retryAttempt),
retryMaxAttempts: integerValue(payload.maxAttempts ?? payload.retryMaxAttempts),
retryBackoffMs: integerValue(payload.backoffMs ?? payload.retryDelayMs),
nextRetryAt: timestampValue(payload.nextRetryAt),
firstObservedAt: timestampValue(payload.firstObservedAt),
observedAt: timestampValue(payload.observedAt),
runnerJobId: firstText(payload.runnerJobId),
transitionKey: firstText(payload.transitionKey)
}
};
}
function eventMatchesFilters(value, filters) {
return Object.values(eventFilterResults(value, filters)).every(Boolean);
}
+56 -1
View File
@@ -57,7 +57,12 @@ export function traceDisplayRows(trace: Record<string, unknown> = {}, events: Tr
if (renderedSourceEvents.has(sourceEventKey)) continue;
renderedSourceEvents.add(sourceEventKey);
}
if (!event || isNoisyTraceEvent(event)) continue;
if (!event) continue;
if (isSemanticFailureRetryEvent(event)) {
rows.push(traceSemanticFailureRetryRow(event, displayOptions));
continue;
}
if (isNoisyTraceEvent(event)) continue;
if (isSupersededTraceItemLifecycleEvent(event, orderedEvents, index)) continue;
if (isToolTraceEvent(event)) {
const identity = toolIdentity(event);
@@ -111,6 +116,56 @@ export function traceDisplayRows(trace: Record<string, unknown> = {}, events: Tr
.map((event) => traceDisplayRow(event, displayOptions));
}
function isSemanticFailureRetryEvent(event: TraceEvent): boolean {
return Boolean(nonEmptyString(event.failureDomain) && nonEmptyString(event.retryPhase));
}
function traceSemanticFailureRetryRow(event: TraceEvent, options: ResolvedTraceDisplayRowsOptions): TraceEventRow {
const eventKey = traceEventIdentityToken(event, options.sequenceAuthority);
const domain = nonEmptyString(event.failureDomain) === "upstream" ? "上游故障" : "基础设施故障";
const phase = normalizedRetryPhase(event.retryPhase);
const attempt = numberOrNull(event.retryAttempt ?? event.attempt);
const maxAttempts = numberOrNull(event.retryMaxAttempts ?? event.maxAttempts);
const backoffMs = numberOrNull(event.retryBackoffMs ?? event.backoffMs);
const summary = nonEmptyString(event.summary ?? event.message) ?? "执行依赖暂时不可用";
const retryProgress = attempt !== null && maxAttempts !== null ? `${attempt}/${maxAttempts}` : null;
const phaseText = phase.includes("exhausted")
? "有限重试已耗尽"
: phase.includes("recovered")
? "故障已恢复"
: phase.includes("started")
? `${retryProgress ? `${retryProgress}重试` : "重试"}已开始`
: phase.includes("scheduled")
? `${retryProgress ? `${retryProgress}重试` : "重试"}已安排${backoffMs !== null ? `,退让 ${formatRetryDelay(backoffMs)}` : ""}`
: "已观测";
const details = [
summary,
nonEmptyString(event.failureComponent ?? event.component) ? `组件:${nonEmptyString(event.failureComponent ?? event.component)}` : null,
nonEmptyString(event.failureCode ?? event.code) ? `代码:${nonEmptyString(event.failureCode ?? event.code)}` : null,
nonEmptyString(event.nextRetryAt) ? `下次重试:${nonEmptyString(event.nextRetryAt)}` : null
].filter((value): value is string => Boolean(value));
return {
rowId: `event:semantic-retry:${eventKey ?? `${event.failureCode ?? event.code ?? "failure"}:${attempt ?? 0}:${phase}`}`,
seq: traceEventDisplaySeq(event, options.sequenceAuthority),
sequenceAuthority: options.sequenceAuthority,
tone: phase.includes("exhausted") ? "blocked" : phase.includes("recovered") ? "ok" : "warn",
header: `${traceEventClock(event, options)} ${domain} · ${phaseText}`.trim(),
body: details.join("\n"),
bodyFormat: "text",
terminal: phase.includes("exhausted") ? true : undefined
};
}
function normalizedRetryPhase(value: unknown): string {
return String(value ?? "").trim().toLowerCase().replace(/[^a-z]/gu, "");
}
function formatRetryDelay(milliseconds: number): string {
if (milliseconds < 1_000) return `${milliseconds} 毫秒`;
const seconds = Math.ceil(milliseconds / 1_000);
return seconds < 60 ? `${seconds}` : `${Math.floor(seconds / 60)}${seconds % 60}`;
}
function isOuterFinalResponseTraceEvent(
event: TraceEvent,
options: ResolvedTraceDisplayRowsOptions,
@@ -134,6 +134,51 @@ describe("TraceTimeline sequence authority", () => {
wrapper.unmount();
});
it("renders semantic infrastructure retry as a readable row", async () => {
const wrapper = mount(TraceTimeline, {
props: {
autoExpanded: true,
trace: {
traceId: "trc_retry_timeline",
eventSource: "hwlab-kafka-sse",
status: "running",
eventCount: 1,
events: [{
sourceEventId: "evt_retry_timeline",
runId: "run_retry",
sourceSeq: 5,
source: "agentrun.kafka",
type: "backend",
status: "retrying",
label: "agentrun:backend:runner-startup-retry-scheduled",
failureDomain: "infrastructure",
failureComponent: "runner-image-pull",
failureCode: "runner-image-registry-unreachable",
summary: "Runner 镜像仓库暂时不可达",
retryPhase: "retryScheduled",
retryAttempt: 2,
retryMaxAttempts: 5,
retryBackoffMs: 8_000,
nextRetryAt: "2026-07-20T10:00:09.000Z",
createdAt: "2026-07-20T10:00:01.000Z"
}]
}
}
});
await nextTick();
expect(wrapper.text()).toContain("基础设施故障");
const row = wrapper.findAll('[data-testid="trace-render-row"]').find((item) => item.text().includes("基础设施故障"));
expect(row).toBeDefined();
if (!row) throw new Error("semantic retry row was not rendered");
expect(row.attributes("data-event-status")).toBe("warn");
expect(row.text()).toContain("基础设施故障");
expect(row.text()).toContain("第 2/5 次重试已安排");
expect(row.text()).toContain("Runner 镜像仓库暂时不可达");
expect(row.text()).toContain("runner-image-registry-unreachable");
wrapper.unmount();
});
it("keeps a tool row stable while projecting visual and accessibility state", async () => {
const wrapper = mount(TraceTimeline, {
props: {
@@ -0,0 +1,63 @@
// @vitest-environment jsdom
import { mount } from "@vue/test-utils";
import { describe, expect, it } from "vitest";
import WorkbenchMessageCard from "./WorkbenchMessageCard.vue";
describe("WorkbenchMessageCard semantic retry visibility", () => {
it("shows an infrastructure retry banner without opening Trace details", () => {
const wrapper = mount(WorkbenchMessageCard, {
props: {
detailEnabled: false,
message: {
id: "msg_retry_agent",
messageId: "msg_retry_agent",
role: "agent",
title: "Code Agent",
text: "",
status: "running",
traceId: "trc_retry_banner",
sessionId: "ses_retry_banner",
createdAt: "2026-07-20T10:00:00.000Z",
runnerTrace: {
traceId: "trc_retry_banner",
eventSource: "hwlab-kafka-sse",
status: "running",
eventCount: 1,
events: [{
sourceEventId: "evt_retry_banner",
sourceSeq: 5,
type: "backend",
status: "retrying",
failureDomain: "infrastructure",
failureCode: "runner-image-registry-unreachable",
summary: "Runner 镜像仓库暂时不可达",
retryPhase: "retryScheduled",
retryAttempt: 2,
retryMaxAttempts: 5,
nextRetryAt: new Date(Date.now() + 12_000).toISOString(),
createdAt: new Date().toISOString()
}]
}
}
},
global: {
stubs: {
TraceTimeline: true,
LoadingState: true,
MessageMarkdown: true,
ApiErrorDiagnostic: true,
StatusBadge: true
}
}
});
const banner = wrapper.get(".semantic-failure-banner");
expect(banner.text()).toContain("基础设施故障:Runner 镜像仓库暂时不可达");
expect(banner.text()).toContain("第 2/5 次重试已安排");
expect(banner.text()).toMatch(/ \d+ /u);
expect(banner.text()).toContain("runner-image-registry-unreachable");
wrapper.unmount();
});
});
@@ -36,6 +36,7 @@ const diagnostic = computed(() => messageDiagnosticView(props.message));
const visibleText = computed(() => visibleMessageText(props.message));
const durationMeta = computed(() => messageDurationMeta(props.message));
const activityMeta = computed(() => messageActivityMeta(props.message));
const semanticFailure = computed(() => semanticFailureView(props.message, nowMs.value));
const autoExpanded = computed(() => props.traceAutoExpanded ?? traceLifecycleExpanded(props.message, traceTimelinePolicy.value));
function isRunningMessage(message: ChatMessage): boolean {
@@ -121,6 +122,51 @@ function formatDuration(ms: number): string {
function traceStorageKey(message: ChatMessage): string {
return `${props.storageKeyPrefix}.${message.traceId ?? message.runnerTrace?.traceId ?? message.id}`;
}
function semanticFailureView(message: ChatMessage, currentTimeMs: number): { title: string; detail: string; tone: string; code: string | null } | null {
const events = Array.isArray(message.runnerTrace?.events) ? message.runnerTrace.events : [];
const event = [...events].reverse().find((item) => typeof item.failureDomain === "string" && typeof item.retryPhase === "string");
if (!event) return null;
const phase = String(event.retryPhase).toLowerCase().replace(/[^a-z]/gu, "");
const domain = event.failureDomain === "upstream" ? "上游故障" : "基础设施故障";
const summary = firstText(event.summary, event.message, event.failureCode, event.code, "执行依赖暂时不可用");
const attempt = finiteInteger(event.retryAttempt ?? event.attempt);
const maxAttempts = finiteInteger(event.retryMaxAttempts ?? event.maxAttempts);
const progress = attempt !== null && maxAttempts !== null ? `${attempt}/${maxAttempts}` : "本次";
let detail = "正在判定有限重试";
let tone = "warning";
if (phase.includes("scheduled")) {
const nextRetryMs = timestampMs(event.nextRetryAt);
const remainingSeconds = nextRetryMs === null ? null : Math.max(0, Math.ceil((nextRetryMs - currentTimeMs) / 1_000));
detail = `${progress}重试已安排${remainingSeconds === null ? "" : `,预计 ${remainingSeconds} 秒后开始`}`;
} else if (phase.includes("started")) {
detail = `${progress}重试已开始`;
} else if (phase.includes("recovered")) {
detail = "故障已恢复,继续执行";
tone = "recovered";
} else if (phase.includes("exhausted")) {
detail = "有限重试已耗尽,执行已失败";
tone = "failed";
}
return {
title: `${domain}${summary}`,
detail,
tone,
code: firstText(event.failureCode, event.code)
};
}
function firstText(...values: unknown[]): string {
for (const value of values) {
if (typeof value === "string" && value.trim()) return value.trim();
}
return "";
}
function finiteInteger(value: unknown): number | null {
const number = Number(value);
return Number.isSafeInteger(number) && number >= 0 ? number : null;
}
</script>
<template>
@@ -145,9 +191,44 @@ function traceStorageKey(message: ChatMessage): string {
<button v-if="detailEnabled && message.role === 'agent'" class="message-detail-button" type="button" aria-label="运行详情" title="运行详情" @click="emit('details', message)">!</button>
</div>
</header>
<section v-if="semanticFailure" class="semantic-failure-banner" :data-tone="semanticFailure.tone" role="status" aria-live="polite">
<strong>{{ semanticFailure.title }}</strong>
<span>{{ semanticFailure.detail }}</span>
<code v-if="semanticFailure.code">{{ semanticFailure.code }}</code>
</section>
<TraceTimeline v-if="message.role === 'agent' && message.runnerTrace" :trace="message.runnerTrace" :auto-expanded="autoExpanded" :storage-key="traceStorageKey(message)" final-response-placement="outer" :outer-final-response-text="visibleText" />
<LoadingState v-if="isAwaitingAgentBody(message)" class="message-loading" label="思考中..." compact />
<MessageMarkdown v-if="visibleText" class="message-text" :source="visibleText" />
<ApiErrorDiagnostic v-if="diagnostic.visible" class="message-diagnostic projection-diagnostic" :error="diagnostic.text" :api-error="diagnostic.apiError" :diagnostic="diagnostic.diagnostic" :show-message="diagnostic.showMessage" compact />
</article>
</template>
<style scoped>
.semantic-failure-banner {
display: grid;
gap: 0.2rem;
margin: 0.55rem 0;
border-left: 3px solid #d97706;
border-radius: 0.35rem;
background: #fffbeb;
color: #92400e;
padding: 0.55rem 0.7rem;
}
.semantic-failure-banner[data-tone="recovered"] {
border-left-color: #15803d;
background: #f0fdf4;
color: #166534;
}
.semantic-failure-banner[data-tone="failed"] {
border-left-color: #b91c1c;
background: #fef2f2;
color: #991b1b;
}
.semantic-failure-banner span,
.semantic-failure-banner code {
font-size: 0.82rem;
}
</style>