// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // Responsibility: AgentRun projection polling/backoff I/O, durable polling state, and projection_sync observability attributes. import { agentRunJson } from "./code-agent-agentrun-runtime.ts"; import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; import { parsePositiveInteger, safeTraceId, text } from "./server-http-utils.ts"; const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS = 1_000; const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS = 30_000; const AGENTRUN_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "cancelled", "canceled", "timeout"]); export function agentRunProjectionPollCount(projectionState = null) { return nonNegativeNumber(projectionState?.pollCount) + 1; } export function agentRunProjectionPollingState({ projectionState = null, env = process.env, pollCount = null, cursorAdvance = 0, terminalFromEvents = null, error = null, rootCause = null, terminalConsistencyGap = 0 } = {}) { const retryable = agentRunPollingRetryableError(error); const noProgress = !error && !terminalFromEvents && Number(cursorAdvance ?? 0) <= 0; const previousNoProgress = nonNegativeNumber(projectionState?.noProgressPollCount); const noProgressPollCount = retryable || noProgress ? previousNoProgress + 1 : 0; const backoffReason = retryable ? "provider_retry_or_rate_limit" : noProgress ? "projection_no_progress" : null; const backoffMs = backoffReason ? agentRunProjectionBackoffMs(noProgressPollCount, env) : 0; return { pollCount: Number.isInteger(Number(pollCount)) && Number(pollCount) > 0 ? Math.floor(Number(pollCount)) : agentRunProjectionPollCount(projectionState), noProgressPollCount, backoffMs, backoffReason, rootCause: boundedAgentRunRootCause(rootCause ?? agentRunPollingRootCause(error) ?? backoffReason ?? "projection_sync"), terminalConsistencyGap: nonNegativeNumber(terminalConsistencyGap), nextRetryAt: backoffMs > 0 ? new Date(Date.now() + backoffMs).toISOString() : null, valuesPrinted: false }; } export function agentRunPollingRetryableError(error = null) { const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0); if ([408, 425, 429, 500, 502, 503, 504].includes(statusCode)) return true; const code = String(error?.code ?? error?.agentRunError?.failureKind ?? "").toLowerCase(); return error?.retryable === true || code === "agentrun_timeout" || /rate.?limit|retry|timeout|temporar|unavailable|overload/u.test(code); } export function agentRunPollingRootCause(error = null) { if (!error) return null; const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0); if (statusCode === 429) return "provider_rate_limited"; if ([408, 425, 500, 502, 503, 504].includes(statusCode)) return `provider_retry_http_${statusCode}`; return error?.code ?? error?.agentRunError?.failureKind ?? error?.name ?? "agentrun_sync_failed"; } export function agentRunErrorWithPolling(error, polling = null) { if (!error || !polling) return error; error.polling = polling; error.backoffMs = polling.backoffMs; error.nextRetryAt = polling.nextRetryAt; error.rootCause = polling.rootCause; return error; } export function agentRunTerminalConsistencyGap({ mapped = {}, eventsResponse = null, result = null } = {}) { const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {}; const terminal = agentRunPollingTerminalStatus(result?.terminalStatus ?? result?.commandState ?? result?.status ?? mapped?.status ?? agentRun.terminalStatus ?? agentRun.commandState) || Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events)); if (!terminal) return 0; const latest = Math.max( nonNegativeNumber(eventsResponse?.maxSeq), nonNegativeNumber(result?.eventCount), nonNegativeNumber(result?.lastSeq), nonNegativeNumber(agentRun.lastSeq) ); const observed = Math.max( nonNegativeNumber(eventsResponse?.traceLastSeq), nonNegativeNumber(result?.lastSeq), nonNegativeNumber(agentRun.lastSeq) ); return Math.max(0, latest - observed); } export function agentRunTerminalResultRefreshSatisfied(mapped = {}, traceId = null) { const safeId = safeTraceId(traceId ?? mapped?.traceId); const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null; const traceSummary = mapped?.traceSummary && typeof mapped.traceSummary === "object" ? mapped.traceSummary : null; const finalResponse = mapped?.finalResponse && typeof mapped.finalResponse === "object" ? mapped.finalResponse : null; const providerTrace = agentRun?.providerTrace && typeof agentRun.providerTrace === "object" ? agentRun.providerTrace : mapped?.providerTrace && typeof mapped.providerTrace === "object" ? mapped.providerTrace : null; const commandId = text(agentRun?.commandId); const runId = text(agentRun?.runId); if (!safeId || !runId || !commandId) return false; if (!agentRunPollingTerminalStatus(mapped?.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? agentRun?.status)) return false; if (safeTraceId(mapped?.traceId ?? agentRun?.traceId) !== safeId) return false; if (safeTraceId(providerTrace?.traceId) !== safeId) return false; const finalTraceId = safeTraceId(finalResponse?.traceId ?? safeId); if (finalTraceId && finalTraceId !== safeId) return false; if (!String(finalResponse?.text ?? "").trim()) return false; if (traceSummary?.source !== "agentrun-command-result") return false; if (safeTraceId(traceSummary.traceId ?? safeId) !== safeId) return false; const summaryAgentRun = traceSummary.agentRun && typeof traceSummary.agentRun === "object" ? traceSummary.agentRun : null; if (text(summaryAgentRun?.runId) !== runId) return false; if (text(summaryAgentRun?.commandId) !== commandId) return false; return true; } export function emitAgentRunProjectionPollingSpan({ traceId, env = process.env, agentRun = {}, polling = {}, terminalRefreshSkipped = false } = {}) { void emitCodeAgentOtelSpan("projection_sync", traceId, env, { attributes: { runId: agentRun?.runId ?? null, commandId: agentRun?.commandId ?? null, terminalRefreshSkipped: terminalRefreshSkipped === true, pollCount: nonNegativeNumber(polling.pollCount), backoffMs: nonNegativeNumber(polling.backoffMs), rootCause: boundedAgentRunRootCause(polling.rootCause), terminalConsistencyGap: nonNegativeNumber(polling.terminalConsistencyGap) } }); } export async function fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) { const runId = requiredString(mapping.runId, "runId"); const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : ""; const replayWindow = agentRunTraceReplayWindow(mapping); const afterSeqOverride = Number(mapping.afterSeqOverride); const afterSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? Math.floor(afterSeqOverride) : replayWindow.afterSeq; const endSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? 0 : replayWindow.endSeq; const limit = parsePositiveInteger(mapping.eventsPageLimit ?? env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT, 500); const path = `/api/v1/runs/${encodeURIComponent(runId)}/events?afterSeq=${encodeURIComponent(String(afterSeq))}&limit=${encodeURIComponent(String(limit))}`; const response = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env }); const rawEvents = Array.isArray(response?.items) ? response.items : []; const events = rawEvents.filter((event) => agentRunEventBelongsToTrace(event, { currentCommandId, afterSeq, endSeq })); const maxSeq = Math.max(afterSeq, ...rawEvents.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); const traceLastSeq = Math.max(afterSeq, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); void emitCodeAgentOtelSpan("projection_sync", mapping.traceId ?? mapping.hwlabTraceId ?? mapping.businessTraceId, env, { attributes: { runId, commandId: currentCommandId || null, afterSeq, endSeq, limit, rawEventCount: rawEvents.length, eventCount: events.length, commandFiltered: Boolean(currentCommandId), maxSeq, traceLastSeq, terminalFromRawEvents: agentRunTerminalStatusFromEvents(rawEvents), terminalFromEvents: agentRunTerminalStatusFromEvents(events), pollCount: nonNegativeNumber(mapping.pollCount), backoffMs: nonNegativeNumber(mapping.backoffMs), rootCause: boundedAgentRunRootCause(mapping.rootCause), terminalConsistencyGap: nonNegativeNumber(mapping.terminalConsistencyGap) } }); return { events, afterSeq, endSeq, rawEventCount: rawEvents.length, commandFiltered: Boolean(currentCommandId), maxSeq, traceLastSeq }; } export async function measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) { const startedAt = nowMs(); try { const response = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping }); performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: "ok", durationMs: nowMs() - startedAt }); return response; } catch (error) { performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: agentRunMetricStatusForError(error), durationMs: nowMs() - startedAt }); performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_events_fetch_failed" }); throw error; } } export async function measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped = {}, eventsResponse = null }) { const startedAt = nowMs(); const path = `/api/v1/runs/${encodeURIComponent(mapped.agentRun.runId)}/commands/${encodeURIComponent(mapped.agentRun.commandId)}/result`; try { const result = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env }); const eventCount = agentRunResultEventCount(result, eventsResponse); performanceStore?.recordAgentRunResult?.({ durationMs: nowMs() - startedAt, eventCount, eventsScanned: eventCount, pagesScanned: estimateAgentRunResultPages(eventCount), status: result?.terminalStatus ?? result?.status ?? "ok", budget: timeoutMs }); return result; } catch (error) { const status = agentRunMetricStatusForError(error); const eventCount = agentRunResultEventCount(null, eventsResponse); performanceStore?.recordAgentRunResult?.({ durationMs: nowMs() - startedAt, eventCount, eventsScanned: eventCount, pagesScanned: estimateAgentRunResultPages(eventCount), status, budget: timeoutMs, timedOut: status === "timeout" }); performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_result_failed" }); throw error; } } export function recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse = null, previousLastSeq = 0, nextLastSeq = 0) { if (!performanceStore || !eventsResponse) return; const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0)); if (cursorAdvance > 0) { performanceStore.recordWorkbenchProjectionCursorAdvance?.({ status: "advanced", reason: eventsResponse.commandFiltered ? "command_filtered_events" : "run_events", projectionStatus: "projecting", source: "agentrun_v01", count: cursorAdvance }); } const counts = new Map(); for (const event of eventsResponse.events ?? []) { const kind = String(event?.type ?? event?.payload?.phase ?? "unknown").trim() || "unknown"; counts.set(kind, (counts.get(kind) ?? 0) + 1); } if (counts.size === 0 && Number(eventsResponse.rawEventCount ?? 0) > 0) counts.set("filtered", Number(eventsResponse.rawEventCount)); for (const [eventKind, count] of counts.entries()) { performanceStore.recordWorkbenchProjectorEvents?.({ eventKind, status: "ok", count }); } } export function agentRunMetricStatusForError(error) { return error?.code === "agentrun_timeout" || error?.name === "AbortError" || Number(error?.statusCode) === 504 ? "timeout" : "error"; } export function agentRunTraceCursorSeq(eventsResponse = {}, previousLastSeq = 0) { const afterSeq = Number(eventsResponse.afterSeq ?? 0); const endSeq = Number(eventsResponse.endSeq ?? 0); const traceLastSeq = Number(eventsResponse.traceLastSeq ?? 0); if (Number.isFinite(traceLastSeq) && traceLastSeq > afterSeq) return Math.floor(traceLastSeq); if (Number.isFinite(endSeq) && endSeq > 0) return Math.floor(endSeq); if (eventsResponse.commandFiltered === true) return Math.max(Number(previousLastSeq ?? 0), afterSeq); return Math.max(Number(previousLastSeq ?? 0), Number(eventsResponse.maxSeq ?? 0)); } export function agentRunTerminalStatusFromEvents(events = []) { if (!Array.isArray(events)) return null; for (const event of [...events].reverse()) { const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; const phase = payload.phase ?? event?.phase; const candidates = [ event?.type === "terminal_status" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null, phase === "command-terminal" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null, phase === "turn:completed" || phase === "turn/steer:completed" ? "completed" : null, event?.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null, payload.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null ]; for (const value of candidates) { const status = normalizeAgentRunStatus(value); if (agentRunPollingTerminalStatus(status)) return status; } } return null; } export function agentRunEventCommandId(event) { const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; if (typeof event?.commandId === "string") return event.commandId; return typeof payload.commandId === "string" ? payload.commandId : ""; } export function nonNegativeNumber(value) { const number = Number(value ?? 0); return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0; } function agentRunProjectionBackoffMs(attempt, env = process.env) { const initialMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS); const maxMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS); const exponent = Math.max(0, Math.min(10, nonNegativeNumber(attempt) - 1)); return Math.min(maxMs, initialMs * 2 ** exponent); } function boundedAgentRunRootCause(value) { const raw = String(value ?? "").trim().replace(/[^a-z0-9_.:-]+/giu, "_").replace(/^_+|_+$/gu, "").toLowerCase(); return raw ? raw.slice(0, 120) : null; } function agentRunResultEventCount(result = null, eventsResponse = null) { const candidates = [ result?.traceSummary?.sourceEventCount, result?.traceSummary?.eventCount, result?.sourceEventCount, result?.eventCount, result?.providerTrace?.eventCount, eventsResponse?.maxSeq, eventsResponse?.rawEventCount, eventsResponse?.events?.length ]; for (const value of candidates) { const count = Number(value); if (Number.isFinite(count) && count >= 0) return Math.floor(count); } return 0; } function estimateAgentRunResultPages(eventCount) { const count = Number(eventCount); if (!Number.isFinite(count) || count <= 0) return 0; return Math.max(1, Math.ceil(count / 500)); } function agentRunTraceReplayWindow(mapping = {}) { const eventStartSeq = Number(mapping.eventStartSeq ?? mapping.commandStartSeq ?? mapping.startSeq ?? 0); const summary = mapping.traceSummary && typeof mapping.traceSummary === "object" ? mapping.traceSummary : null; const summaryAgentRun = summary?.agentRun && typeof summary.agentRun === "object" ? summary.agentRun : null; const summaryLastSeq = Number(summaryAgentRun?.lastSeq ?? summary?.lastSeq ?? 0); const currentLastSeq = Number(mapping.lastSeq ?? 0); const endSeq = Number.isFinite(summaryLastSeq) && summaryLastSeq > 0 ? Math.floor(summaryLastSeq) : 0; if (Number.isFinite(eventStartSeq) && eventStartSeq > 0) return { afterSeq: Math.max(0, Math.floor(eventStartSeq) - 1), endSeq }; if (endSeq > 0) return { afterSeq: Math.max(0, endSeq - 500), endSeq }; if (Number.isFinite(currentLastSeq) && currentLastSeq > 0) return { afterSeq: Math.floor(currentLastSeq), endSeq: 0 }; return { afterSeq: 0, endSeq: 0 }; } function agentRunEventBelongsToTrace(event, { currentCommandId = "", afterSeq = 0, endSeq = 0 } = {}) { const eventCommandId = agentRunEventCommandId(event); const seq = Number(event?.seq ?? 0); if (currentCommandId && eventCommandId && eventCommandId !== currentCommandId) return false; if (endSeq > 0 && Number.isFinite(seq)) return seq > afterSeq && seq <= endSeq; return true; } function agentRunPollingTerminalStatus(value) { return AGENTRUN_TERMINAL_STATUSES.has(normalizeAgentRunStatus(value)); } function normalizeAgentRunStatus(value) { const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); return status === "cancelled" ? "canceled" : status; } function requiredString(value, fieldName) { if (typeof value === "string" && value.trim()) return value.trim(); throw Object.assign(new Error(`AgentRun response missing ${fieldName}`), { code: "agentrun_response_invalid", statusCode: 502 }); } function nowMs() { if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); return Date.now(); }