feat: make Kafka projector the Workbench realtime authority
This commit is contained in:
root
@@ -1,5 +1,5 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: AgentRun v0.1 adapter, incremental Workbench projection cursor sync, and upstream timing projection for Cloud API observability.
|
||||
// Responsibility: AgentRun command admission and control adapter; Kafka owns event/result projection.
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
@@ -13,32 +13,12 @@ import {
|
||||
safeTraceId,
|
||||
text
|
||||
} from "./server-http-utils.ts";
|
||||
import { currentBackendPerformanceContext } from "./backend-performance.ts";
|
||||
import {
|
||||
DEFAULT_AGENTRUN_MGR_URL,
|
||||
agentRunJson,
|
||||
isAgentRunManagerInternalServiceHost,
|
||||
resolveAgentRunManagerUrl
|
||||
} from "./code-agent-agentrun-runtime.ts";
|
||||
import {
|
||||
agentRunErrorWithPolling,
|
||||
agentRunEventCommandId,
|
||||
agentRunMetricStatusForError,
|
||||
agentRunPollingRetryableError,
|
||||
agentRunPollingRootCause,
|
||||
agentRunProjectionPollCount,
|
||||
agentRunProjectionPollingState,
|
||||
agentRunTerminalConsistencyGap,
|
||||
agentRunTerminalResultRefreshSatisfied,
|
||||
agentRunTerminalStatusFromEvents,
|
||||
agentRunTraceCursorSeq,
|
||||
emitAgentRunProjectionPollingSpan,
|
||||
fetchAgentRunEventsForTrace,
|
||||
measuredAgentRunEventsFetch,
|
||||
measuredAgentRunResultFetch,
|
||||
nonNegativeNumber,
|
||||
recordAgentRunEventsProjectionMetrics
|
||||
} from "./code-agent-agentrun-projection-polling.ts";
|
||||
import {
|
||||
agentRunAlreadyTerminalCancelPayload,
|
||||
agentRunCancelBlockedPayload,
|
||||
@@ -65,7 +45,6 @@ import {
|
||||
agentRunPromptMetadataFromText,
|
||||
messageAuthorityTextValue
|
||||
} from "./code-agent-agentrun-prompt.ts";
|
||||
import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts";
|
||||
import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import * as agentRunResult from "./code-agent-agentrun-result.ts";
|
||||
|
||||
@@ -700,333 +679,11 @@ function sleepAgentRunDispatchRetry(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0)));
|
||||
}
|
||||
|
||||
export async function syncAgentRunChatResult({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore, appendResultEvent = true, refreshEvents = true, forceResultSync = false, retryParams = null }) {
|
||||
const initial = currentResult ?? await loadPersistedAgentRunResult(traceId, options);
|
||||
if (isAgentRunUserCancelSealed(initial)) {
|
||||
const runnerTrace = initial.runnerTrace ?? traceStore.snapshot(traceId);
|
||||
return { result: initial, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true };
|
||||
}
|
||||
const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial;
|
||||
if (isAgentRunUserCancelSealed(mapped)) {
|
||||
const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId);
|
||||
return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true };
|
||||
}
|
||||
const env = options.env ?? process.env;
|
||||
const performanceStore = options.backendPerformanceStore ?? currentBackendPerformanceContext();
|
||||
if (!forceResultSync && agentRunTerminalResultRefreshSatisfied(mapped, traceId)) {
|
||||
const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId);
|
||||
const polling = agentRunProjectionPollingState({ env, terminalFromEvents: "completed", rootCause: "terminal_result_sealed", terminalConsistencyGap: 0 });
|
||||
emitAgentRunProjectionPollingSpan({ traceId, env, agentRun: mapped?.agentRun, polling, terminalRefreshSkipped: true });
|
||||
return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, polling };
|
||||
}
|
||||
if (!mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return { result: currentResult, runnerTrace: traceStore.snapshot(traceId), found: Boolean(currentResult), eventsRefreshed: false, resultSynced: false };
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
||||
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
||||
const projectionState = await loadWorkbenchProjectionStateForAgentRun(options.runtimeStore, { traceId, agentRun: mapped.agentRun });
|
||||
const projectionRetryParams = retryParams ?? projectionState?.retryParams ?? null;
|
||||
const projectionPollCount = agentRunProjectionPollCount(projectionState);
|
||||
const fetchPlan = buildAgentRunProjectionEventsFetchPlan({
|
||||
projectionState,
|
||||
agentRun: mapped.agentRun,
|
||||
pageLimit: env.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT
|
||||
});
|
||||
const previousLastSeq = Number(fetchPlan.afterSeq ?? mapped.agentRun.lastSeq ?? 0);
|
||||
let eventsResponse = null;
|
||||
try {
|
||||
eventsResponse = refreshEvents ? await measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, lastSeq: fetchPlan.afterSeq, afterSeqOverride: fetchPlan.afterSeq, eventsPageLimit: fetchPlan.limit, traceSummary: mapped.traceSummary, pollCount: projectionPollCount, backoffMs: nonNegativeNumber(projectionState?.backoffMs), rootCause: projectionState?.rootCause ?? null, terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) } }) : null;
|
||||
} catch (error) {
|
||||
const polling = agentRunProjectionPollingState({ projectionState, env, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) });
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: mapped,
|
||||
agentRun: { ...mapped.agentRun, lastSeq: previousLastSeq },
|
||||
eventsResponse: null,
|
||||
resultSyncState: projectionState?.resultSyncState ?? null,
|
||||
projectionStatus: "degraded",
|
||||
projectionHealth: "degraded",
|
||||
retryParams: projectionRetryParams,
|
||||
polling,
|
||||
error,
|
||||
now: options.now
|
||||
});
|
||||
throw agentRunErrorWithPolling(error, polling);
|
||||
}
|
||||
if (eventsResponse) appendAgentRunEventsToTrace(traceStore, traceId, eventsResponse.events, mapped.agentRun);
|
||||
const nextLastSeq = eventsResponse ? agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq) : mapped.agentRun.lastSeq;
|
||||
const backendProfile = resolveAgentRunBackendProfile(env, {
|
||||
providerProfile: firstNonEmpty(
|
||||
projectionRetryParams?.providerProfile,
|
||||
projectionRetryParams?.codeAgentProviderProfile,
|
||||
projectionRetryParams?.backendProfile,
|
||||
mapped.agentRun.backendProfile
|
||||
)
|
||||
});
|
||||
recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse, previousLastSeq, nextLastSeq);
|
||||
const terminalFromEvents = agentRunTerminalStatusFromEvents(eventsResponse?.events);
|
||||
const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0));
|
||||
const terminalConsistencyGap = agentRunTerminalConsistencyGap({ mapped, eventsResponse });
|
||||
const projectionPolling = agentRunProjectionPollingState({
|
||||
projectionState,
|
||||
env,
|
||||
pollCount: projectionPollCount,
|
||||
cursorAdvance,
|
||||
terminalFromEvents,
|
||||
rootCause: terminalFromEvents ? "terminal_from_events" : cursorAdvance > 0 ? "cursor_advanced" : "projection_no_progress",
|
||||
terminalConsistencyGap
|
||||
});
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: mapped,
|
||||
agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq },
|
||||
eventsResponse,
|
||||
resultSyncState: terminalFromEvents ? "pending" : null,
|
||||
projectionStatus: terminalFromEvents ? "terminal" : null,
|
||||
polling: projectionPolling,
|
||||
retryParams: projectionRetryParams,
|
||||
now: options.now
|
||||
});
|
||||
const deferResultSync = terminalFromEvents ? false : agentRunResultSyncDeferred({ forceResultSync, options, env });
|
||||
if (!forceResultSync && (deferResultSync || !agentRunCommandResultSyncRequired(mapped, eventsResponse))) {
|
||||
const nextMapping = withAgentRunRunnerJobCount({
|
||||
...mapped.agentRun,
|
||||
lastSeq: nextLastSeq,
|
||||
status: mapped.agentRun.status ?? "running",
|
||||
runStatus: mapped.agentRun.runStatus ?? null,
|
||||
commandState: mapped.agentRun.commandState ?? null,
|
||||
terminalStatus: null,
|
||||
resultSyncState: terminalFromEvents ? "pending" : mapped.agentRun.resultSyncState ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
});
|
||||
const payload = decorateAgentRunRunningResult({ base: { ...mapped, status: "running", agentRun: nextMapping, updatedAt: nowIso(options.now) }, mapping: nextMapping, traceStore, traceId });
|
||||
options.codeAgentChatResults?.set?.(traceId, payload);
|
||||
return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, polling: projectionPolling };
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
result = await measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped, eventsResponse });
|
||||
} catch (error) {
|
||||
const retryable = agentRunPollingRetryableError(error);
|
||||
const polling = agentRunProjectionPollingState({ projectionState, env, pollCount: projectionPollCount, cursorAdvance, terminalFromEvents, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap });
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: mapped,
|
||||
agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq },
|
||||
eventsResponse,
|
||||
resultSyncState: retryable ? "pending" : agentRunMetricStatusForError(error) === "timeout" ? "timed_out" : "failed",
|
||||
projectionStatus: retryable ? "degraded" : terminalFromEvents ? "terminal" : null,
|
||||
projectionHealth: "degraded",
|
||||
retryParams: projectionRetryParams,
|
||||
polling,
|
||||
error,
|
||||
now: options.now
|
||||
});
|
||||
throw agentRunErrorWithPolling(error, polling);
|
||||
}
|
||||
const nextMapping = withAgentRunRunnerJobCount({
|
||||
...mapped.agentRun,
|
||||
...agentRunResultRefs(result),
|
||||
lastSeq: nextLastSeq,
|
||||
status: result?.status ?? mapped.agentRun.status ?? "running",
|
||||
runStatus: result?.runStatus ?? mapped.agentRun.runStatus ?? null,
|
||||
commandState: result?.commandState ?? mapped.agentRun.commandState ?? null,
|
||||
terminalStatus: result?.terminalStatus ?? mapped.agentRun.terminalStatus ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
});
|
||||
const freshSessionRetry = await retryAgentRunTurnAfterFreshSessionFailure({
|
||||
params: projectionRetryParams,
|
||||
options,
|
||||
traceId,
|
||||
traceStore,
|
||||
mapped,
|
||||
failedResult: result,
|
||||
failedMapping: nextMapping,
|
||||
env,
|
||||
managerUrl,
|
||||
fetchImpl,
|
||||
timeoutMs,
|
||||
backendProfile
|
||||
});
|
||||
if (freshSessionRetry) {
|
||||
options.codeAgentChatResults?.set?.(traceId, freshSessionRetry.payload);
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: freshSessionRetry.payload,
|
||||
agentRun: freshSessionRetry.mapping,
|
||||
eventsResponse: null,
|
||||
resultSyncState: null,
|
||||
projectionStatus: "projecting",
|
||||
retryParams: projectionRetryParams,
|
||||
now: options.now
|
||||
});
|
||||
const retrySync = await syncAgentRunChatResult({
|
||||
traceId,
|
||||
currentResult: freshSessionRetry.payload,
|
||||
options,
|
||||
traceStore,
|
||||
appendResultEvent,
|
||||
refreshEvents: true,
|
||||
forceResultSync,
|
||||
retryParams: projectionRetryParams
|
||||
});
|
||||
if (retrySync?.result) {
|
||||
return {
|
||||
...retrySync,
|
||||
found: true,
|
||||
eventsRefreshed: Boolean(eventsResponse) || Boolean(retrySync.eventsRefreshed),
|
||||
freshSessionRetried: true
|
||||
};
|
||||
}
|
||||
return { result: freshSessionRetry.payload, runnerTrace: freshSessionRetry.payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, freshSessionRetried: true };
|
||||
}
|
||||
const base = { ...mapped, agentRun: nextMapping, updatedAt: nowIso(options.now) };
|
||||
const payload = agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent });
|
||||
options.codeAgentChatResults?.set?.(traceId, payload);
|
||||
const resultPolling = agentRunProjectionPollingState({
|
||||
projectionState,
|
||||
env,
|
||||
pollCount: projectionPollCount,
|
||||
cursorAdvance,
|
||||
terminalFromEvents: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? (nextMapping.terminalStatus ?? payload?.status) : terminalFromEvents,
|
||||
rootCause: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? "terminal_result_synced" : "result_synced",
|
||||
terminalConsistencyGap: agentRunTerminalConsistencyGap({ mapped: payload, eventsResponse, result })
|
||||
});
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: payload,
|
||||
agentRun: nextMapping,
|
||||
eventsResponse,
|
||||
resultSyncState: "synced",
|
||||
projectionStatus: isAgentRunTerminalStatus(payload?.status) ? "terminal" : null,
|
||||
polling: resultPolling,
|
||||
retryParams: projectionRetryParams,
|
||||
now: options.now
|
||||
});
|
||||
return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: true, polling: resultPolling };
|
||||
}
|
||||
|
||||
async function loadWorkbenchProjectionStateForAgentRun(runtimeStore, { traceId, agentRun = {} } = {}) {
|
||||
if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null;
|
||||
try {
|
||||
const result = await runtimeStore.getWorkbenchProjectionState({ traceId });
|
||||
const state = result?.projectionState ?? null;
|
||||
if (!state) return null;
|
||||
if (text(state.runId ?? state.sourceRunId) !== text(agentRun.runId)) return null;
|
||||
if (text(state.commandId ?? state.sourceCommandId) !== text(agentRun.commandId)) return null;
|
||||
return state;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeWorkbenchProjectionStateForAgentRun(runtimeStore, params = {}) {
|
||||
if (!runtimeStore || typeof runtimeStore.writeWorkbenchProjectionState !== "function") return null;
|
||||
const projectionState = buildWorkbenchProjectionStateUpdate(params);
|
||||
if (!projectionState.traceId || !projectionState.runId || !projectionState.commandId) return null;
|
||||
return await runtimeStore.writeWorkbenchProjectionState({ projectionState });
|
||||
}
|
||||
|
||||
function agentRunCommandResultSyncRequired(mapped = {}, eventsResponse = null) {
|
||||
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {};
|
||||
if (agentRunTerminalResultRefreshSatisfied(mapped, mapped?.traceId)) return true;
|
||||
if (isAgentRunTerminalStatus(agentRun.terminalStatus ?? agentRun.commandState)) return true;
|
||||
return Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events));
|
||||
}
|
||||
|
||||
function isAgentRunTerminalStatus(value) {
|
||||
const status = normalizeAgentRunStatus(value);
|
||||
return status === "timeout" || TERMINAL_RUN_STATUSES.has(status);
|
||||
}
|
||||
|
||||
function normalizeAgentRunStatus(value) {
|
||||
const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
||||
return status === "cancelled" ? "canceled" : status;
|
||||
}
|
||||
|
||||
function isAgentRunUserCancelSealed(payload = null) {
|
||||
if (!payload || typeof payload !== "object") return false;
|
||||
const status = normalizeAgentRunStatus(payload.status);
|
||||
const cancelStatus = normalizeAgentRunStatus(payload.cancelStatus ?? payload.cancelDisposition?.status);
|
||||
const terminalStatus = normalizeAgentRunStatus(payload.agentRun?.terminalStatus ?? payload.agentRun?.commandState ?? payload.agentRun?.status);
|
||||
return payload.canceled === true || status === "canceled" || cancelStatus === "canceled" || terminalStatus === "canceled";
|
||||
}
|
||||
|
||||
async function resolveAgentRunTraceCommandMapping({ traceId, mapped, options = {} }) {
|
||||
const safeId = safeTraceId(traceId);
|
||||
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null;
|
||||
if (!safeId || !agentRun?.runId) return mapped;
|
||||
const agentRunTraceId = safeTraceId(agentRun.traceId);
|
||||
const providerTraceId = safeTraceId(agentRun.providerTrace?.traceId);
|
||||
const traceSummaryCommandId = text(mapped?.traceSummary?.agentRun?.commandId);
|
||||
const commandId = text(agentRun.commandId);
|
||||
const traceFieldsMatch = agentRunTraceId === safeId && providerTraceId === safeId && (!traceSummaryCommandId || traceSummaryCommandId === commandId);
|
||||
if (traceFieldsMatch && (mapped.status === "running" || agentRunTerminalResultRefreshSatisfied(mapped, safeId))) return mapped;
|
||||
const env = options.env ?? process.env;
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, agentRun.managerUrl);
|
||||
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
||||
const command = await findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId: agentRun.runId, traceId: safeId });
|
||||
if (!command?.id) {
|
||||
throw Object.assign(new Error(`AgentRun command registry has no command for ${safeId} in run ${agentRun.runId}`), {
|
||||
code: "agentrun_trace_command_not_found",
|
||||
statusCode: 404,
|
||||
traceId: safeId,
|
||||
runId: agentRun.runId
|
||||
});
|
||||
}
|
||||
if (command.id === commandId) {
|
||||
return {
|
||||
...mapped,
|
||||
finalResponse: null,
|
||||
traceSummary: null,
|
||||
agentRun: {
|
||||
...agentRun,
|
||||
traceId: safeId,
|
||||
lastSeq: 0,
|
||||
providerTrace: null,
|
||||
commandState: command.state ?? agentRun.commandState ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
}
|
||||
};
|
||||
}
|
||||
return {
|
||||
...mapped,
|
||||
finalResponse: null,
|
||||
traceSummary: null,
|
||||
agentRun: {
|
||||
...agentRun,
|
||||
commandId: command.id,
|
||||
traceId: safeId,
|
||||
lastSeq: 0,
|
||||
providerTrace: null,
|
||||
commandState: command.state ?? agentRun.commandState ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function refreshAgentRunTrace({ traceId, result = null, options = {}, traceStore = defaultCodeAgentTraceStore }) {
|
||||
const initial = result ?? await loadPersistedAgentRunResult(traceId, options);
|
||||
const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial;
|
||||
if (!mapped?.agentRun?.runId) return traceStore.snapshot(traceId);
|
||||
const env = options.env ?? process.env;
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
||||
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
||||
const eventsResponse = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, traceSummary: mapped.traceSummary } });
|
||||
const events = eventsResponse.events;
|
||||
appendAgentRunEventsToTrace(traceStore, traceId, events, mapped.agentRun);
|
||||
const lastSeq = agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq);
|
||||
if (lastSeq !== Number(mapped.agentRun.lastSeq ?? 0)) {
|
||||
options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun: { ...mapped.agentRun, lastSeq, updatedAt: nowIso(options.now) } });
|
||||
}
|
||||
return traceStore.snapshot(traceId);
|
||||
}
|
||||
|
||||
export async function cancelAgentRunChatTurn({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore }) {
|
||||
let mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options);
|
||||
if (!mapped?.agentRun?.commandId) return null;
|
||||
@@ -1043,12 +700,6 @@ export async function cancelAgentRunChatTurn({ traceId, currentResult = null, op
|
||||
};
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
||||
try {
|
||||
const synced = await syncAgentRunChatResult({ traceId, currentResult: mapped, options: cancelOptions, traceStore, forceResultSync: true });
|
||||
mapped = synced?.result ?? synced ?? mapped;
|
||||
} catch {
|
||||
// Cancel must still expose the command cancel disposition; result refresh failures are reported by the normal trace/result paths.
|
||||
}
|
||||
const terminalStatus = agentRunCancelTerminalStatus(mapped);
|
||||
if (terminalStatus) return agentRunAlreadyTerminalCancelPayload({ traceId, payload: mapped, terminalStatus, options: cancelOptions, traceStore });
|
||||
const cancelPath = `/api/v1/commands/${encodeURIComponent(mapped.agentRun.commandId)}/cancel`;
|
||||
@@ -1155,7 +806,7 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par
|
||||
accepted: true,
|
||||
status: "running",
|
||||
shortConnection: true,
|
||||
controlSemantics: "steer-active-turn-and-poll-target-trace",
|
||||
controlSemantics: "steer-active-turn-and-project-sse",
|
||||
route: "/v1/agent/chat/steer",
|
||||
traceId,
|
||||
targetTraceId: traceId,
|
||||
@@ -1163,8 +814,8 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par
|
||||
conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null,
|
||||
sessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null,
|
||||
threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null,
|
||||
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
||||
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
||||
workbenchEventsUrl: `/v1/workbench/events?sessionId=${encodeURIComponent(safeSessionId(mapped.sessionId ?? params.sessionId) || "")}&traceId=${encodeURIComponent(traceId)}`,
|
||||
agentRun: {
|
||||
...agentRun,
|
||||
steerCommandId,
|
||||
@@ -1243,18 +894,6 @@ function agentRunSeedFromSession(session, traceId) {
|
||||
return topLevelAgentRun?.runId ? topLevelAgentRun : null;
|
||||
}
|
||||
|
||||
async function findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId, traceId }) {
|
||||
if (!runId || !safeTraceId(traceId)) return null;
|
||||
const commands = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands?afterSeq=0&limit=100`, { method: "GET", timeoutMs, env });
|
||||
return (Array.isArray(commands?.items) ? commands.items : []).find((item) => agentRunCommandMatchesTrace(item, traceId)) ?? null;
|
||||
}
|
||||
|
||||
function agentRunCommandMatchesTrace(command, traceId) {
|
||||
const payload = command?.payload && typeof command.payload === "object" ? command.payload : {};
|
||||
return command?.idempotencyKey === traceId || payload.traceId === traceId;
|
||||
}
|
||||
|
||||
|
||||
function withoutUserBillingReservation(value = {}) {
|
||||
if (!value || typeof value !== "object") return value;
|
||||
const { userBillingReservation, ...rest } = value;
|
||||
@@ -1353,126 +992,6 @@ function newSessionIdAfterEviction(baseSessionId, traceId) {
|
||||
return baseSessionId + "-reset-" + profile;
|
||||
}
|
||||
|
||||
async function retryAgentRunTurnAfterFreshSessionFailure({
|
||||
params = null,
|
||||
options = {},
|
||||
traceId,
|
||||
traceStore = defaultCodeAgentTraceStore,
|
||||
mapped = {},
|
||||
failedResult = {},
|
||||
failedMapping = {},
|
||||
env = process.env,
|
||||
managerUrl,
|
||||
fetchImpl,
|
||||
timeoutMs,
|
||||
backendProfile
|
||||
} = {}) {
|
||||
const failure = agentRunFreshSessionFailure(failedResult);
|
||||
if (!failure.requiresFreshSession) return null;
|
||||
if (mapped?.agentRun?.freshSessionRetryOf || failedMapping?.freshSessionRetryOf) return null;
|
||||
const promptText = messageAuthorityTextValue(params?.message ?? params?.prompt ?? params?.text ?? "");
|
||||
if (!promptText) {
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "failed",
|
||||
label: "agentrun:session-reset:skipped",
|
||||
errorCode: "agentrun_retry_prompt_missing",
|
||||
message: "AgentRun session storage was evicted, but hwlab-cloud-api cannot retry the current turn because the original prompt text is unavailable in this sync path.",
|
||||
runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null,
|
||||
commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null,
|
||||
terminal: true,
|
||||
valuesPrinted: false
|
||||
}, failedMapping || mapped?.agentRun || backendProfile));
|
||||
return null;
|
||||
}
|
||||
const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId);
|
||||
const sessionId = newSessionIdAfterEviction(baseSessionId, traceId);
|
||||
const retryParams = paramsWithAgentRunSessionThread(params, null);
|
||||
const startedAt = nowIso(options.now);
|
||||
const previous = {
|
||||
runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null,
|
||||
commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null,
|
||||
sessionId: failedMapping?.sessionId ?? mapped?.agentRun?.sessionId ?? baseSessionId,
|
||||
threadId: failedMapping?.threadId ?? mapped?.agentRun?.threadId ?? mapped?.threadId ?? null,
|
||||
failureKind: failure.code,
|
||||
failureMessage: failure.message,
|
||||
valuesPrinted: false
|
||||
};
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:session-reset:retry-current-turn",
|
||||
message: "AgentRun session storage/thread resume failed; hwlab-cloud-api is retrying the same user turn with a fresh AgentRun session and threadId=null.",
|
||||
runId: previous.runId,
|
||||
commandId: previous.commandId,
|
||||
sessionId,
|
||||
previousSessionId: previous.sessionId,
|
||||
previousThreadId: previous.threadId,
|
||||
errorCode: failure.code,
|
||||
backendProfile,
|
||||
waitingFor: "agentrun-run-create",
|
||||
valuesPrinted: false
|
||||
}, failedMapping || mapped?.agentRun || backendProfile));
|
||||
await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore });
|
||||
const toolCapabilities = await resolveToolCapabilities({ params: retryParams, options });
|
||||
const runInput = buildAgentRunCreateRunInput({ params: retryParams, env, traceId, backendProfile, sessionId, toolCapabilities });
|
||||
const run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create-after-session-reset" });
|
||||
const runId = requiredString(run?.id, "run.id");
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:run:created-after-session-reset",
|
||||
message: "AgentRun fresh-session retry run " + runId + " created for the current user turn.",
|
||||
runId,
|
||||
backendProfile,
|
||||
waitingFor: "agentrun-command-create",
|
||||
valuesPrinted: false
|
||||
}, backendProfile));
|
||||
const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile });
|
||||
const commandInput = buildAgentRunCommandInput({ params: retryParams, traceId, backendProfile, sessionId, dispatchInput });
|
||||
const command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create-after-session-reset" });
|
||||
const commandId = requiredString(command?.id, "command.id");
|
||||
const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent);
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:command:created-after-session-reset",
|
||||
message: "AgentRun fresh-session retry command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.",
|
||||
runId,
|
||||
commandId,
|
||||
backendProfile,
|
||||
dispatchIntentId: dispatchIntent.id,
|
||||
runnerJobId: dispatchIntent.runnerJobId,
|
||||
waitingFor: "agentrun-result",
|
||||
valuesPrinted: false
|
||||
}, backendProfile));
|
||||
const mapping = {
|
||||
...agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: retryParams }),
|
||||
freshSessionRetryAttempt: 1,
|
||||
freshSessionRetryOf: previous,
|
||||
status: `dispatch-intent-${dispatchIntent.state}`,
|
||||
valuesPrinted: false
|
||||
};
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:dispatch-intent:admitted-after-session-reset",
|
||||
message: "AgentRun owns the durable fresh-session runner dispatch; HWLAB has no process-local runner Job creation window.",
|
||||
runId: mapping.runId,
|
||||
commandId: mapping.commandId,
|
||||
backendProfile,
|
||||
dispatchIntentId: dispatchIntent.id,
|
||||
dispatchIntentState: dispatchIntent.state,
|
||||
runnerJobId: dispatchIntent.runnerJobId,
|
||||
waitingFor: "agentrun-result",
|
||||
valuesPrinted: false
|
||||
}, mapping));
|
||||
const base = initialAgentRunChatResult({ params: retryParams, options, traceId });
|
||||
const payload = decorateAgentRunRunningResult({ base: { ...base, freshSessionRetryOf: previous }, mapping, traceStore, traceId });
|
||||
return { payload, mapping };
|
||||
}
|
||||
|
||||
|
||||
function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId, toolCapabilities = null }) {
|
||||
const commitId = requireAgentRunSourceCommit(env);
|
||||
const runtimeAuthority = requireAgentRunRuntimeAuthority(env);
|
||||
@@ -1536,7 +1055,6 @@ function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, ses
|
||||
kind: "hwlab-cloud-api",
|
||||
traceId,
|
||||
...codeAgentOtelTraceFields(traceId, env),
|
||||
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
||||
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
||||
valuesPrinted: false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user