lyon1998
4524 lines
196 KiB
TypeScript
4524 lines
196 KiB
TypeScript
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// Responsibility: Code Agent HTTP turn admission, lifecycle compatibility wrappers, and Workbench projection writer/finalizer integration.
|
|
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
|
|
import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts";
|
|
import {
|
|
agentRunSessionEvidence,
|
|
cancelAgentRunChatTurn,
|
|
codeAgentAgentRunAdapterEnabled,
|
|
initialAgentRunChatResult,
|
|
loadPersistedAgentRunResult,
|
|
steerAgentRunChatTurn,
|
|
submitAgentRunChatTurn,
|
|
syncAgentRunChatResult
|
|
} from "./code-agent-agentrun-adapter.ts";
|
|
import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts";
|
|
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
|
import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts";
|
|
import { scheduleWorkbenchProjectionFinalizer } from "./workbench-projection-finalizer.ts";
|
|
import { writeWorkbenchProjectionSession, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts";
|
|
import { buildAgentRunProjectionEventsFetchPlan } from "./workbench-projection-cursor.ts";
|
|
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
|
import { publishHwlabAgentRunCommandShadow } from "./kafka-shadow-producer.ts";
|
|
import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
|
import {
|
|
firstHeaderValue,
|
|
getHeader,
|
|
parsePositiveInteger,
|
|
positiveInteger,
|
|
readBody,
|
|
safeConversationId,
|
|
safeOpaqueId,
|
|
safeSessionId,
|
|
safeTraceId,
|
|
sendJson,
|
|
truthyFlag
|
|
} from "./server-http-utils.ts";
|
|
|
|
const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120;
|
|
const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100;
|
|
const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100;
|
|
const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500;
|
|
const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench";
|
|
const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek";
|
|
const DEFAULT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS = 5000;
|
|
const DEFAULT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS = 60000;
|
|
const DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS = 30000;
|
|
const DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE = 100;
|
|
const DEFAULT_AGENTRUN_PROJECTION_RESUME_JITTER_MS = 15000;
|
|
const DEFAULT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS = 120000;
|
|
const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]);
|
|
const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" });
|
|
const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/u;
|
|
const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({
|
|
"deepseek": "DeepSeek",
|
|
"codex-api": "Codex API",
|
|
"minimax-m3": "MiniMax-M3 via AgentRun",
|
|
"runtime-default": "运行默认"
|
|
});
|
|
const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5";
|
|
const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses";
|
|
const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat";
|
|
const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3";
|
|
|
|
export async function handleCodeAgentChatHttp(request, response, options) {
|
|
const perf = options.backendPerformance;
|
|
const body = perf ? await perf.measure("body_read", () => readBody(request, options.bodyLimitBytes)) : await readBody(request, options.bodyLimitBytes);
|
|
let params = {};
|
|
|
|
try {
|
|
params = body ? JSON.parse(body) : {};
|
|
} catch (error) {
|
|
sendJson(response, 400, {
|
|
...createCodeAgentErrorPayload({
|
|
code: "parse_error",
|
|
message: "Invalid JSON body",
|
|
reason: error.message,
|
|
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
|
model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown",
|
|
layer: "api",
|
|
retryable: true
|
|
})
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (!params || typeof params !== "object" || Array.isArray(params)) {
|
|
sendJson(response, 400, {
|
|
...createCodeAgentErrorPayload({
|
|
code: "invalid_params",
|
|
message: "Code Agent chat body must be a JSON object",
|
|
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
|
model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown",
|
|
layer: "api",
|
|
retryable: true
|
|
})
|
|
});
|
|
return;
|
|
}
|
|
params = withMdtodoExecutionPrompt(params);
|
|
|
|
const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId) || `trc_${randomUUID()}`;
|
|
const lifecycle = codeAgentTurnLifecycleFields(traceId, params);
|
|
const chatParams = {
|
|
...params,
|
|
traceId,
|
|
...lifecycle,
|
|
ownerUserId: options.actor?.id ?? params.ownerUserId,
|
|
ownerRole: options.actor?.role ?? params.ownerRole
|
|
};
|
|
const manualSession = perf ? await perf.measure("manual_session", () => prepareManualCodeAgentSession({ params: chatParams, options, traceId, response })) : await prepareManualCodeAgentSession({ params: chatParams, options, traceId, response });
|
|
if (manualSession?.blocked) return;
|
|
const nativeSessionChatParams = stripSyntheticConversationContext(chatParams, traceId, options);
|
|
|
|
if (codeAgentChatShortConnectionRequested(request, params, options)) {
|
|
if (perf) perf.recordPhase({ phase: "short_connection_accept", durationMs: 0, outcome: "ok" });
|
|
let submitted = null;
|
|
try {
|
|
submitted = await submitCodeAgentChatTurn({
|
|
params: nativeSessionChatParams,
|
|
options,
|
|
traceId
|
|
});
|
|
} catch (error) {
|
|
if (isCodeAgentAdmissionUnavailableError(error)) {
|
|
sendJson(response, error.statusCode ?? 503, error.payload);
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
const traceUrl = `/v1/agent/traces/${encodeURIComponent(traceId)}`;
|
|
const turnUrl = `/v1/agent/turns/${encodeURIComponent(traceId)}`;
|
|
const statusCode = 202;
|
|
const admittedTiming = codeAgentAdmissionTimingFields(submitted?.createdAt ?? submitted?.updatedAt);
|
|
void emitCodeAgentOtelSpan("POST /v1/agent/chat", traceId, options.env ?? process.env, {
|
|
kind: 2,
|
|
attributes: { "http.method": "POST", "http.route": "/v1/agent/chat", "http.status_code": statusCode, sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, turnId: lifecycle.turnId, ...agentChatOtelAttributes(nativeSessionChatParams) }
|
|
});
|
|
sendJson(response, 202, {
|
|
accepted: true,
|
|
status: "running",
|
|
shortConnection: true,
|
|
controlSemantics: "submit-and-poll",
|
|
traceId,
|
|
...codeAgentOtelTraceFields(traceId, options.env ?? process.env),
|
|
...lifecycle,
|
|
sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null,
|
|
createdAt: submitted?.createdAt ?? admittedTiming.createdAt,
|
|
updatedAt: submitted?.updatedAt ?? admittedTiming.updatedAt,
|
|
timing: submitted?.timing ?? admittedTiming.timing,
|
|
startedAt: submitted?.startedAt ?? admittedTiming.startedAt,
|
|
lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt,
|
|
finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt,
|
|
durationMs: submitted?.durationMs ?? admittedTiming.durationMs,
|
|
traceUrl,
|
|
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
|
turnUrl,
|
|
streamUrl: `${traceUrl}/stream`,
|
|
cancelUrl: "/v1/agent/chat/cancel",
|
|
polling: {
|
|
resultIntervalMs: parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_RESULT_POLL_INTERVAL_MS, 1000),
|
|
traceIntervalMs: parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_TRACE_POLL_INTERVAL_MS, 1000)
|
|
},
|
|
runnerTrace: (options.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId)
|
|
});
|
|
return;
|
|
}
|
|
|
|
const admittedBase = codeAgentAdmittedFailureBasePayload({ params: nativeSessionChatParams, options, traceId });
|
|
try {
|
|
await recordCodeAgentTurnAdmission({ payload: admittedBase, params: nativeSessionChatParams, options, traceId });
|
|
} catch (error) {
|
|
if (isCodeAgentAdmissionUnavailableError(error)) {
|
|
sendJson(response, error.statusCode ?? 503, error.payload);
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
const billingPreflight = perf ? await perf.measure("billing_preflight", () => preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false })) : await preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false });
|
|
if (billingPreflight?.blocked) {
|
|
const failed = await settleAdmittedCodeAgentFailure({
|
|
base: admittedBase,
|
|
params: nativeSessionChatParams,
|
|
options,
|
|
traceId,
|
|
results: options.codeAgentChatResults,
|
|
failure: billingPreflight,
|
|
traceLabel: "billing:preflight:failed"
|
|
});
|
|
sendJson(response, billingPreflight.statusCode ?? 503, failed);
|
|
return;
|
|
}
|
|
const nativeSessionChatParamsWithBilling = { ...nativeSessionChatParams, userBillingReservation: billingPreflight?.reservation ?? null };
|
|
const payload = perf ? await perf.measure("code_agent_run", () => runCodeAgentChat(nativeSessionChatParamsWithBilling, options)) : await runCodeAgentChat(nativeSessionChatParamsWithBilling, options);
|
|
await (perf ? perf.measure("billing_finalize", () => finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })) : finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options }));
|
|
await (perf ? perf.measure("session_owner_record", () => recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })) : recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) }));
|
|
const responsePayload = annotateOwner(payload, nativeSessionChatParamsWithBilling);
|
|
|
|
sendJson(response, responsePayload.status === "failed" && responsePayload.error?.code === "invalid_params" ? 400 : 200, responsePayload);
|
|
}
|
|
|
|
export async function handleCodeAgentSessionsHttp(request, response, url, options) {
|
|
const match = url.pathname.match(/^\/v1\/agent\/sessions(?:\/([^/]+)(?:\/(select))?)?$/u);
|
|
const sessionId = safeSessionId(match?.[1] ? decodeURIComponent(match[1]) : "");
|
|
const action = match?.[2] ?? null;
|
|
if (!match) {
|
|
sendJson(response, 404, manualSessionErrorPayload({ code: "session_route_not_found", message: "Code Agent session route is not implemented." }));
|
|
return;
|
|
}
|
|
if (request.method === "GET" && !sessionId) {
|
|
await listManualCodeAgentSessions(request, response, url, options);
|
|
return;
|
|
}
|
|
if (request.method === "POST" && !sessionId) {
|
|
await createManualCodeAgentSession(request, response, options);
|
|
return;
|
|
}
|
|
if (request.method === "GET" && sessionId && !action) {
|
|
await getManualCodeAgentSession(response, options, sessionId);
|
|
return;
|
|
}
|
|
if (request.method === "DELETE" && sessionId && !action) {
|
|
await archiveManualCodeAgentSession(response, options, sessionId);
|
|
return;
|
|
}
|
|
if (request.method === "POST" && sessionId && action === "select") {
|
|
await selectManualCodeAgentSession(request, response, options, sessionId);
|
|
return;
|
|
}
|
|
sendJson(response, 405, manualSessionErrorPayload({ code: "session_method_not_allowed", message: "Unsupported Code Agent session method." }));
|
|
}
|
|
|
|
async function listManualCodeAgentSessions(request, response, url, options) {
|
|
const projectId = textValue(url.searchParams.get("projectId")) || DEFAULT_CODE_AGENT_PROJECT_ID;
|
|
const limit = parsePositiveInteger(url.searchParams.get("limit"), 20);
|
|
const sessions = await options.accessController?.store?.listAgentSessionsForUser?.({
|
|
ownerUserId: options.actor?.id,
|
|
actorRole: options.actor?.role,
|
|
ownerScoped: true,
|
|
projectId,
|
|
limit
|
|
}) ?? [];
|
|
sendJson(response, 200, {
|
|
ok: true,
|
|
status: "succeeded",
|
|
contractVersion: "code-agent-manual-session-v1",
|
|
projectId,
|
|
sessions: sessions.map(publicManualAgentSession),
|
|
count: sessions.length,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
});
|
|
}
|
|
|
|
async function createManualCodeAgentSession(request, response, options) {
|
|
const body = await readJsonObjectBody(request, options.bodyLimitBytes);
|
|
if (!body.ok) {
|
|
sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason }));
|
|
return;
|
|
}
|
|
const params = body.value;
|
|
const projectId = textValue(params.projectId) || DEFAULT_CODE_AGENT_PROJECT_ID;
|
|
const conversationId = safeConversationId(params.conversationId) || `cnv_${randomUUID()}`;
|
|
const sessionId = safeSessionId(params.sessionId) || `ses_${randomUUID()}`;
|
|
const providerProfile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile);
|
|
const now = new Date().toISOString();
|
|
const session = await options.accessController.recordAgentSessionOwner({
|
|
ownerUserId: options.actor?.id,
|
|
ownerRole: options.actor?.role,
|
|
sessionId,
|
|
projectId,
|
|
agentId: "hwlab-code-agent",
|
|
status: "idle",
|
|
conversationId,
|
|
threadId: null,
|
|
traceId: null,
|
|
session: {
|
|
source: "manual-session-create",
|
|
providerProfile,
|
|
sessionStatus: "idle",
|
|
createdAt: now,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
}
|
|
});
|
|
await writeWorkbenchSessionAdmissionFact({
|
|
runtimeStore: options.runtimeStore,
|
|
session,
|
|
ownerUserId: options.actor?.id,
|
|
ownerRole: options.actor?.role,
|
|
projectId,
|
|
conversationId,
|
|
threadId: null,
|
|
status: "idle",
|
|
now
|
|
});
|
|
sendJson(response, 201, {
|
|
ok: true,
|
|
status: "created",
|
|
contractVersion: "code-agent-manual-session-v1",
|
|
session: publicManualAgentSession(session),
|
|
nextCommands: manualSessionNextCommands(session),
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
});
|
|
}
|
|
|
|
async function getManualCodeAgentSession(response, options, sessionId) {
|
|
const session = await getVisibleManualAgentSession(options, sessionId);
|
|
if (!session) {
|
|
sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId }));
|
|
return;
|
|
}
|
|
sendJson(response, 200, {
|
|
ok: true,
|
|
status: "found",
|
|
contractVersion: "code-agent-manual-session-v1",
|
|
session: publicManualAgentSession(session),
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
});
|
|
}
|
|
|
|
async function archiveManualCodeAgentSession(response, options, sessionId) {
|
|
const session = await getVisibleManualAgentSession(options, sessionId);
|
|
if (!session) {
|
|
sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId }));
|
|
return;
|
|
}
|
|
const result = await options.accessController?.store?.archiveAgentConversation?.({
|
|
ownerUserId: options.actor?.id,
|
|
actorRole: options.actor?.role,
|
|
ownerScoped: true,
|
|
sessionId,
|
|
conversationId: session.conversationId,
|
|
projectId: session.projectId
|
|
}) ?? { count: 0 };
|
|
sendJson(response, 200, {
|
|
ok: true,
|
|
status: "archived",
|
|
contractVersion: "code-agent-manual-session-v1",
|
|
archivedCount: result.count ?? 0,
|
|
session: publicManualAgentSession({ ...session, status: "archived", endedAt: session.endedAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }),
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
});
|
|
}
|
|
|
|
async function selectManualCodeAgentSession(request, response, options, sessionId) {
|
|
const body = await readJsonObjectBody(request, options.bodyLimitBytes);
|
|
if (!body.ok) {
|
|
sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason, sessionId }));
|
|
return;
|
|
}
|
|
const session = await getVisibleManualAgentSession(options, sessionId);
|
|
if (!session) {
|
|
sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId }));
|
|
return;
|
|
}
|
|
if (!manualSessionUsable(session.status)) {
|
|
sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, sessionId, session }));
|
|
return;
|
|
}
|
|
sendJson(response, 200, {
|
|
ok: true,
|
|
status: "selected",
|
|
contractVersion: "code-agent-manual-session-v1",
|
|
session: publicManualAgentSession(session),
|
|
nextCommands: manualSessionNextCommands(session),
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
});
|
|
}
|
|
|
|
async function prepareManualCodeAgentSession({ params, options, traceId, response }) {
|
|
if (!options.actor?.id) {
|
|
sendJson(response, 401, manualSessionErrorPayload({
|
|
code: "auth_required",
|
|
message: "Authentication is required before using an explicit HWLAB Code Agent session.",
|
|
traceId,
|
|
retryable: true
|
|
}));
|
|
return { blocked: true };
|
|
}
|
|
const sessionId = safeSessionId(params.sessionId);
|
|
if (!sessionId) {
|
|
sendJson(response, 409, manualSessionErrorPayload({
|
|
code: "session_required",
|
|
message: "Code Agent session is explicit in HWLAB v0.2; create/select a session before sending a turn.",
|
|
traceId,
|
|
retryable: true,
|
|
nextCommands: ["hwlab-cli client agent session create --provider-profile PROFILE", "hwlab-cli client agent send --session-id <SESSION_ID> --message TEXT"]
|
|
}));
|
|
return { blocked: true };
|
|
}
|
|
const session = await getVisibleManualAgentSession(options, sessionId);
|
|
if (!session) {
|
|
sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", traceId, sessionId }));
|
|
return { blocked: true };
|
|
}
|
|
const profileRecovery = manualSessionProfileRecoveryAllowed(session, params);
|
|
if (!manualSessionUsable(session.status) && !profileRecovery) {
|
|
sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, traceId, sessionId, session }));
|
|
return { blocked: true };
|
|
}
|
|
const requestedProjectId = textValue(params.projectId);
|
|
const sessionProjectId = textValue(session.projectId);
|
|
if (requestedProjectId && sessionProjectId && requestedProjectId !== sessionProjectId) {
|
|
sendJson(response, 409, manualSessionErrorPayload({ code: "session_project_mismatch", message: `sessionId ${sessionId} belongs to ${sessionProjectId}; requested ${requestedProjectId}.`, traceId, sessionId, session }));
|
|
return { blocked: true };
|
|
}
|
|
const storedConversationId = safeConversationId(session.conversationId);
|
|
const requestedConversationId = safeConversationId(params.conversationId);
|
|
if (requestedConversationId && storedConversationId && requestedConversationId !== storedConversationId) {
|
|
sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_mismatch", message: `sessionId ${sessionId} is bound to ${storedConversationId}; requested ${requestedConversationId}.`, traceId, sessionId, session }));
|
|
return { blocked: true };
|
|
}
|
|
const conversationId = requestedConversationId || storedConversationId;
|
|
if (!conversationId) {
|
|
sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_required", message: "Code Agent session has no bound conversationId; create a new session before continuing.", traceId, sessionId, session }));
|
|
return { blocked: true };
|
|
}
|
|
params.conversationId = conversationId;
|
|
params.sessionId = sessionId;
|
|
params.projectId = requestedProjectId || sessionProjectId || DEFAULT_CODE_AGENT_PROJECT_ID;
|
|
params.threadId = profileRecovery ? undefined : safeOpaqueId(params.threadId) || safeOpaqueId(session.threadId) || undefined;
|
|
return { session };
|
|
}
|
|
|
|
function manualSessionProfileRecoveryAllowed(session, params = {}) {
|
|
const status = String(session?.status ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
|
if (status !== "thread-resume-failed") return false;
|
|
const requestedProfile = manualSessionProviderProfileToken(params.providerProfile ?? params.codeAgentProviderProfile);
|
|
const sessionProfile = manualSessionProviderProfileToken(session?.session?.providerProfile);
|
|
return Boolean(requestedProfile && sessionProfile && requestedProfile !== sessionProfile);
|
|
}
|
|
|
|
function manualSessionProviderProfileToken(value) {
|
|
const profile = String(value ?? "").trim().toLowerCase();
|
|
if (!profile || profile === "runtime-default") return "";
|
|
if (profile === "codex-api") return "codex";
|
|
return profile;
|
|
}
|
|
|
|
async function getVisibleManualAgentSession(options, sessionId) {
|
|
if (!safeSessionId(sessionId) || !options.actor?.id || !options.accessController?.getAgentSession) return null;
|
|
const session = await options.accessController.getAgentSession(sessionId);
|
|
if (!session) return null;
|
|
if (session.ownerUserId === options.actor.id) return session;
|
|
return null;
|
|
}
|
|
|
|
function publicManualAgentSession(session) {
|
|
if (!session || typeof session !== "object") return null;
|
|
return {
|
|
sessionId: session.id,
|
|
agentId: session.agentId ?? "hwlab-code-agent",
|
|
status: session.status ?? null,
|
|
usable: manualSessionUsable(session.status),
|
|
threadId: safeOpaqueId(session.threadId) || null,
|
|
lastTraceId: safeTraceId(session.lastTraceId) || null,
|
|
providerProfile: textValue(session.session?.providerProfile) || null,
|
|
createdAt: session.startedAt ?? null,
|
|
updatedAt: session.updatedAt ?? null,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
function manualSessionUsable(status) {
|
|
const value = String(status ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
|
return !["blocked", "expired", "stale", "thread-resume-failed"].includes(value);
|
|
}
|
|
|
|
function manualSessionErrorPayload({ code, message, reason = null, traceId = null, sessionId = null, session = null, retryable = true, nextCommands = undefined } = {}) {
|
|
return {
|
|
ok: false,
|
|
accepted: false,
|
|
status: "blocked",
|
|
traceId: safeTraceId(traceId) || null,
|
|
sessionId: safeSessionId(sessionId) || safeSessionId(session?.id) || null,
|
|
session: session ? publicManualAgentSession(session) : null,
|
|
error: {
|
|
code,
|
|
layer: "api",
|
|
category: "session_blocked",
|
|
retryable,
|
|
message,
|
|
reason,
|
|
userMessage: message,
|
|
route: "/v1/agent/chat"
|
|
},
|
|
blocker: {
|
|
code,
|
|
layer: "api",
|
|
retryable,
|
|
summary: message
|
|
},
|
|
nextCommands,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
function manualSessionNextCommands(session) {
|
|
const sessionId = session?.id ?? "<SESSION_ID>";
|
|
return [
|
|
`hwlab-cli client agent send --session-id ${sessionId} --message TEXT`,
|
|
`hwlab-cli client agent session status ${sessionId}`
|
|
];
|
|
}
|
|
|
|
async function readJsonObjectBody(request, bodyLimitBytes) {
|
|
const body = await readBody(request, bodyLimitBytes);
|
|
try {
|
|
const value = body ? JSON.parse(body) : {};
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, code: "invalid_params", message: "body must be a JSON object" };
|
|
return { ok: true, value };
|
|
} catch (error) {
|
|
return { ok: false, code: "parse_error", message: "Invalid JSON body", reason: error.message };
|
|
}
|
|
}
|
|
|
|
async function runCodeAgentChat(params, options) {
|
|
return handleCodeAgentChat(params, await codeAgentChatExecutionOptions(options, params));
|
|
}
|
|
|
|
function stripSyntheticConversationContext(params = {}, traceId, options = {}) {
|
|
if (Array.isArray(params.conversationContext?.messages) || Array.isArray(params.messages)) {
|
|
(options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, {
|
|
type: "synthetic-context",
|
|
status: "ignored",
|
|
label: "synthetic-context:ignored",
|
|
message: "Synthetic conversation context was stripped; Code Agent continuation uses only Codex stdio native thread/resume and turn/start.",
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
const { conversationContext, conversationContextTruncated, contextSource, messages, ...nativeParams } = params;
|
|
return nativeParams;
|
|
}
|
|
|
|
async function codeAgentChatExecutionOptions(options = {}, params = {}) {
|
|
const env = codeAgentProviderProfileEnv(options.env ?? process.env, params);
|
|
Object.assign(env, await codeAgentAuthEnv(options, env, params));
|
|
return {
|
|
callProvider: options.callCodeAgentProvider,
|
|
timeoutMs: options.codeAgentTimeoutMs,
|
|
hardTimeoutMs: options.codeAgentHardTimeoutMs,
|
|
env,
|
|
workspace: options.workspace,
|
|
skillsDirs: options.skillsDirs,
|
|
skillsDirsExact: options.skillsDirsExact,
|
|
skillsDiscoveryDelayMs: options.skillsDiscoveryDelayMs,
|
|
sessionRegistry: options.sessionRegistry,
|
|
m3IoSkillRequestJson: options.m3IoSkillRequestJson ?? createCodeAgentM3HwlabApiRequestJson(options),
|
|
codexStdioManager: options.codexStdioManager,
|
|
traceStore: options.traceStore,
|
|
gatewayRegistry: options.gatewayRegistry
|
|
};
|
|
}
|
|
|
|
async function codeAgentAuthEnv(options = {}, env = process.env, params = {}) {
|
|
const ownerUserId = textValue(params.ownerUserId ?? options.actor?.id ?? "");
|
|
const store = options.accessController?.store;
|
|
const key = ownerUserId && store?.findActiveDefaultApiKeyForUser
|
|
? await store.findActiveDefaultApiKeyForUser(ownerUserId)
|
|
: null;
|
|
const envRuntimeLane = runtimeLaneForEnv(env);
|
|
const runtimeNamespace = runtimeNamespaceForEnv(env);
|
|
const namespaceRuntimeLane = runtimeLaneForNamespace(runtimeNamespace);
|
|
const runtimeLane = resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane });
|
|
const inClusterApiUrl = internalCodeAgentApiUrl(env, runtimeNamespace);
|
|
const inClusterWebUrl = internalCodeAgentWebUrl(env, runtimeNamespace);
|
|
const apiUrl = firstNonEmptyValue(
|
|
codeAgentAgentRunAdapterEnabled(env) ? inClusterApiUrl : null,
|
|
env.HWLAB_RUNTIME_API_URL,
|
|
sameRuntimeEndpoint(env.HWLAB_CLOUD_API_URL, runtimeNamespace, runtimeLane),
|
|
env.HWLAB_PUBLIC_ENDPOINT,
|
|
localCloudApiUrl(env)
|
|
);
|
|
return {
|
|
HWLAB_CLOUD_API_URL: apiUrl,
|
|
HWLAB_RUNTIME_API_URL: apiUrl,
|
|
HWLAB_RUNTIME_WEB_URL: firstNonEmptyValue(codeAgentAgentRunAdapterEnabled(env) ? inClusterWebUrl : null, env.HWLAB_RUNTIME_WEB_URL, apiUrl),
|
|
HWLAB_RUNTIME_NAMESPACE: runtimeNamespace,
|
|
HWLAB_RUNTIME_LANE: runtimeLane,
|
|
HWLAB_RUNTIME_ENDPOINT_SOURCE: codeAgentAgentRunAdapterEnabled(env) ? "runtime-namespace" : "runtime-env",
|
|
HWLAB_RUNTIME_ENDPOINT_LOCKED: "1",
|
|
HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1",
|
|
...(options.actorApiKeySecret ? { HWLAB_API_KEY: options.actorApiKeySecret } : key?.displaySecret ? { HWLAB_API_KEY: key.displaySecret } : {})
|
|
};
|
|
}
|
|
|
|
function internalCodeAgentApiUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) {
|
|
const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_SERVICE_NAME, env.HWLAB_CLOUD_API_SERVICE_NAME, "hwlab-cloud-api");
|
|
const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_PORT, env.HWLAB_CLOUD_API_PORT), 6667);
|
|
return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`;
|
|
}
|
|
|
|
function internalCodeAgentWebUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) {
|
|
const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_SERVICE_NAME, env.HWLAB_CLOUD_WEB_SERVICE_NAME, "hwlab-cloud-web");
|
|
const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_PORT, env.HWLAB_CLOUD_WEB_PORT), 8080);
|
|
return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`;
|
|
}
|
|
|
|
function runtimeNamespaceForEnv(env = process.env) {
|
|
return firstNonEmptyValue(
|
|
env.HWLAB_RUNTIME_NAMESPACE,
|
|
env.POD_NAMESPACE,
|
|
env.HWLAB_NAMESPACE,
|
|
defaultRuntimeNamespace(env)
|
|
);
|
|
}
|
|
|
|
function runtimeLaneForEnv(env = process.env) {
|
|
const raw = firstNonEmptyValue(env.HWLAB_RUNTIME_LANE, env.HWLAB_GITOPS_LANE, env.HWLAB_GITOPS_PROFILE, env.HWLAB_ENVIRONMENT, env.HWLAB_BOOT_REF);
|
|
if (!raw) return null;
|
|
const lane = normalizeRuntimeLane(raw);
|
|
if (!lane) {
|
|
throw runtimeAuthorityError("unsupported_runtime_lane", `unsupported HWLAB runtime lane: ${String(raw).trim()}`);
|
|
}
|
|
return lane;
|
|
}
|
|
|
|
function runtimeLaneForNamespace(namespace) {
|
|
const value = String(namespace ?? "").trim().toLowerCase();
|
|
if (value === "hwlab-prod") return "prod";
|
|
if (value === "hwlab-dev") return "dev";
|
|
const versionMatch = value.match(/^hwlab-v(\d+)$/u);
|
|
if (versionMatch) return versionRuntimeLane(versionMatch[1]);
|
|
return null;
|
|
}
|
|
|
|
function sameRuntimeEndpoint(value, namespace, lane) {
|
|
const text = firstNonEmptyValue(value);
|
|
if (!text) return null;
|
|
const endpointLane = runtimeLaneForEndpoint(text);
|
|
const endpointNamespace = runtimeNamespaceForEndpoint(text);
|
|
if (namespace && endpointNamespace && endpointNamespace !== namespace) return null;
|
|
if (lane && endpointLane && endpointLane !== lane) return null;
|
|
return text;
|
|
}
|
|
|
|
function runtimeNamespaceForEndpoint(value) {
|
|
const match = String(value ?? "").match(/\.((?:hwlab-[a-z0-9-]+))\.svc\.cluster\.local(?::|\/|$)/iu);
|
|
return match?.[1] ?? null;
|
|
}
|
|
|
|
function runtimeLaneForEndpoint(value) {
|
|
const text = String(value ?? "").toLowerCase();
|
|
const namespaceLane = runtimeLaneForNamespace(runtimeNamespaceForEndpoint(text));
|
|
if (namespaceLane) return namespaceLane;
|
|
if (/hwlab-prod|:18666(?:\/|$)|:18667(?:\/|$)/u.test(text)) return "prod";
|
|
if (/hwlab-dev|:17666(?:\/|$)|:17667(?:\/|$)|:16666(?:\/|$)|:16667(?:\/|$)/u.test(text)) return "dev";
|
|
return null;
|
|
}
|
|
|
|
function normalizeRuntimeLane(value) {
|
|
const profile = String(value ?? "").trim().toLowerCase();
|
|
if (!profile) return null;
|
|
if (profile === "prod" || profile === "production") return "prod";
|
|
if (profile === "dev" || profile === "development" || profile === "local") return "dev";
|
|
const dotted = profile.match(/^v?0+\.(\d+)$/u);
|
|
if (dotted) return versionRuntimeLane(dotted[1]);
|
|
const compact = profile.match(/^v?0*(\d+)$/u);
|
|
if (compact) return versionRuntimeLane(compact[1]);
|
|
return null;
|
|
}
|
|
|
|
function versionRuntimeLane(value) {
|
|
const version = Number.parseInt(value, 10);
|
|
if (!Number.isFinite(version) || version <= 0) return null;
|
|
return `v${String(version).padStart(2, "0")}`;
|
|
}
|
|
|
|
function resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }) {
|
|
if (!namespaceRuntimeLane) {
|
|
throw runtimeAuthorityError("unsupported_runtime_namespace", `unsupported HWLAB runtime namespace: ${runtimeNamespace}`);
|
|
}
|
|
if (envRuntimeLane && envRuntimeLane !== namespaceRuntimeLane) {
|
|
throw runtimeAuthorityError(
|
|
"runtime_authority_mismatch",
|
|
`HWLAB runtime lane ${envRuntimeLane} does not match namespace ${runtimeNamespace} (${namespaceRuntimeLane})`
|
|
);
|
|
}
|
|
return namespaceRuntimeLane;
|
|
}
|
|
|
|
function runtimeAuthorityError(code, message) {
|
|
return Object.assign(new Error(message), {
|
|
code,
|
|
statusCode: 500,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
|
|
function defaultRuntimeNamespace(env = process.env) {
|
|
const profile = runtimeLaneForEnv(env) ?? "dev";
|
|
if (profile === "prod") return "hwlab-prod";
|
|
if (profile === "dev") return "hwlab-dev";
|
|
return `hwlab-${profile}`;
|
|
}
|
|
|
|
function localCloudApiUrl(env = process.env) {
|
|
const port = parsePositiveInteger(env.HWLAB_CLOUD_API_PORT, 6667);
|
|
return `http://127.0.0.1:${port}`;
|
|
}
|
|
|
|
function codeAgentProviderProfileEnv(env = process.env, params = {}) {
|
|
const profile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile ?? env.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE);
|
|
if (profile === "runtime-default") return env;
|
|
const overlay = { ...env };
|
|
overlay.HWLAB_CODE_AGENT_SELECTED_PROVIDER_PROFILE = profile;
|
|
overlay.HWLAB_CODE_AGENT_PROVIDER_PROFILE_LABEL = CODE_AGENT_PROVIDER_PROFILE_LABELS[profile] ?? profile;
|
|
overlay.HWLAB_CODE_AGENT_PROVIDER = "codex-stdio";
|
|
if (profile === "codex-api") {
|
|
overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_MODEL, env.HWLAB_CODE_AGENT_LEGACY_CODEX_MODEL, DEFAULT_CODE_AGENT_CODEX_API_MODEL);
|
|
overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_BASE_URL, env.HWLAB_CODE_AGENT_LEGACY_OPENAI_BASE_URL, DEFAULT_CODE_AGENT_CODEX_API_BASE_URL);
|
|
} else if (profile === "minimax-m3") {
|
|
overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_MINIMAX_M3_MODEL, DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL);
|
|
} else {
|
|
const dynamicProfile = profile !== "deepseek";
|
|
overlay.HWLAB_CODE_AGENT_MODEL = dynamicProfile
|
|
? firstNonEmptyValue(env[`HWLAB_CODE_AGENT_${providerProfileEnvKey(profile)}_MODEL`], profile)
|
|
: firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_MODEL, DEFAULT_CODE_AGENT_DEEPSEEK_MODEL);
|
|
if (!dynamicProfile || !codeAgentAgentRunAdapterEnabled(env)) {
|
|
overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL, defaultDeepSeekBaseUrlForEnv(env));
|
|
}
|
|
}
|
|
overlay.NO_PROXY = mergeNoProxyEntries(env.NO_PROXY, codeAgentClusterNoProxyEntries(env));
|
|
overlay.no_proxy = mergeNoProxyEntries(env.no_proxy ?? env.NO_PROXY, codeAgentClusterNoProxyEntries(env));
|
|
return overlay;
|
|
}
|
|
|
|
function normalizeCodeAgentProviderProfile(value) {
|
|
const text = String(value ?? DEFAULT_CODE_AGENT_PROVIDER_PROFILE).trim().toLowerCase();
|
|
if (!text) return DEFAULT_CODE_AGENT_PROVIDER_PROFILE;
|
|
if (text === "runtime-default") return text;
|
|
const normalized = CODE_AGENT_PROVIDER_PROFILE_ALIASES[text] ?? text;
|
|
return CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN.test(normalized) ? normalized : DEFAULT_CODE_AGENT_PROVIDER_PROFILE;
|
|
}
|
|
|
|
function providerProfileEnvKey(profile) {
|
|
return String(profile ?? "")
|
|
.trim()
|
|
.toUpperCase()
|
|
.replace(/[^A-Z0-9]+/gu, "_")
|
|
.replace(/^_+|_+$/gu, "") || "PROFILE";
|
|
}
|
|
|
|
function defaultDeepSeekBaseUrlForEnv(env = process.env) {
|
|
const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev";
|
|
return `http://hwlab-deepseek-proxy.${namespace}.svc.cluster.local:4000/v1/responses`;
|
|
}
|
|
|
|
function codeAgentClusterNoProxyEntries(env = process.env) {
|
|
const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev";
|
|
return [
|
|
`${namespace}.svc.cluster.local`,
|
|
".svc",
|
|
".cluster.local",
|
|
"127.0.0.1",
|
|
"localhost",
|
|
"::1",
|
|
"10.0.0.0/8",
|
|
"10.42.0.0/16",
|
|
"10.43.0.0/16",
|
|
"hyueapi.com",
|
|
".hyueapi.com",
|
|
"hyui.com",
|
|
".hyui.com",
|
|
"api.minimaxi.com",
|
|
".minimaxi.com"
|
|
];
|
|
}
|
|
|
|
function mergeNoProxyEntries(current, additions = []) {
|
|
return uniqueStrings([...String(current ?? "").split(","), ...additions]).join(",");
|
|
}
|
|
|
|
function firstNonEmptyValue(...values) {
|
|
for (const value of values) {
|
|
const text = String(value ?? "").trim();
|
|
if (text) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function safeOtelConfigText(value) {
|
|
const text = firstNonEmptyValue(value);
|
|
return text === null ? null : text.slice(0, 120);
|
|
}
|
|
|
|
function safeOtelUrlHost(value) {
|
|
const text = firstNonEmptyValue(value);
|
|
if (text === null) return null;
|
|
try {
|
|
return new URL(text).host.slice(0, 120);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function withMdtodoExecutionPrompt(params = {}) {
|
|
const execution = mdtodoExecutionContextFromParams(params);
|
|
if (!execution?.hwpodId || !execution?.hwpodWorkspaceArgs) return params;
|
|
const message = firstNonEmptyValue(params.message, params.prompt, params.text);
|
|
if (!message || /hwpodWorkspaceArgs\s*:/u.test(message)) return params;
|
|
const taskRef = firstNonEmptyValue(params.taskRef, execution.taskRef) ?? "-";
|
|
const mdtodoRootRef = execution.mdtodoRootRef ?? "docs/MDTODO";
|
|
const prefix = [
|
|
"# MDTODO HWPOD 执行上下文",
|
|
`TaskRef: ${taskRef}`,
|
|
`sourceId: ${execution.sourceId ?? "-"}`,
|
|
`hwpodId: ${execution.hwpodId}`,
|
|
`nodeId: ${execution.nodeId ?? "-"}`,
|
|
`mdtodoRootRef: ${mdtodoRootRef}`,
|
|
`hwpodWorkspaceArgs: ${execution.hwpodWorkspaceArgs}`,
|
|
"所有 hwpod/hwpod-ctl 命令都必须携带上述 hwpodWorkspaceArgs。读取 MDTODO、报告和工程文件必须通过 HWPOD workspace/node-ops 入口，不得猜测或访问普通容器本地路径，也不得创建本地 .hwlab/hwpod-spec.yaml fallback。"
|
|
].join("\n");
|
|
const nextMessage = `${prefix}\n\n${message}`;
|
|
return {
|
|
...params,
|
|
message: nextMessage,
|
|
prompt: nextMessage,
|
|
launchContext: {
|
|
...(params.launchContext && typeof params.launchContext === "object" ? params.launchContext : {}),
|
|
hwpodId: execution.hwpodId,
|
|
nodeId: execution.nodeId,
|
|
mdtodoRootRef: execution.mdtodoRootRef,
|
|
hwpodWorkspaceArgs: execution.hwpodWorkspaceArgs,
|
|
contextFingerprint: execution.contextFingerprint,
|
|
executionContext: execution,
|
|
valuesRedacted: true
|
|
}
|
|
};
|
|
}
|
|
|
|
function mdtodoExecutionContextFromParams(params = {}) {
|
|
const launchContext = params.launchContext && typeof params.launchContext === "object" && !Array.isArray(params.launchContext) ? params.launchContext : null;
|
|
const execution = launchContext?.executionContext && typeof launchContext.executionContext === "object" && !Array.isArray(launchContext.executionContext) ? launchContext.executionContext : launchContext;
|
|
if (!execution || typeof execution !== "object") return null;
|
|
return {
|
|
sourceId: safeOtelConfigText(execution.sourceId ?? launchContext?.sourceId),
|
|
sourceKind: safeOtelConfigText(execution.sourceKind ?? launchContext?.sourceKind),
|
|
projectId: safeOtelConfigText(execution.projectId ?? launchContext?.projectId ?? params.projectId),
|
|
taskRef: safeOtelText(execution.taskRef ?? launchContext?.taskRef ?? params.taskRef),
|
|
fileRef: safeOtelConfigText(execution.fileRef ?? launchContext?.fileRef),
|
|
relativePath: safeOtelText(execution.relativePath ?? launchContext?.relativePath),
|
|
taskId: safeOtelConfigText(execution.taskId ?? launchContext?.taskId),
|
|
hwpodId: safeOtelText(execution.hwpodId ?? launchContext?.hwpodId),
|
|
nodeId: safeOtelText(execution.nodeId ?? launchContext?.nodeId),
|
|
mdtodoRootRef: safeOtelText(execution.mdtodoRootRef ?? launchContext?.mdtodoRootRef),
|
|
workspaceRootHash: safeOtelText(execution.workspaceRootHash ?? launchContext?.workspaceRootHash),
|
|
hwpodWorkspaceArgs: safeOtelText(execution.hwpodWorkspaceArgs ?? launchContext?.hwpodWorkspaceArgs, 500),
|
|
contextFingerprint: safeOtelText(execution.contextFingerprint ?? launchContext?.contextFingerprint),
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function agentChatOtelAttributes(params = {}) {
|
|
const execution = mdtodoExecutionContextFromParams(params);
|
|
return {
|
|
"agent.chat.session_id": safeSessionId(params.sessionId) || null,
|
|
"agent.chat.task_ref": safeOtelText(params.taskRef ?? execution?.taskRef),
|
|
"agent.chat.source_id": safeOtelConfigText(execution?.sourceId ?? params.sourceId),
|
|
"agent.chat.hwpod_id": safeOtelText(execution?.hwpodId),
|
|
"agent.chat.workspace_root_hash": safeOtelText(execution?.workspaceRootHash),
|
|
"agent.chat.context_fingerprint": safeOtelText(execution?.contextFingerprint)
|
|
};
|
|
}
|
|
|
|
function safeOtelText(value, max = 360) {
|
|
const text = firstNonEmptyValue(value);
|
|
return text === null ? null : text.slice(0, max);
|
|
}
|
|
|
|
async function submitCodeAgentChatTurn({ params, options, traceId }) {
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
const results = options.codeAgentChatResults ?? createCodeAgentChatResultStore();
|
|
const timestamp = new Date().toISOString();
|
|
const runtimeEnv = options.env ?? process.env;
|
|
const adapterEnabled = codeAgentAgentRunAdapterEnabled(runtimeEnv);
|
|
const acceptedPayload = {
|
|
accepted: true,
|
|
status: "running",
|
|
traceId,
|
|
...codeAgentOtelTraceFields(traceId, process.env),
|
|
...codeAgentTurnLifecycleFields(traceId, params),
|
|
conversationId: safeConversationId(params.conversationId) || null,
|
|
sessionId: safeSessionId(params.sessionId) || null,
|
|
threadId: safeOpaqueId(params.threadId) || null,
|
|
projectId: params.projectId ?? null,
|
|
...codeAgentAdmissionTimingFields(timestamp)
|
|
};
|
|
await recordCodeAgentTurnAdmission({ payload: acceptedPayload, params, options, traceId });
|
|
results.set(traceId, annotateOwner(acceptedPayload, params));
|
|
void emitCodeAgentOtelSpan("provider_decision", traceId, runtimeEnv, {
|
|
attributes: {
|
|
...agentChatOtelAttributes(params),
|
|
adapterEnabled,
|
|
adapter: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_ADAPTER ?? runtimeEnv.HWLAB_CODE_AGENT_PROVIDER),
|
|
provider: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_PROVIDER),
|
|
defaultProviderProfile: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE),
|
|
agentRunManagerHost: safeOtelUrlHost(runtimeEnv.AGENTRUN_MGR_URL ?? runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_MGR_URL),
|
|
agentRunRunnerNamespace: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE ?? runtimeEnv.AGENTRUN_RUNTIME_NAMESPACE),
|
|
agentRunSourceCommitPresent: Boolean(firstNonEmptyValue(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT))
|
|
}
|
|
});
|
|
if (adapterEnabled) {
|
|
const run = async () => {
|
|
let initial = null;
|
|
let executionOptions = options;
|
|
try {
|
|
initial = initialAgentRunChatResult({ params, options, traceId });
|
|
results.set(traceId, annotateOwner(initial, params));
|
|
const billingStartedAt = Date.now();
|
|
const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false });
|
|
void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, {
|
|
startTimeMs: billingStartedAt,
|
|
attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) },
|
|
status: billingPreflight?.blocked ? "error" : "ok",
|
|
error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null
|
|
});
|
|
if (billingPreflight?.blocked) {
|
|
await settleAdmittedCodeAgentFailure({
|
|
base: initial,
|
|
params,
|
|
options,
|
|
traceId,
|
|
results,
|
|
failure: billingPreflight,
|
|
traceLabel: "billing:preflight:failed"
|
|
});
|
|
return;
|
|
}
|
|
const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null };
|
|
initial = withCodeAgentBillingReservation(initial, dispatchParams);
|
|
results.set(traceId, annotateOwner(initial, dispatchParams));
|
|
executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) };
|
|
const dispatchStartedAt = Date.now();
|
|
const payload = await submitAgentRunChatTurn({ params: dispatchParams, options: executionOptions, traceId, traceStore, results });
|
|
void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, {
|
|
startTimeMs: dispatchStartedAt,
|
|
attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: payload?.agentRun?.runId ?? null, commandId: payload?.agentRun?.commandId ?? null, providerProfile: dispatchParams.providerProfile ?? null }
|
|
});
|
|
if (isCodeAgentResultCanceled(results.get(traceId))) return;
|
|
const owned = annotateOwner(withCodeAgentBillingReservation(payload, dispatchParams), dispatchParams);
|
|
publishHwlabAgentRunCommandShadow({
|
|
params: dispatchParams,
|
|
payload: owned,
|
|
traceId,
|
|
lifecycle: codeAgentTurnLifecycleFields(traceId, dispatchParams),
|
|
env: executionOptions.env ?? process.env
|
|
});
|
|
await recordCodeAgentSessionOwner({ payload: owned, params: dispatchParams, options: executionOptions, status: "running" });
|
|
results.set(traceId, owned);
|
|
scheduleAgentRunProjectionSync({ traceId, params: dispatchParams, options: executionOptions, traceStore, currentResult: owned });
|
|
void emitCodeAgentOtelSpan("projection_write", traceId, executionOptions.env ?? process.env, {
|
|
attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: owned?.agentRun?.runId ?? null, commandId: owned?.agentRun?.commandId ?? null, projection: "scheduled" }
|
|
});
|
|
} catch (error) {
|
|
if (isCodeAgentResultCanceled(results.get(traceId))) return;
|
|
const dispatchErrorAttributes = codeAgentDispatchErrorOtelAttributes(error);
|
|
void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, {
|
|
status: "error",
|
|
error,
|
|
attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, ...dispatchErrorAttributes }
|
|
});
|
|
await settleAdmittedCodeAgentFailure({
|
|
base: initial ?? codeAgentAdmittedFailureBasePayload({ params, options, traceId }),
|
|
params,
|
|
options: executionOptions,
|
|
traceId,
|
|
results,
|
|
failure: codeAgentDispatchFailure(error),
|
|
traceLabel: "agentrun:dispatch:failed"
|
|
});
|
|
}
|
|
};
|
|
setImmediate(() => { run(); });
|
|
return annotateOwner(acceptedPayload, params);
|
|
}
|
|
traceStore.ensure(traceId, {
|
|
runnerKind: "codex-app-server-stdio-runner",
|
|
workspace: options.workspace ?? options.env?.HWLAB_CODE_AGENT_CODEX_WORKSPACE ?? options.env?.HWLAB_CODE_AGENT_WORKSPACE ?? null,
|
|
sandbox: options.env?.HWLAB_CODE_AGENT_CODEX_SANDBOX ?? null,
|
|
sessionMode: "codex-app-server-stdio-long-lived",
|
|
implementationType: "repo-owned-codex-app-server-stdio-session"
|
|
});
|
|
traceStore.append(traceId, {
|
|
type: "request",
|
|
status: "accepted",
|
|
label: "request:accepted-short-connection",
|
|
message: "Code Agent request accepted; client must poll trace/result with short HTTP requests.",
|
|
waitingFor: "codex-stdio"
|
|
});
|
|
|
|
const run = async () => {
|
|
try {
|
|
const billingStartedAt = Date.now();
|
|
const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false });
|
|
void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, {
|
|
startTimeMs: billingStartedAt,
|
|
attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) },
|
|
status: billingPreflight?.blocked ? "error" : "ok",
|
|
error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null
|
|
});
|
|
if (billingPreflight?.blocked) {
|
|
await settleAdmittedCodeAgentFailure({
|
|
base: codeAgentAdmittedFailureBasePayload({ params, options, traceId }),
|
|
params,
|
|
options,
|
|
traceId,
|
|
results,
|
|
failure: billingPreflight,
|
|
traceLabel: "billing:preflight:failed"
|
|
});
|
|
return;
|
|
}
|
|
const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null };
|
|
const payload = await runCodeAgentChat(dispatchParams, options);
|
|
if (isCodeAgentResultCanceled(results.get(traceId))) return;
|
|
await finalizeCodeAgentBillingUsage({ payload, params: dispatchParams, options });
|
|
await recordCodeAgentSessionOwner({ payload, params: dispatchParams, options, status: codeAgentOwnerStatusForResult(payload) });
|
|
results.set(traceId, annotateOwner(payload, dispatchParams));
|
|
traceStore.append(traceId, {
|
|
type: "result",
|
|
status: payload.status === "completed" ? "completed" : "failed",
|
|
label: `result:${payload.status}`,
|
|
message: payload.status === "completed"
|
|
? "Code Agent result is ready for short-connection polling."
|
|
: payload.error?.message ?? "Code Agent completed without a successful reply.",
|
|
sessionId: payload.sessionId ?? payload.session?.sessionId,
|
|
sessionStatus: payload.session?.status,
|
|
terminal: true
|
|
});
|
|
} catch (error) {
|
|
if (isCodeAgentResultCanceled(results.get(traceId))) return;
|
|
const payload = {
|
|
status: "failed",
|
|
traceId,
|
|
conversationId: safeConversationId(params.conversationId) || null,
|
|
sessionId: safeSessionId(params.sessionId) || null,
|
|
error: {
|
|
code: "code_agent_background_failed",
|
|
layer: "api",
|
|
retryable: true,
|
|
message: error?.message ?? "Code Agent background turn failed",
|
|
userMessage: "Code Agent 后台任务失败；trace/result 轮询已保留错误。"
|
|
},
|
|
updatedAt: new Date().toISOString()
|
|
};
|
|
await finalizeCodeAgentBillingUsage({ payload, params, options });
|
|
results.set(traceId, payload);
|
|
traceStore.append(traceId, {
|
|
type: "result",
|
|
status: "failed",
|
|
label: "result:failed",
|
|
errorCode: payload.error.code,
|
|
message: payload.error.message,
|
|
terminal: true
|
|
});
|
|
}
|
|
};
|
|
setImmediate(() => {
|
|
run();
|
|
});
|
|
return annotateOwner(acceptedPayload, params);
|
|
}
|
|
|
|
async function recordCodeAgentTurnAdmission({ payload = {}, params = {}, options = {}, traceId } = {}) {
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
let ownerRecord = null;
|
|
try {
|
|
ownerRecord = await recordCodeAgentSessionOwner({ payload, params, options, status: "running" });
|
|
if (!ownerRecord) {
|
|
const error = new Error("Code Agent turn admission did not durably persist Workbench session ownership.");
|
|
error.code = "workbench_admission_owner_not_persisted";
|
|
error.valuesRedacted = true;
|
|
throw error;
|
|
}
|
|
await recordCodeAgentSessionInputFact({
|
|
params,
|
|
options,
|
|
traceId,
|
|
delivery: "queue",
|
|
status: "admitted",
|
|
messageId: codeAgentTurnLifecycleFields(traceId, params).userMessageId,
|
|
commandId: payload.agentRun?.commandId ?? null
|
|
});
|
|
} catch (error) {
|
|
throw codeAgentAdmissionUnavailableError(error, { params, traceId });
|
|
}
|
|
traceStore.append(traceId, {
|
|
type: "request",
|
|
status: "admitted",
|
|
label: "turn:admitted",
|
|
message: "Code Agent turn was durably admitted before billing or dispatch preflight.",
|
|
sessionId: safeSessionId(params.sessionId) || null,
|
|
threadId: safeOpaqueId(params.threadId) || null,
|
|
...codeAgentOtelTraceFields(traceId, options.env ?? process.env),
|
|
waitingFor: "billing-or-dispatch",
|
|
valuesPrinted: false
|
|
});
|
|
void emitCodeAgentOtelSpan("durable_admission", traceId, options.env ?? process.env, {
|
|
attributes: { sessionId: safeSessionId(params.sessionId) || null, threadId: safeOpaqueId(params.threadId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId }
|
|
});
|
|
return ownerRecord;
|
|
}
|
|
|
|
async function recordCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, error = null } = {}) {
|
|
const runtimeStore = options.runtimeStore ?? null;
|
|
if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null;
|
|
const resolvedTraceId = safeTraceId(traceId ?? params.traceId) || null;
|
|
const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, params);
|
|
const sessionId = safeSessionId(params.sessionId ?? params.session?.sessionId) || null;
|
|
if (!sessionId) return null;
|
|
const resolvedMessageId = safeMessageId(messageId ?? params.messageId ?? lifecycle.userMessageId) || lifecycle.userMessageId;
|
|
const resolvedCommandId = textValue(commandId ?? params.commandId ?? params.agentRun?.commandId) || null;
|
|
const normalizedDelivery = codeAgentInputDelivery(delivery);
|
|
const normalizedStatus = codeAgentInputStatus(status);
|
|
const now = new Date().toISOString();
|
|
const sourceEventId = `${sessionId}:${lifecycle.turnId}:${normalizedDelivery}:${resolvedTraceId ?? resolvedCommandId ?? resolvedMessageId}`;
|
|
const inputId = stableCodeAgentInputId({ sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, commandId: resolvedCommandId, delivery: normalizedDelivery, sourceEventId });
|
|
const fact = {
|
|
inputId,
|
|
sessionId,
|
|
turnId: lifecycle.turnId,
|
|
traceId: resolvedTraceId,
|
|
messageId: resolvedMessageId,
|
|
commandId: resolvedCommandId,
|
|
delivery: normalizedDelivery,
|
|
status: normalizedStatus,
|
|
errorCode: textValue(error?.code ?? error?.error?.code) || null,
|
|
sourceEventId,
|
|
promptHash: params.message || params.prompt || params.text ? sha256Text(params.message ?? params.prompt ?? params.text).slice(0, 16) : null,
|
|
textBytes: params.message || params.prompt || params.text ? Buffer.byteLength(String(params.message ?? params.prompt ?? params.text), "utf8") : 0,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false,
|
|
createdAt: now,
|
|
updatedAt: now
|
|
};
|
|
return runtimeStore.writeWorkbenchFacts({ facts: { inputs: [fact] } }, {
|
|
sessionId,
|
|
traceId: resolvedTraceId,
|
|
turnId: lifecycle.turnId,
|
|
messageId: resolvedMessageId,
|
|
ownerUserId: options.actor?.id ?? params.ownerUserId,
|
|
projectId: params.projectId ?? DEFAULT_CODE_AGENT_PROJECT_ID,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
|
|
function stableCodeAgentInputId(value) {
|
|
const payload = Object.fromEntries(Object.entries(value ?? {}).filter(([, item]) => item !== undefined && item !== null && item !== "").sort(([left], [right]) => left.localeCompare(right)));
|
|
return `wsi_${sha256Text(JSON.stringify(payload)).slice(0, 32)}`;
|
|
}
|
|
|
|
function codeAgentInputDelivery(value) {
|
|
const text = textValue(value).toLowerCase();
|
|
return ["queue", "steer", "cancel"].includes(text) ? text : "queue";
|
|
}
|
|
|
|
function codeAgentInputStatus(value) {
|
|
const text = textValue(value).toLowerCase().replace(/-/gu, "_");
|
|
return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted";
|
|
}
|
|
|
|
function codeAgentAdmissionUnavailableError(error, { params = {}, traceId } = {}) {
|
|
const payload = codeAgentAdmissionUnavailablePayload({ error, params, traceId });
|
|
return Object.assign(new Error(payload.error.message), {
|
|
code: "admission_unavailable",
|
|
statusCode: 503,
|
|
payload,
|
|
alreadyClassified: true
|
|
});
|
|
}
|
|
|
|
function isCodeAgentAdmissionUnavailableError(error) {
|
|
return error?.code === "admission_unavailable" && error?.payload;
|
|
}
|
|
|
|
function codeAgentAdmissionUnavailablePayload({ error, params = {}, traceId } = {}) {
|
|
const message = error?.message ?? "Code Agent turn admission is unavailable; the user message was not persisted.";
|
|
return {
|
|
ok: false,
|
|
accepted: false,
|
|
status: "failed",
|
|
traceId: safeTraceId(traceId) || null,
|
|
...codeAgentOtelTraceFields(traceId, options.env ?? process.env),
|
|
...codeAgentTurnLifecycleFields(traceId, params),
|
|
conversationId: safeConversationId(params.conversationId) || null,
|
|
sessionId: safeSessionId(params.sessionId) || null,
|
|
threadId: safeOpaqueId(params.threadId) || null,
|
|
error: {
|
|
code: "admission_unavailable",
|
|
layer: "admission",
|
|
retryable: true,
|
|
message,
|
|
userMessage: "本次输入尚未持久化，请保留草稿后重试。",
|
|
route: "/v1/agent/chat"
|
|
},
|
|
blocker: {
|
|
code: "admission_unavailable",
|
|
layer: "admission",
|
|
retryable: true,
|
|
summary: message
|
|
},
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
function codeAgentDispatchFailure(error) {
|
|
const message = error?.message ?? "AgentRun dispatch failed";
|
|
return {
|
|
statusCode: 503,
|
|
payload: {
|
|
error: {
|
|
code: error?.code ?? "agentrun_dispatch_failed",
|
|
layer: "agentrun",
|
|
retryable: true,
|
|
message,
|
|
userMessage: "AgentRun 调度失败；trace/result 轮询已保留错误。",
|
|
route: "/v1/agent/chat"
|
|
},
|
|
blocker: {
|
|
code: error?.code ?? "agentrun_dispatch_failed",
|
|
layer: "agentrun",
|
|
retryable: true,
|
|
summary: message
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
function codeAgentDispatchErrorOtelAttributes(error) {
|
|
const agentRunError = error?.agentRunError && typeof error.agentRunError === "object" ? error.agentRunError : null;
|
|
const details = agentRunError?.error?.details && typeof agentRunError.error.details === "object" ? agentRunError.error.details : null;
|
|
return {
|
|
failureKind: safeOtelText(error?.code ?? agentRunError?.failureKind, 120),
|
|
upstreamStatusCode: Number.isInteger(error?.statusCode) ? error.statusCode : null,
|
|
upstreamTraceId: safeTraceId(agentRunError?.traceId) || null,
|
|
upstreamMessage: safeOtelText(agentRunError?.message ?? error?.message, 300),
|
|
upstreamStderrTail: safeOtelText(details?.stderr, 600)
|
|
};
|
|
}
|
|
|
|
function safeOtelText(value, limit = 300) {
|
|
const text = String(value ?? "").replace(/[\u0000-\u001f\u007f]+/gu, " ").trim();
|
|
return text ? text.slice(0, limit) : null;
|
|
}
|
|
|
|
function codeAgentAdmittedFailureBasePayload({ params = {}, options = {}, traceId } = {}) {
|
|
const now = new Date().toISOString();
|
|
return {
|
|
accepted: true,
|
|
status: "running",
|
|
shortConnection: true,
|
|
traceId,
|
|
...codeAgentTurnLifecycleFields(traceId, params),
|
|
messageId: codeAgentTurnLifecycleFields(traceId, params).assistantMessageId,
|
|
projectId: params.projectId ?? null,
|
|
conversationId: safeConversationId(params.conversationId) || null,
|
|
sessionId: safeSessionId(params.sessionId) || null,
|
|
threadId: safeOpaqueId(params.threadId) || null,
|
|
...codeAgentAdmissionTimingFields(now),
|
|
provider: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun" : "codex-stdio",
|
|
model: options.env?.HWLAB_CODE_AGENT_MODEL ?? "unknown",
|
|
backend: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun-v01" : "hwlab-cloud-api/code-agent-chat",
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function codeAgentAdmissionTimingFields(value = null) {
|
|
const admittedAt = timestampIsoOrNow(value);
|
|
const timing = {
|
|
startedAt: admittedAt,
|
|
lastEventAt: admittedAt,
|
|
finishedAt: null,
|
|
durationMs: null,
|
|
observedAt: admittedAt,
|
|
lastEventAgeMs: 0,
|
|
valuesRedacted: true
|
|
};
|
|
return {
|
|
createdAt: admittedAt,
|
|
updatedAt: admittedAt,
|
|
timing,
|
|
startedAt: admittedAt,
|
|
lastEventAt: admittedAt,
|
|
finishedAt: null,
|
|
durationMs: null
|
|
};
|
|
}
|
|
|
|
function codeAgentResultTimingFields(...sources) {
|
|
const records = sources.filter((source) => source && typeof source === "object");
|
|
const timingSource = records.map((source) => source.timing).find((timing) => timing && typeof timing === "object") ?? {};
|
|
const startedAt = firstTimestampIso(timingSource.startedAt, ...records.map((source) => source.startedAt), ...records.map((source) => source.createdAt));
|
|
const lastEventAt = firstLatestTimestampIso(timingSource.lastEventAt, ...records.map((source) => source.lastEventAt), ...records.map((source) => source.updatedAt), ...records.map((source) => source.createdAt));
|
|
const finishedAt = firstLatestTimestampIso(timingSource.finishedAt, ...records.map((source) => source.finishedAt), ...records.map((source) => source.completedAt));
|
|
const durationMs = numberOrNull(timingSource.durationMs ?? records.map((source) => source.durationMs).find((value) => numberOrNull(value) !== null));
|
|
const observedAt = timestampIsoOrNull(timingSource.observedAt);
|
|
const lastEventAgeMs = numberOrNull(timingSource.lastEventAgeMs);
|
|
if (!startedAt && !lastEventAt && !finishedAt && durationMs === null && lastEventAgeMs === null) return {};
|
|
const timing = {
|
|
...(timingSource && typeof timingSource === "object" ? timingSource : {}),
|
|
startedAt: startedAt ?? null,
|
|
lastEventAt: lastEventAt ?? null,
|
|
finishedAt: finishedAt ?? null,
|
|
durationMs,
|
|
observedAt: observedAt ?? null,
|
|
lastEventAgeMs,
|
|
valuesRedacted: timingSource.valuesRedacted !== false
|
|
};
|
|
return {
|
|
timing,
|
|
startedAt: timing.startedAt,
|
|
lastEventAt: timing.lastEventAt,
|
|
finishedAt: timing.finishedAt,
|
|
durationMs: timing.durationMs
|
|
};
|
|
}
|
|
|
|
function firstTimestampIso(...values) {
|
|
for (const value of values) {
|
|
const timestamp = timestampIsoOrNull(value);
|
|
if (timestamp) return timestamp;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function firstLatestTimestampIso(...values) {
|
|
let latest = null;
|
|
let latestMs = Number.NEGATIVE_INFINITY;
|
|
for (const value of values) {
|
|
const timestamp = timestampIsoOrNull(value);
|
|
if (!timestamp) continue;
|
|
const ms = Date.parse(timestamp);
|
|
if (!Number.isFinite(ms) || ms < latestMs) continue;
|
|
latest = timestamp;
|
|
latestMs = ms;
|
|
}
|
|
return latest;
|
|
}
|
|
|
|
async function settleAdmittedCodeAgentFailure({ base = {}, params = {}, options = {}, traceId, results, failure = {}, traceLabel = "code-agent:failed" } = {}) {
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
const error = failure.payload?.error ?? failure.error ?? {};
|
|
const blocker = failure.payload?.blocker ?? failure.blocker ?? null;
|
|
const message = error.message ?? blocker?.summary ?? "Code Agent request failed after admission.";
|
|
traceStore.append(traceId, {
|
|
type: "result",
|
|
status: "failed",
|
|
label: traceLabel,
|
|
errorCode: error.code ?? "code_agent_failed_after_admission",
|
|
message,
|
|
terminal: true,
|
|
valuesPrinted: false
|
|
});
|
|
const now = new Date().toISOString();
|
|
const payload = annotateOwner({
|
|
...base,
|
|
accepted: true,
|
|
status: "failed",
|
|
traceId,
|
|
...codeAgentTurnLifecycleFields(traceId, base),
|
|
messageId: base.messageId ?? codeAgentTurnLifecycleFields(traceId, base).assistantMessageId,
|
|
conversationId: safeConversationId(base.conversationId ?? params.conversationId) || null,
|
|
sessionId: safeSessionId(base.sessionId ?? params.sessionId) || null,
|
|
threadId: safeOpaqueId(base.threadId ?? params.threadId) || null,
|
|
error: {
|
|
code: error.code ?? "code_agent_failed_after_admission",
|
|
layer: error.layer ?? blocker?.layer ?? "api",
|
|
retryable: error.retryable !== false,
|
|
message,
|
|
userMessage: error.userMessage ?? message,
|
|
route: error.route ?? "/v1/agent/chat"
|
|
},
|
|
blocker: blocker ? {
|
|
code: blocker.code ?? error.code ?? "code_agent_failed_after_admission",
|
|
layer: blocker.layer ?? error.layer ?? "api",
|
|
retryable: blocker.retryable !== false,
|
|
summary: blocker.summary ?? message
|
|
} : null,
|
|
finalResponse: {
|
|
text: error.userMessage ?? message,
|
|
textChars: String(error.userMessage ?? message).length,
|
|
role: "assistant",
|
|
status: "failed",
|
|
traceId,
|
|
updatedAt: now,
|
|
source: "code-agent-admitted-failure",
|
|
valuesPrinted: false
|
|
},
|
|
runnerTrace: traceStore.snapshot(traceId),
|
|
updatedAt: now,
|
|
valuesPrinted: false,
|
|
secretMaterialStored: false
|
|
}, params);
|
|
await finalizeCodeAgentBillingUsage({ payload, params, options });
|
|
payload.runnerTrace = traceStore.snapshot(traceId);
|
|
recordCodeAgentConversationFact(payload, options);
|
|
results?.set?.(traceId, payload);
|
|
await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload) });
|
|
return payload;
|
|
}
|
|
|
|
function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore, currentResult = null } = {}) {
|
|
if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null;
|
|
const env = options.env ?? process.env;
|
|
const noResponseTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS, null);
|
|
const pollIntervalMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS, 1_000);
|
|
const refreshOptions = codeAgentTurnStatusRefreshOptions(options);
|
|
return scheduleWorkbenchProjectionFinalizer({
|
|
traceId,
|
|
currentResult,
|
|
traceStore,
|
|
noResponseTimeoutMs,
|
|
pollIntervalMs,
|
|
getCachedResult: () => options.codeAgentChatResults?.get?.(traceId),
|
|
isCanceled: isCodeAgentResultCanceled,
|
|
isTerminal: agentRunProjectionObservedTerminal,
|
|
activitySignature: agentRunProjectionActivitySignature,
|
|
sleep: sleepAgentRunProjectionSync,
|
|
performanceStore: options.backendPerformanceStore,
|
|
syncResult: ({ result }) => syncAgentRunChatResult({
|
|
traceId,
|
|
currentResult: result,
|
|
options: refreshOptions,
|
|
traceStore,
|
|
forceResultSync: agentRunProjectionObservedTerminal(result, traceStore?.snapshot?.(traceId))
|
|
}),
|
|
onTerminal: (payload, finalizerOptions = {}) => scheduleCodeAgentTerminalTurnStatusEffects({ payload: codeAgentPayloadWithObservedTerminalStatus(payload, traceStore?.snapshot?.(traceId)) ?? payload, params, options, preserveLastTraceId: finalizerOptions.preserveLastTraceId === true })
|
|
});
|
|
}
|
|
|
|
export function startAgentRunProjectionResume({ options = {}, traceStore = defaultCodeAgentTraceStore, logger = console } = {}) {
|
|
const env = options.env ?? process.env;
|
|
if (!truthyFlag(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED)) {
|
|
return { started: false, reason: "disabled", stop() {}, valuesPrinted: false };
|
|
}
|
|
const initialDelayMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS);
|
|
const intervalMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS);
|
|
const minAgeMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS);
|
|
const batchSize = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE);
|
|
const jitterMs = intervalMs > 0 ? Math.min(parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_JITTER_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_JITTER_MS), intervalMs) : 0;
|
|
const failureBackoffMs = Math.max(intervalMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS));
|
|
const retryInitialDelayFloorMs = Math.max(1000, intervalMs * 2);
|
|
const candidateRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_ATTEMPTS, 5);
|
|
const candidateRetryInitialDelayMs = Math.max(retryInitialDelayFloorMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_INITIAL_DELAY_MS, retryInitialDelayFloorMs));
|
|
const candidateRetryMaxDelayMs = Math.max(candidateRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_DELAY_MS, 600000));
|
|
const passRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_ATTEMPTS, 5);
|
|
const passRetryInitialDelayMs = Math.max(failureBackoffMs, retryInitialDelayFloorMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_INITIAL_DELAY_MS, retryInitialDelayFloorMs));
|
|
const passRetryMaxDelayMs = Math.max(passRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_DELAY_MS, 600000));
|
|
const candidateRetryState = new Map();
|
|
let passFailureAttempt = 0;
|
|
let stopped = false;
|
|
let active = false;
|
|
let timer = null;
|
|
let nextOffset = 0;
|
|
|
|
const schedule = (delayMs, reason) => {
|
|
if (stopped) return;
|
|
const scheduledDelayMs = Math.max(0, Number(delayMs) || 0) + randomJitterMs(jitterMs);
|
|
timer = setTimeout(() => {
|
|
timer = null;
|
|
const offset = nextOffset;
|
|
active = true;
|
|
let nextDelayMs = intervalMs;
|
|
let nextReason = "interval";
|
|
void runAgentRunProjectionResumePass({ options, traceStore, logger, batchSize, minAgeMs, offset, reason, candidateRetryState, candidateRetryMaxAttempts, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs })
|
|
.then((result) => {
|
|
passFailureAttempt = 0;
|
|
const candidateFailed = result?.failed > 0;
|
|
nextOffset = candidateFailed ? offset : (result?.scannedSessions >= batchSize ? offset + batchSize : 0);
|
|
nextDelayMs = candidateFailed ? failureBackoffMs : intervalMs;
|
|
nextReason = candidateFailed ? "failure_backoff" : "interval";
|
|
})
|
|
.catch((error) => {
|
|
passFailureAttempt += 1;
|
|
const retryDelayMs = projectionResumeCandidateRetryDelayMs(passFailureAttempt, passRetryInitialDelayMs, passRetryMaxDelayMs);
|
|
const retryExhausted = passFailureAttempt >= passRetryMaxAttempts;
|
|
nextDelayMs = retryDelayMs;
|
|
nextReason = retryExhausted ? "pass_retry_exhausted" : "pass_failure_backoff";
|
|
logAgentRunProjectionResume(logger, {
|
|
event: "workbench_projection_resume",
|
|
ok: false,
|
|
status: retryExhausted ? "stopped" : "failed",
|
|
reason,
|
|
offset,
|
|
errorCode: error?.code ?? "projection_resume_pass_failed",
|
|
message: error?.message ?? "AgentRun projection resume pass failed.",
|
|
passRetryAttempt: passFailureAttempt,
|
|
passRetryMaxAttempts,
|
|
passRetryLabel: passFailureAttempt + "/" + passRetryMaxAttempts,
|
|
passRetryDelayMs: retryDelayMs,
|
|
passRetryExhausted: retryExhausted,
|
|
valuesRedacted: true
|
|
});
|
|
if (retryExhausted) stopped = true;
|
|
})
|
|
.finally(() => {
|
|
active = false;
|
|
if (!stopped && intervalMs > 0) schedule(nextDelayMs, nextReason);
|
|
});
|
|
}, scheduledDelayMs);
|
|
timer.unref?.();
|
|
};
|
|
|
|
schedule(initialDelayMs, "startup");
|
|
return {
|
|
started: true,
|
|
initialDelayMs,
|
|
intervalMs,
|
|
minAgeMs,
|
|
batchSize,
|
|
jitterMs,
|
|
failureBackoffMs,
|
|
candidateRetryMaxAttempts,
|
|
candidateRetryInitialDelayMs,
|
|
candidateRetryMaxDelayMs,
|
|
passRetryMaxAttempts,
|
|
passRetryInitialDelayMs,
|
|
passRetryMaxDelayMs,
|
|
valuesPrinted: false,
|
|
stop() {
|
|
stopped = true;
|
|
if (timer) clearTimeout(timer);
|
|
timer = null;
|
|
},
|
|
active: () => active
|
|
};
|
|
}
|
|
|
|
async function runAgentRunProjectionResumePass({ options = {}, traceStore = defaultCodeAgentTraceStore, logger = console, batchSize, minAgeMs, offset = 0, reason = "manual", candidateRetryState = new Map(), candidateRetryMaxAttempts = 5, candidateRetryInitialDelayMs = 30000, candidateRetryMaxDelayMs = 600000 } = {}) {
|
|
const stateCandidates = await listAgentRunProjectionResumeStateCandidates(options, batchSize, offset, { minAgeMs });
|
|
const sessions = stateCandidates.length > 0 ? [] : await listAgentRunProjectionResumeSessions(options, batchSize, offset);
|
|
const candidates = stateCandidates.length > 0 ? stateCandidates : sessions.flatMap((session) => agentRunProjectionResumeCandidates(session, { minAgeMs }));
|
|
let resumed = 0;
|
|
let terminal = 0;
|
|
let nonTerminal = 0;
|
|
let failed = 0;
|
|
let skippedCooldown = 0;
|
|
let skippedExhausted = 0;
|
|
const nowMs = Date.now();
|
|
for (const candidate of candidates) {
|
|
const retryKey = projectionResumeCandidateRetryKey(candidate);
|
|
const retryState = candidateRetryState.get(retryKey);
|
|
if (retryState?.exhausted === true) {
|
|
skippedExhausted += 1;
|
|
continue;
|
|
}
|
|
if (Number.isFinite(retryState?.nextRetryAtMs) && retryState.nextRetryAtMs > nowMs) {
|
|
skippedCooldown += 1;
|
|
continue;
|
|
}
|
|
try {
|
|
const result = await resumeAgentRunProjectionCandidate({ candidate, options, traceStore });
|
|
resumed += 1;
|
|
const currentTraceSnapshot = safeTraceStoreSnapshot(traceStore, candidate.traceId);
|
|
if (agentRunProjectionObservedTerminal(result, result?.runnerTrace ?? currentTraceSnapshot) || traceSnapshotIsTerminal(currentTraceSnapshot)) {
|
|
terminal += 1;
|
|
candidateRetryState.delete(retryKey);
|
|
} else {
|
|
nonTerminal += 1;
|
|
const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0;
|
|
const retryAttempt = previousAttempts + 1;
|
|
const retryExhausted = retryAttempt >= candidateRetryMaxAttempts;
|
|
const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs);
|
|
const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs;
|
|
const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null;
|
|
candidateRetryState.set(retryKey, {
|
|
attempts: retryAttempt,
|
|
nextRetryAtMs,
|
|
exhausted: retryExhausted,
|
|
lastErrorCode: "projection_resume_non_terminal"
|
|
});
|
|
appendProjectionResumeRetry(traceStore, candidate, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted, status: result?.status ?? null });
|
|
}
|
|
} catch (error) {
|
|
failed += 1;
|
|
const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0;
|
|
const retryAttempt = previousAttempts + 1;
|
|
const retryExhausted = retryAttempt >= candidateRetryMaxAttempts;
|
|
const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs);
|
|
const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs;
|
|
const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null;
|
|
candidateRetryState.set(retryKey, {
|
|
attempts: retryAttempt,
|
|
nextRetryAtMs,
|
|
exhausted: retryExhausted,
|
|
lastErrorCode: error?.code ?? "projection_resume_sync_failed"
|
|
});
|
|
appendProjectionResumeError(traceStore, candidate, error, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted });
|
|
}
|
|
}
|
|
logAgentRunProjectionResume(logger, {
|
|
event: "workbench_projection_resume",
|
|
ok: failed === 0,
|
|
status: failed === 0 ? "completed" : "degraded",
|
|
reason,
|
|
offset,
|
|
scannedProjectionStates: stateCandidates.length,
|
|
scannedSessions: sessions.length,
|
|
candidates: candidates.length,
|
|
resumed,
|
|
terminal,
|
|
nonTerminal,
|
|
failed,
|
|
skippedCooldown,
|
|
skippedExhausted,
|
|
candidateRetryTracked: candidateRetryState.size,
|
|
candidateRetryMaxAttempts,
|
|
valuesRedacted: true
|
|
});
|
|
return { offset, scannedSessions: sessions.length, candidates: candidates.length, resumed, terminal, nonTerminal, failed, skippedCooldown, skippedExhausted };
|
|
}
|
|
|
|
async function listAgentRunProjectionResumeStateCandidates(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS } = {}) {
|
|
const runtimeStore = options.runtimeStore;
|
|
if (typeof runtimeStore?.queryWorkbenchProjectionStates !== "function") return [];
|
|
let result;
|
|
try {
|
|
result = await runtimeStore.queryWorkbenchProjectionStates({
|
|
projectionStatuses: ["projecting", "degraded"],
|
|
dueAt: new Date().toISOString(),
|
|
limit: batchSize,
|
|
offset
|
|
});
|
|
} catch {
|
|
return [];
|
|
}
|
|
return (Array.isArray(result?.states) ? result.states : [])
|
|
.map((state) => agentRunProjectionResumeCandidateFromState(state, { minAgeMs }))
|
|
.filter(Boolean);
|
|
}
|
|
|
|
async function listAgentRunProjectionResumeSessions(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0) {
|
|
const accessStore = options.accessController?.store ?? options.accessController ?? null;
|
|
if (typeof accessStore?.listAgentSessionsForUser !== "function") return [];
|
|
return await accessStore.listAgentSessionsForUser({
|
|
actorRole: "admin",
|
|
ownerScoped: false,
|
|
includeArchived: false,
|
|
limit: batchSize,
|
|
offset
|
|
}) ?? [];
|
|
}
|
|
|
|
function agentRunProjectionResumeCandidates(session = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) {
|
|
const snapshot = session?.session && typeof session.session === "object" ? session.session : {};
|
|
const records = new Map();
|
|
const traceResults = snapshot.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null;
|
|
if (traceResults) {
|
|
for (const [traceId, record] of Object.entries(traceResults)) {
|
|
const safeId = safeTraceId(traceId);
|
|
if (safeId && record && typeof record === "object") records.set(safeId, { ...record, traceId: safeId });
|
|
}
|
|
}
|
|
const directTraceId = safeTraceId(snapshot.traceId ?? session.lastTraceId);
|
|
if (directTraceId && snapshot.agentRun && typeof snapshot.agentRun === "object") {
|
|
records.set(directTraceId, { ...snapshot, traceId: directTraceId });
|
|
}
|
|
return [...records.values()]
|
|
.map((record) => agentRunProjectionResumeCandidate(session, record, { minAgeMs, nowMs }))
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function agentRunProjectionResumeCandidate(session = {}, record = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) {
|
|
const traceId = safeTraceId(record.traceId ?? session.lastTraceId);
|
|
const agentRun = record.agentRun && typeof record.agentRun === "object" ? record.agentRun : null;
|
|
if (!traceId || !agentRun?.runId || !agentRun?.commandId) return null;
|
|
if (isTraceCommandTerminalStatus(record.status ?? agentRun.terminalStatus ?? agentRun.commandState ?? agentRun.status)) return null;
|
|
const updatedAtMs = Date.parse(record.updatedAt ?? session.updatedAt ?? "");
|
|
if (minAgeMs > 0 && Number.isFinite(updatedAtMs) && nowMs - updatedAtMs < minAgeMs) return null;
|
|
const sessionId = safeSessionId(record.sessionId ?? session.id) || null;
|
|
const conversationId = safeConversationId(record.conversationId ?? session.conversationId) || null;
|
|
const threadId = safeOpaqueId(record.threadId ?? session.threadId) || null;
|
|
const ownerUserId = textValue(record.ownerUserId ?? session.ownerUserId);
|
|
const ownerRole = textValue(record.ownerRole ?? session.ownerRole) || "user";
|
|
return {
|
|
traceId,
|
|
params: {
|
|
traceId,
|
|
sessionId,
|
|
conversationId,
|
|
threadId,
|
|
projectId: record.projectId ?? session.projectId ?? null,
|
|
ownerUserId,
|
|
ownerRole
|
|
},
|
|
result: {
|
|
...record,
|
|
traceId,
|
|
sessionId,
|
|
conversationId,
|
|
threadId,
|
|
ownerUserId,
|
|
ownerRole,
|
|
status: record.status || session.status || "running",
|
|
finalResponse: null,
|
|
traceSummary: null,
|
|
agentRun: {
|
|
...agentRun,
|
|
traceId,
|
|
providerTrace: agentRun.providerTrace ?? { traceId, runId: agentRun.runId, commandId: agentRun.commandId, valuesPrinted: false },
|
|
lastSeq: Number.isFinite(Number(agentRun.lastSeq)) ? Number(agentRun.lastSeq) : 0,
|
|
valuesPrinted: false
|
|
},
|
|
valuesRedacted: true
|
|
}
|
|
};
|
|
}
|
|
|
|
function agentRunProjectionResumeCandidateFromState(state = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) {
|
|
const traceId = safeTraceId(state.traceId);
|
|
const runId = textValue(state.runId ?? state.sourceRunId);
|
|
const commandId = textValue(state.commandId ?? state.sourceCommandId);
|
|
if (!traceId || !runId || !commandId) return null;
|
|
if (isTraceCommandTerminalStatus(state.projectionStatus)) return null;
|
|
const updatedAtMs = Date.parse(state.updatedAt ?? "");
|
|
if (minAgeMs > 0 && Number.isFinite(updatedAtMs) && nowMs - updatedAtMs < minAgeMs) return null;
|
|
const sessionId = safeSessionId(state.sessionId) || null;
|
|
const conversationId = safeConversationId(state.conversationId) || null;
|
|
const threadId = safeOpaqueId(state.threadId) || null;
|
|
const ownerUserId = textValue(state.ownerUserId);
|
|
const ownerRole = textValue(state.ownerRole) || "user";
|
|
const lastSeq = Number.isFinite(Number(state.lastSourceSeq ?? state.lastAgentRunSeq)) ? Number(state.lastSourceSeq ?? state.lastAgentRunSeq) : 0;
|
|
return {
|
|
traceId,
|
|
params: {
|
|
traceId,
|
|
sessionId,
|
|
conversationId,
|
|
threadId,
|
|
ownerUserId,
|
|
ownerRole
|
|
},
|
|
result: {
|
|
accepted: true,
|
|
status: "running",
|
|
shortConnection: true,
|
|
traceId,
|
|
sessionId,
|
|
conversationId,
|
|
threadId,
|
|
ownerUserId,
|
|
ownerRole,
|
|
finalResponse: null,
|
|
traceSummary: null,
|
|
agentRun: {
|
|
traceId,
|
|
runId,
|
|
commandId,
|
|
lastSeq,
|
|
managerUrl: state.managerUrl ?? null,
|
|
backendProfile: state.backendProfile ?? null,
|
|
providerId: state.providerId ?? null,
|
|
status: "running",
|
|
commandState: "running",
|
|
resultSyncState: state.resultSyncState ?? null,
|
|
providerTrace: { traceId, runId, commandId, valuesPrinted: false },
|
|
valuesPrinted: false
|
|
},
|
|
valuesRedacted: true
|
|
}
|
|
};
|
|
}
|
|
|
|
async function resumeAgentRunProjectionCandidate({ candidate, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) {
|
|
const { currentResult } = await agentRunProjectionResumeResultWithCursor(candidate, options);
|
|
const resumeOptions = { ...options, deferAgentRunResultSync: false };
|
|
const synced = await syncAgentRunChatResult({
|
|
traceId: candidate.traceId,
|
|
currentResult,
|
|
options: resumeOptions,
|
|
traceStore,
|
|
forceResultSync: true,
|
|
refreshEvents: true
|
|
});
|
|
const payload = codeAgentPayloadWithObservedTerminalStatus(synced?.result ?? currentResult, synced?.runnerTrace ?? traceStore?.snapshot?.(candidate.traceId)) ?? (synced?.result ?? currentResult);
|
|
if (isTraceCommandTerminalStatus(payload?.status)) {
|
|
await recordCodeAgentTerminalTurnStatusEffects({ payload, params: candidate.params, options, preserveLastTraceId: true });
|
|
}
|
|
return payload;
|
|
}
|
|
|
|
async function agentRunProjectionResumeResultWithCursor(candidate, options = {}) {
|
|
const projectionState = await loadAgentRunProjectionState(options.runtimeStore, candidate);
|
|
const plan = buildAgentRunProjectionEventsFetchPlan({
|
|
projectionState,
|
|
agentRun: candidate.result.agentRun,
|
|
pageLimit: options.env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT
|
|
});
|
|
return {
|
|
projectionState,
|
|
currentResult: {
|
|
...candidate.result,
|
|
traceSummary: null,
|
|
agentRun: {
|
|
...candidate.result.agentRun,
|
|
lastSeq: plan.afterSeq,
|
|
valuesPrinted: false
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
async function loadAgentRunProjectionState(runtimeStore, candidate) {
|
|
if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null;
|
|
try {
|
|
const result = await runtimeStore.getWorkbenchProjectionState({ traceId: candidate.traceId });
|
|
const state = result?.projectionState ?? null;
|
|
if (!state) return null;
|
|
if (String(state.runId ?? state.sourceRunId ?? "") !== String(candidate.result.agentRun.runId ?? "")) return null;
|
|
if (String(state.commandId ?? state.sourceCommandId ?? "") !== String(candidate.result.agentRun.commandId ?? "")) return null;
|
|
return state;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function projectionResumeCandidateRetryKey(candidate) {
|
|
const traceId = safeTraceId(candidate?.traceId) || "trace-unknown";
|
|
const runId = String(candidate?.result?.agentRun?.runId ?? "run-unknown");
|
|
const commandId = String(candidate?.result?.agentRun?.commandId ?? "command-unknown");
|
|
return [traceId, runId, commandId].join("|");
|
|
}
|
|
|
|
function projectionResumeCandidateRetryDelayMs(retryAttempt, initialDelayMs, maxDelayMs) {
|
|
const attempt = Math.max(1, Number(retryAttempt) || 1);
|
|
return Math.min(maxDelayMs, initialDelayMs * (2 ** Math.max(0, attempt - 1)));
|
|
}
|
|
|
|
function appendProjectionResumeError(traceStore, candidate, error, retry = {}) {
|
|
const traceId = safeTraceId(candidate?.traceId);
|
|
if (!traceId) return;
|
|
if (traceSnapshotIsTerminal(safeTraceStoreSnapshot(traceStore, traceId))) return;
|
|
traceStore.append(traceId, {
|
|
type: "turn-status",
|
|
status: "degraded",
|
|
label: "projection-resume:sync-failed",
|
|
errorCode: error?.code ?? "projection_resume_sync_failed",
|
|
message: error?.message ?? "AgentRun projection resume failed.",
|
|
runId: candidate?.result?.agentRun?.runId ?? null,
|
|
commandId: candidate?.result?.agentRun?.commandId ?? null,
|
|
waitingFor: "agentrun-result-resume",
|
|
retryAttempt: retry.retryAttempt ?? null,
|
|
retryMaxAttempts: retry.retryMaxAttempts ?? null,
|
|
retryLabel: retry.retryLabel ?? null,
|
|
retryDelayMs: retry.retryDelayMs ?? null,
|
|
nextRetryAt: retry.nextRetryAt ?? null,
|
|
retryExhausted: retry.retryExhausted === true,
|
|
terminal: false,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
|
|
function appendProjectionResumeRetry(traceStore, candidate, retry = {}) {
|
|
const traceId = safeTraceId(candidate?.traceId);
|
|
if (!traceId) return;
|
|
if (traceSnapshotIsTerminal(safeTraceStoreSnapshot(traceStore, traceId))) return;
|
|
traceStore.append(traceId, {
|
|
type: "turn-status",
|
|
status: "degraded",
|
|
label: "projection-resume:non-terminal-backoff",
|
|
errorCode: "projection_resume_non_terminal",
|
|
message: "AgentRun projection resume did not reach a terminal status; next resume is delayed.",
|
|
runId: candidate?.result?.agentRun?.runId ?? null,
|
|
commandId: candidate?.result?.agentRun?.commandId ?? null,
|
|
waitingFor: "agentrun-result-resume",
|
|
observedStatus: retry.status ?? null,
|
|
retryAttempt: retry.retryAttempt ?? null,
|
|
retryMaxAttempts: retry.retryMaxAttempts ?? null,
|
|
retryLabel: retry.retryLabel ?? null,
|
|
retryDelayMs: retry.retryDelayMs ?? null,
|
|
nextRetryAt: retry.nextRetryAt ?? null,
|
|
retryExhausted: retry.retryExhausted === true,
|
|
terminal: false,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
|
|
function safeTraceStoreSnapshot(traceStore, traceId) {
|
|
try {
|
|
return typeof traceStore?.snapshot === "function" ? traceStore.snapshot(traceId) : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function traceSnapshotIsTerminal(snapshot) {
|
|
if (!snapshot || typeof snapshot !== "object") return false;
|
|
if (snapshot.finishedAt) return true;
|
|
if (isTraceCommandTerminalStatus(snapshot.status)) return true;
|
|
if (isTraceCommandTerminalStatus(snapshot.lastEvent?.status) || snapshot.lastEvent?.terminal === true) return true;
|
|
const events = Array.isArray(snapshot.events) ? snapshot.events : [];
|
|
return events.some((event) => event?.terminal === true || isTraceCommandTerminalStatus(event?.status));
|
|
}
|
|
|
|
function logAgentRunProjectionResume(logger, payload) {
|
|
try {
|
|
const line = JSON.stringify(payload);
|
|
if (payload.ok === false && typeof logger?.warn === "function") logger.warn(line);
|
|
else if (typeof logger?.log === "function") logger.log(line);
|
|
} catch {
|
|
// Projection resume logging must not affect the projector.
|
|
}
|
|
}
|
|
|
|
function parseNonNegativeInteger(value, fallback) {
|
|
const parsed = Number.parseInt(value ?? "", 10);
|
|
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
|
}
|
|
|
|
function randomJitterMs(maxJitterMs) {
|
|
return maxJitterMs > 0 ? Math.floor(Math.random() * (maxJitterMs + 1)) : 0;
|
|
}
|
|
|
|
function agentRunProjectionActivitySignature(result, runnerTrace) {
|
|
const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {};
|
|
const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace;
|
|
const events = Array.isArray(trace?.events) ? trace.events : [];
|
|
const lastEvent = trace?.lastEvent ?? events.at(-1) ?? null;
|
|
return [
|
|
result?.status,
|
|
result?.updatedAt,
|
|
agentRun.status,
|
|
agentRun.runStatus,
|
|
agentRun.commandState,
|
|
agentRun.terminalStatus,
|
|
agentRun.lastSeq,
|
|
agentRun.updatedAt,
|
|
trace?.status,
|
|
trace?.eventCount ?? events.length,
|
|
trace?.updatedAt,
|
|
trace?.lastEventLabel,
|
|
lastEvent?.seq,
|
|
lastEvent?.sourceSeq,
|
|
lastEvent?.label,
|
|
lastEvent?.type,
|
|
lastEvent?.status,
|
|
lastEvent?.createdAt
|
|
].map((value) => value ?? "").join("|");
|
|
}
|
|
|
|
function agentRunProjectionObservedTerminal(result, runnerTrace) {
|
|
return Boolean(codeAgentObservedTerminalStatus(result, runnerTrace));
|
|
}
|
|
|
|
function codeAgentPayloadWithObservedTerminalStatus(payload, runnerTrace) {
|
|
const terminalStatus = codeAgentObservedTerminalStatus(payload, runnerTrace);
|
|
if (!terminalStatus || !payload || typeof payload !== "object") return null;
|
|
const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : null;
|
|
const agentRunTerminalStatus = terminalStatus === "canceled" ? "cancelled" : terminalStatus;
|
|
return {
|
|
...payload,
|
|
status: terminalStatus,
|
|
runnerTrace: runnerTrace && typeof runnerTrace === "object" ? runnerTrace : payload.runnerTrace,
|
|
...(agentRun ? { agentRun: { ...agentRun, terminalStatus: agentRun.terminalStatus ?? agentRunTerminalStatus, valuesPrinted: false } } : {})
|
|
};
|
|
}
|
|
|
|
function codeAgentObservedTerminalStatus(result, runnerTrace) {
|
|
const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {};
|
|
if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed";
|
|
for (const value of [agentRun.terminalStatus, agentRun.commandState]) {
|
|
const terminalStatus = normalizeCodeAgentTerminalStatus(value);
|
|
if (terminalStatus) return terminalStatus;
|
|
}
|
|
const commandId = textValue(agentRun.commandId);
|
|
const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace;
|
|
const events = Array.isArray(trace?.events) ? trace.events : [];
|
|
for (const event of [...events].reverse()) {
|
|
const payload = event?.payload && typeof event.payload === "object" ? event.payload : {};
|
|
const eventCommandId = textValue(event?.commandId ?? payload.commandId);
|
|
if (commandId && eventCommandId && eventCommandId !== commandId) continue;
|
|
for (const value of [
|
|
event?.type === "terminal_status" ? payload.terminalStatus : null,
|
|
payload.phase === "command-terminal" ? payload.terminalStatus : null,
|
|
payload.phase === "turn:completed" || payload.phase === "turn/steer:completed" ? "completed" : null,
|
|
event?.terminal === true ? payload.terminalStatus ?? payload.status ?? event.status ?? "completed" : null,
|
|
payload.terminal === true ? payload.terminalStatus ?? payload.status ?? "completed" : null
|
|
]) {
|
|
const terminalStatus = normalizeCodeAgentTerminalStatus(value);
|
|
if (terminalStatus) return terminalStatus;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function normalizeCodeAgentTerminalStatus(value) {
|
|
const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
|
if (!CODE_AGENT_TERMINAL_STATUSES.has(status)) return null;
|
|
return status === "cancelled" ? "canceled" : status;
|
|
}
|
|
|
|
function sleepAgentRunProjectionSync(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0)));
|
|
}
|
|
|
|
function codeAgentChatShortConnectionRequested(request, params, options = {}) {
|
|
const header = firstHeaderValue(request, "prefer");
|
|
const explicit = firstHeaderValue(request, "x-hwlab-short-connection");
|
|
const envValue = options.env?.HWLAB_CODE_AGENT_CHAT_SHORT_CONNECTION;
|
|
return codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ||
|
|
/(?:^|,)\s*respond-async\s*(?:,|$)/iu.test(String(header ?? "")) ||
|
|
truthyFlag(explicit) ||
|
|
params?.shortConnection === true ||
|
|
truthyFlag(envValue);
|
|
}
|
|
|
|
async function preflightCodeAgentBilling({ params = {}, options = {}, traceId, response, sendResponse = true } = {}) {
|
|
const client = options.userBillingClient;
|
|
if (!codeAgentBillingEnabled(options) || !client?.configured || !options.userBillingAuth?.active) return null;
|
|
const estimatedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_ESTIMATED_CREDITS, 1);
|
|
const body = {
|
|
...(options.actorApiKeySecret ? { apiKey: options.actorApiKeySecret } : { userId: options.actor?.id }),
|
|
serviceId: "hwlab-code-agent",
|
|
estimatedCredits,
|
|
idempotencyKey: `code-agent:${traceId}:preflight`,
|
|
metadata: codeAgentBillingMetadata(params, traceId, { stage: "preflight" })
|
|
};
|
|
const result = await client.billingPreflight(body);
|
|
if (result.ok && result.body?.allowed !== false) return { reservation: sanitizeBillingReservation(result.body) };
|
|
const status = [402, 403, 429].includes(Number(result.status)) ? Number(result.status) : 503;
|
|
const payload = {
|
|
ok: false,
|
|
accepted: false,
|
|
status: status === 402 ? "payment_required" : status === 403 ? "not_entitled" : status === 429 ? "rate_limited" : "billing_unavailable",
|
|
traceId,
|
|
error: {
|
|
code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"),
|
|
layer: "billing",
|
|
retryable: status === 429 || status === 503,
|
|
message: result.error?.message ?? "Code Agent billing preflight failed",
|
|
route: "/v1/agent/chat"
|
|
},
|
|
blocker: {
|
|
code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"),
|
|
layer: "billing",
|
|
retryable: status === 429 || status === 503,
|
|
summary: result.error?.message ?? "Code Agent billing preflight failed"
|
|
},
|
|
valuesRedacted: true
|
|
};
|
|
if (sendResponse && response) sendJson(response, status, payload);
|
|
return { blocked: true, statusCode: status, payload, error: payload.error, blocker: payload.blocker };
|
|
}
|
|
|
|
async function recordCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) {
|
|
const reservation = params.userBillingReservation ?? payload.userBillingReservation;
|
|
const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : "";
|
|
const client = options.userBillingClient;
|
|
if (payload.billing?.recorded === true || payload.billing?.released === true || payload.status === "running") return payload.billing ?? null;
|
|
if (!reservationId || !client?.configured) return null;
|
|
const traceId = safeTraceId(payload.traceId ?? params.traceId);
|
|
const usedTokens = codeAgentUsedTokens(payload);
|
|
const usedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_USED_CREDITS, 1);
|
|
const result = await client.billingRecord({
|
|
reservationId,
|
|
serviceId: "hwlab-code-agent",
|
|
usedCredits,
|
|
usedTokens,
|
|
idempotencyKey: `code-agent:${traceId}:record`,
|
|
metadata: codeAgentBillingMetadata(params, traceId, { stage: "record", status: payload.status ?? "unknown" })
|
|
});
|
|
const billing = result.ok ? sanitizeBillingRecord(result.body, reservation) : {
|
|
recorded: false,
|
|
reservationId,
|
|
errorCode: result.error?.code ?? "user_billing_record_failed",
|
|
valuesRedacted: true
|
|
};
|
|
payload.billing = billing;
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
if (traceId) {
|
|
traceStore.append(traceId, {
|
|
type: "billing",
|
|
status: result.ok ? "recorded" : "degraded",
|
|
label: result.ok ? "billing:recorded" : "billing:record_failed",
|
|
reservationId,
|
|
recordId: result.ok ? result.body?.recordId ?? null : null,
|
|
errorCode: result.ok ? null : billing.errorCode,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
return billing;
|
|
}
|
|
|
|
async function finalizeCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) {
|
|
if (payload.billing?.recorded === true || payload.billing?.released === true) return payload.billing;
|
|
if (payload.status === "running") return null;
|
|
if (payload.status === "completed") {
|
|
return recordCodeAgentBillingUsage({ payload, params, options });
|
|
}
|
|
return releaseCodeAgentBillingReservation({ payload, params, options });
|
|
}
|
|
|
|
async function releaseCodeAgentBillingReservation({ payload = {}, params = {}, options = {} } = {}) {
|
|
const reservation = params.userBillingReservation ?? payload.userBillingReservation;
|
|
const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : "";
|
|
const client = options.userBillingClient;
|
|
if (payload.billing?.recorded === true || payload.billing?.released === true || !reservationId || !client?.configured) return payload.billing ?? null;
|
|
const traceId = safeTraceId(payload.traceId ?? params.traceId);
|
|
const releaseReason = String(payload.status ?? "not_completed").trim() || "not_completed";
|
|
const releaseBody = {
|
|
reservationId,
|
|
serviceId: "hwlab-code-agent",
|
|
idempotencyKey: `code-agent:${traceId}:release`,
|
|
reason: releaseReason,
|
|
metadata: codeAgentBillingMetadata(params, traceId, {
|
|
stage: "release",
|
|
status: payload.status ?? "unknown",
|
|
errorCode: payload.error?.code ?? payload.blocker?.code ?? null
|
|
})
|
|
};
|
|
const result = typeof client.billingRelease === "function"
|
|
? await client.billingRelease(releaseBody)
|
|
: { ok: false, error: { code: "user_billing_release_not_supported" } };
|
|
const billing = result.ok ? sanitizeBillingRelease(result.body, reservation) : {
|
|
released: false,
|
|
reservationId,
|
|
errorCode: result.error?.code ?? "user_billing_release_failed",
|
|
valuesRedacted: true
|
|
};
|
|
payload.billing = billing;
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
if (traceId) {
|
|
traceStore.append(traceId, {
|
|
type: "billing",
|
|
status: result.ok ? "released" : "degraded",
|
|
label: result.ok ? "billing:released" : "billing:release_failed",
|
|
reservationId,
|
|
reason: releaseReason,
|
|
errorCode: result.ok ? null : billing.errorCode,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
return billing;
|
|
}
|
|
|
|
function withCodeAgentBillingReservation(payload = {}, params = {}) {
|
|
if (!payload || typeof payload !== "object" || payload.userBillingReservation) return payload;
|
|
const reservation = params.userBillingReservation;
|
|
return reservation ? { ...payload, userBillingReservation: reservation } : payload;
|
|
}
|
|
|
|
function codeAgentBillingEnabled(options = {}) {
|
|
const env = options.env ?? process.env;
|
|
if (String(env.HWLAB_USER_BILLING_CODE_AGENT_ENABLED ?? "").trim() === "0") return false;
|
|
return Boolean(options.userBillingClient?.configured);
|
|
}
|
|
|
|
function codeAgentBillingMetadata(params = {}, traceId, extra = {}) {
|
|
return {
|
|
traceId,
|
|
conversationId: safeConversationId(params.conversationId) || null,
|
|
sessionId: safeSessionId(params.sessionId) || null,
|
|
projectId: textValue(params.projectId) || null,
|
|
route: "/v1/agent/chat",
|
|
resource: "code-agent-turn",
|
|
...extra,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function sanitizeBillingReservation(value = {}) {
|
|
return {
|
|
allowed: value.allowed === true,
|
|
reservationId: textValue(value.reservationId) || null,
|
|
resourceType: textValue(value.resourceType) || null,
|
|
planId: textValue(value.planId) || null,
|
|
estimatedCredits: Number.isFinite(Number(value.estimatedCredits)) ? Number(value.estimatedCredits) : null,
|
|
estimatedTokens: Number.isFinite(Number(value.estimatedTokens)) ? Number(value.estimatedTokens) : null,
|
|
expiresAt: textValue(value.expiresAt) || null,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function sanitizeBillingRecord(value = {}, reservation = {}) {
|
|
return {
|
|
recorded: true,
|
|
reservationId: textValue(reservation.reservationId) || null,
|
|
recordId: textValue(value.recordId) || null,
|
|
credits: Number.isFinite(Number(value.credits)) ? Number(value.credits) : null,
|
|
balance: Number.isFinite(Number(value.balance)) ? Number(value.balance) : null,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function sanitizeBillingRelease(value = {}, reservation = {}) {
|
|
return {
|
|
released: true,
|
|
reservationId: textValue(value.reservationId) || textValue(reservation.reservationId) || null,
|
|
releasedCredits: Number.isFinite(Number(value.releasedCredits)) ? Number(value.releasedCredits) : null,
|
|
status: textValue(value.status) || "cancelled",
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function codeAgentUsedTokens(payload = {}) {
|
|
const usage = payload.usage && typeof payload.usage === "object" ? payload.usage : null;
|
|
const traceSummary = payload.traceSummary && typeof payload.traceSummary === "object" ? payload.traceSummary : null;
|
|
const candidates = [
|
|
usage?.totalTokens,
|
|
usage?.total_tokens,
|
|
usage?.tokens,
|
|
Number(usage?.inputTokens ?? usage?.input_tokens ?? 0) + Number(usage?.outputTokens ?? usage?.output_tokens ?? 0),
|
|
traceSummary?.tokenCount,
|
|
traceSummary?.tokens
|
|
];
|
|
for (const value of candidates) {
|
|
const parsed = Number(value);
|
|
if (Number.isFinite(parsed) && parsed > 0) return Math.ceil(parsed);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
export async function handleCodeAgentChatResultHttp(request, response, url, options) {
|
|
const parts = url.pathname.split("/").filter(Boolean);
|
|
const traceId = decodeURIComponent(parts[4] ?? "");
|
|
if (!safeTraceId(traceId)) {
|
|
sendJson(response, 400, {
|
|
error: {
|
|
code: "invalid_trace_id",
|
|
message: "traceId must start with trc_ and contain only safe identifier characters"
|
|
}
|
|
});
|
|
return;
|
|
}
|
|
const context = await measureCodeAgentHttpPhase(options, "result_projection_read", () => readCodeAgentCompatProjection(traceId, options));
|
|
if (context.statusCode) {
|
|
sendJson(response, context.statusCode, context.body);
|
|
return;
|
|
}
|
|
const result = context.result;
|
|
if (result && result.status !== "running") {
|
|
sendJson(response, 200, codeAgentCompatProjectionPayload(compactCodeAgentChatResultPayload(result, options), context));
|
|
return;
|
|
}
|
|
if (context.found) {
|
|
const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: context.refreshError ?? null, options });
|
|
sendJson(response, body.terminal === true ? 200 : 202, codeAgentCompatProjectionPayload({
|
|
accepted: body.ok,
|
|
shortConnection: true,
|
|
waitingFor: body.waitingFor ?? context.trace?.waitingFor ?? "workbench-projection",
|
|
...body
|
|
}, context));
|
|
return;
|
|
}
|
|
sendJson(response, 404, {
|
|
...codeAgentCompatProjectionPayload({ traceId, status: "unknown" }, context),
|
|
error: {
|
|
code: "code_agent_result_not_found",
|
|
message: `No Code Agent result is registered for ${traceId}`
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function handleCodeAgentTurnHttp(request, response, url, options) {
|
|
const startedAt = nowMs();
|
|
const parts = url.pathname.split("/").filter(Boolean);
|
|
const traceId = decodeURIComponent(parts[3] ?? "");
|
|
if (!safeTraceId(traceId)) {
|
|
const body = {
|
|
ok: false,
|
|
status: "unknown",
|
|
running: false,
|
|
terminal: false,
|
|
error: {
|
|
code: "invalid_trace_id",
|
|
message: "traceId must start with trc_ and contain only safe identifier characters"
|
|
}
|
|
};
|
|
recordCodeAgentTurnReadMetric(options, url, 400, body, startedAt);
|
|
sendJson(response, 400, body);
|
|
return;
|
|
}
|
|
const requestOptions = codeAgentRequestScopedOptions(options);
|
|
let completed = false;
|
|
const emitTurnStatusReadSpan = (name, attributes = {}, error = null) => {
|
|
void emitCodeAgentOtelSpan(name, traceId, requestOptions.env ?? process.env, {
|
|
startTimeMs: startedAt,
|
|
status: error ? "error" : undefined,
|
|
error,
|
|
attributes: {
|
|
"http.method": "GET",
|
|
"http.route": "/v1/agent/turns/:traceId",
|
|
durationMs: nowMs() - startedAt,
|
|
...attributes
|
|
}
|
|
});
|
|
};
|
|
response.once?.("close", () => {
|
|
if (completed) return;
|
|
emitTurnStatusReadSpan("turn_status_read.client_closed", {
|
|
status: "client_closed",
|
|
phase: "response_close",
|
|
terminal: false
|
|
}, new Error("Code Agent turn status client connection closed before response completed."));
|
|
});
|
|
try {
|
|
const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "turn_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "turn"));
|
|
if (projectMismatch) {
|
|
completed = true;
|
|
recordCodeAgentTurnReadMetric(options, url, projectMismatch.statusCode, projectMismatch.body, startedAt);
|
|
emitTurnStatusReadSpan("turn_status_read", { "http.status_code": projectMismatch.statusCode, status: projectMismatch.body?.status ?? null, terminal: projectMismatch.body?.terminal ?? null, phase: "project_check" });
|
|
sendJson(response, projectMismatch.statusCode, projectMismatch.body);
|
|
return;
|
|
}
|
|
const resolved = await measureCodeAgentHttpPhase(requestOptions, "turn_resolve_status", () => resolveCodeAgentTurnStatusSnapshot(traceId, requestOptions));
|
|
completed = true;
|
|
recordCodeAgentTurnReadMetric(options, url, resolved.statusCode, resolved.body, startedAt);
|
|
emitTurnStatusReadSpan("turn_status_read", { "http.status_code": resolved.statusCode, status: resolved.body?.status ?? null, terminal: resolved.body?.terminal ?? null, phase: "resolved" });
|
|
sendJson(response, resolved.statusCode, resolved.body);
|
|
} catch (error) {
|
|
completed = true;
|
|
const body = {
|
|
ok: false,
|
|
status: "unknown",
|
|
running: false,
|
|
terminal: false,
|
|
traceId,
|
|
error: {
|
|
code: error?.code ?? "turn_status_read_failed",
|
|
message: error?.message ?? "Code Agent turn status read failed."
|
|
},
|
|
valuesRedacted: true
|
|
};
|
|
recordCodeAgentTurnReadMetric(options, url, 500, body, startedAt);
|
|
emitTurnStatusReadSpan("turn_status_read", { "http.status_code": 500, status: body.status, terminal: false, phase: "failed", errorCode: body.error.code }, error);
|
|
sendJson(response, 500, body);
|
|
}
|
|
}
|
|
|
|
async function resolveCodeAgentTurnStatusSnapshot(traceId, options) {
|
|
const context = await readCodeAgentCompatProjection(traceId, options);
|
|
if (context.statusCode) return context;
|
|
const body = await measureCodeAgentHttpPhase(options, "turn_payload", () => codeAgentTurnStatusPayload({
|
|
traceId,
|
|
result: context.result,
|
|
snapshot: context.trace,
|
|
resultPollError: null,
|
|
refreshError: context.refreshError ?? null,
|
|
options
|
|
}));
|
|
return { statusCode: body.ok ? 200 : 404, body: codeAgentCompatProjectionPayload(body, context) };
|
|
}
|
|
|
|
function codeAgentTurnStatusRefreshOptions(options = {}) {
|
|
const env = options.env ?? process.env;
|
|
const configuredBudgetMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS, DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS);
|
|
const upstreamTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, configuredBudgetMs);
|
|
const timeoutMs = Math.max(1, Math.min(configuredBudgetMs, upstreamTimeoutMs));
|
|
return {
|
|
...options,
|
|
skipAgentRunTerminalRefreshIfComplete: true,
|
|
env: {
|
|
...env,
|
|
HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: String(timeoutMs),
|
|
HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: String(timeoutMs)
|
|
}
|
|
};
|
|
}
|
|
|
|
function codeAgentRequestScopedOptions(options = {}) {
|
|
return {
|
|
...options,
|
|
codeAgentTraceSessionCache: options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : new Map()
|
|
};
|
|
}
|
|
|
|
async function measureCodeAgentHttpPhase(options, phase, callback) {
|
|
const perf = options?.backendPerformance;
|
|
return perf && typeof perf.measure === "function" ? await perf.measure(phase, callback) : await callback();
|
|
}
|
|
|
|
function forbiddenTurnSnapshot(traceId) {
|
|
return {
|
|
statusCode: 403,
|
|
body: {
|
|
ok: false,
|
|
status: "unknown",
|
|
running: false,
|
|
terminal: false,
|
|
traceId,
|
|
error: {
|
|
code: "agent_session_owner_required",
|
|
message: "Only the session owner or admin can read this Code Agent turn status"
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
async function readCodeAgentCompatProjection(traceId, options = {}) {
|
|
const readModel = createWorkbenchReadModel(options, options.actor ?? null);
|
|
const session = await readModel.getSessionByTraceId(traceId);
|
|
let result = readModel.resultForTrace(traceId) ?? projectedSessionTraceResult(session, traceId);
|
|
if (result && !canAccessOwnedResult(result, options.actor)) return forbiddenTurnSnapshot(traceId);
|
|
if (result?.ownerUserId && !readModel.canReadOwner(result.ownerUserId)) return forbiddenTurnSnapshot(traceId);
|
|
const projectionTrace = await readModel.traceSnapshot(traceId);
|
|
let trace = traceSnapshotWithTerminalEvidence(projectionTrace, result, traceId, null);
|
|
const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: null });
|
|
recordCodeAgentProjectionMetrics(options, { result, trace, projection, refreshError: null });
|
|
const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true);
|
|
return { result, session, trace, projection, found, refreshError: null };
|
|
}
|
|
|
|
function projectedSessionTraceResult(session, traceId) {
|
|
const safeId = safeTraceId(traceId);
|
|
if (!session || !safeId) return null;
|
|
const snapshot = session.session && typeof session.session === "object" ? session.session : null;
|
|
const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null;
|
|
const stored = traceResults?.[safeId] && typeof traceResults[safeId] === "object" ? traceResults[safeId] : null;
|
|
if (!stored) return null;
|
|
return {
|
|
...stored,
|
|
traceId: safeId,
|
|
conversationId: safeConversationId(stored.conversationId ?? session.conversationId) || null,
|
|
sessionId: safeSessionId(stored.sessionId ?? session.id) || null,
|
|
threadId: safeOpaqueId(stored.threadId ?? session.threadId) || null,
|
|
ownerUserId: session.ownerUserId ?? stored.ownerUserId ?? null,
|
|
ownerRole: session.ownerRole ?? stored.ownerRole ?? null,
|
|
status: stored.status ?? session.status ?? null,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function codeAgentCompatProjectionPayload(payload = {}, context = {}) {
|
|
const traceId = safeTraceId(payload.traceId ?? context.trace?.traceId) ?? null;
|
|
const projection = context.projection && typeof context.projection === "object" ? context.projection : {};
|
|
return {
|
|
...payload,
|
|
projection,
|
|
projectionStatus: projection.projectionStatus ?? null,
|
|
projectionHealth: projection.projectionHealth ?? null,
|
|
lastProjectedSeq: projection.lastProjectedSeq ?? null,
|
|
sourceRunId: projection.sourceRunId ?? null,
|
|
sourceCommandId: projection.sourceCommandId ?? null,
|
|
staleMs: projection.staleMs ?? null,
|
|
blocker: projection.blocker ?? null,
|
|
workbench: traceId ? {
|
|
turnUrl: `/v1/workbench/turns/${encodeURIComponent(traceId)}`,
|
|
traceEventsUrl: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events`
|
|
} : null,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
async function traceProjectMismatchSnapshot(traceId, url, options, resourceKind) {
|
|
const requestedProjectId = textValue(url.searchParams.get("projectId"));
|
|
if (!requestedProjectId || !options.accessController?.getAgentSessionByTraceId) return null;
|
|
const session = await getCodeAgentSessionByTraceId(traceId, options);
|
|
if (!session) return null;
|
|
if (session.ownerUserId && options.actor?.role !== "admin" && session.ownerUserId !== options.actor?.id) return forbiddenTurnSnapshot(traceId);
|
|
const actualProjectId = textValue(session.projectId);
|
|
if (!actualProjectId || actualProjectId === requestedProjectId) return null;
|
|
return {
|
|
statusCode: 404,
|
|
body: {
|
|
ok: false,
|
|
status: "not_found",
|
|
running: false,
|
|
terminal: false,
|
|
traceId,
|
|
error: {
|
|
code: "trace_project_mismatch",
|
|
message: `${resourceKind === "turn" ? "Agent turn" : "Agent trace"} is not visible in the requested project`
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
async function getCodeAgentSessionByTraceId(traceId, options = {}) {
|
|
const safeId = safeTraceId(traceId);
|
|
if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null;
|
|
const cache = options.codeAgentTraceSessionCache;
|
|
if (cache instanceof Map) {
|
|
if (cache.has(safeId)) return cache.get(safeId);
|
|
const session = await options.accessController.getAgentSessionByTraceId(safeId);
|
|
cache.set(safeId, session ?? null);
|
|
return session ?? null;
|
|
}
|
|
return await options.accessController.getAgentSessionByTraceId(safeId);
|
|
}
|
|
|
|
function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError, refreshError, options }) {
|
|
const resultObject = result && typeof result === "object" ? result : null;
|
|
const snapshotObject = snapshot && typeof snapshot === "object" ? snapshot : null;
|
|
const lifecycle = codeAgentTurnLifecycleFields(traceId, resultObject ?? snapshotObject ?? {});
|
|
const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : [];
|
|
const lastEvent = events.at(-1) ?? null;
|
|
const finalResponse = resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId);
|
|
const terminalStatus = codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId);
|
|
const status = normalizeTurnStatus(
|
|
terminalStatus,
|
|
resultObject?.agentRun?.commandState,
|
|
resultObject?.agentRun?.status,
|
|
resultObject?.agentRun?.runStatus,
|
|
codeAgentRunningStatus(snapshotObject?.status),
|
|
codeAgentRunningStatus(snapshotObject?.traceStatus),
|
|
codeAgentRunningStatus(snapshotObject?.runnerTrace?.status),
|
|
codeAgentRunningStatus(resultObject?.status),
|
|
resultObject?.agentRun?.terminalStatus,
|
|
snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus
|
|
);
|
|
const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true);
|
|
const running = isTurnRunningStatus(status);
|
|
const terminal = isTurnTerminalStatus(status);
|
|
const runnerTrace = snapshotObject && snapshotObject.status !== "missing" ? snapshotObject : resultObject?.runnerTrace ?? null;
|
|
const timingFields = codeAgentResultTimingFields(resultObject, snapshotObject, runnerTrace);
|
|
return {
|
|
ok: found,
|
|
action: "code-agent.turn.status",
|
|
contractVersion: "code-agent-turn-status-v1",
|
|
status: found ? status ?? "unknown" : "unknown",
|
|
running: found ? running : false,
|
|
terminal: found ? terminal : false,
|
|
traceId,
|
|
...lifecycle,
|
|
conversationId: safeConversationId(resultObject?.conversationId ?? snapshotObject?.conversationId) || null,
|
|
sessionId: safeSessionId(resultObject?.sessionId ?? resultObject?.session?.sessionId ?? snapshotObject?.sessionId) || null,
|
|
threadId: safeOpaqueId(resultObject?.threadId ?? resultObject?.session?.threadId ?? snapshotObject?.threadId) || null,
|
|
updatedAt: textValue(resultObject?.updatedAt ?? resultObject?.agentRun?.updatedAt ?? snapshotObject?.updatedAt ?? timingFields.lastEventAt ?? timingFields.startedAt) || null,
|
|
...timingFields,
|
|
lastEventLabel: textValue(snapshotObject?.lastEventLabel ?? runnerTrace?.lastEventLabel ?? lastEvent?.label ?? lastEvent?.type) || null,
|
|
waitingFor: textValue(snapshotObject?.waitingFor ?? runnerTrace?.waitingFor) || null,
|
|
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
|
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
|
turnUrl: `/v1/agent/turns/${encodeURIComponent(traceId)}`,
|
|
runnerTrace: runnerTrace ? compactRunnerTraceForResult(runnerTrace, resultTraceEventLimit(options)) : null,
|
|
agentRun: resultObject?.agentRun ?? snapshotObject?.agentRun ?? null,
|
|
terminalEvidence: snapshotObject?.terminalEvidence ?? null,
|
|
finalResponse,
|
|
traceSummary: resultObject?.traceSummary ?? snapshotObject?.traceSummary ?? snapshotObject?.terminalEvidence?.traceSummary ?? null,
|
|
retention: snapshotObject?.retention ?? null,
|
|
eventCount: numberOrNull(snapshotObject?.eventCount ?? runnerTrace?.eventCount ?? events.length),
|
|
error: resultPollError || refreshError
|
|
? codeAgentRefreshErrorPayload(resultPollError ?? refreshError, traceId, resultObject?.agentRun ?? snapshotObject?.agentRun, "turn_status_degraded")
|
|
: resultObject?.error ?? snapshotObject?.error ?? null,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
function codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId) {
|
|
const sealedPayload = codeAgentPayloadHasSealedFinalResponse(resultObject ?? {}) ? resultObject : codeAgentPayloadHasSealedFinalResponse(snapshotObject ?? {}) ? snapshotObject : null;
|
|
if (sealedPayload) return "completed";
|
|
const snapshotStatus = normalizeTurnStatus(snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, snapshotObject?.terminalEvidence?.agentRun?.terminalStatus, snapshotObject?.terminalEvidence?.status, snapshotObject?.status);
|
|
if (snapshotStatus && isTurnTerminalStatus(snapshotStatus) && codeAgentSnapshotHasTerminalAuthority(snapshotObject)) return snapshotStatus;
|
|
const agentRunStatus = normalizeTurnStatus(resultObject?.agentRun?.terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, resultObject?.agentRun?.runStatus);
|
|
if (agentRunStatus && isTurnTerminalStatus(agentRunStatus)) return agentRunStatus;
|
|
const resultStatus = normalizeTurnStatus(resultObject?.status);
|
|
if (resultStatus && isTurnTerminalStatus(resultStatus) && codeAgentResultHasTerminalAuthority(resultObject, traceId)) return resultStatus;
|
|
return null;
|
|
}
|
|
|
|
function codeAgentSnapshotHasTerminalAuthority(snapshot = null) {
|
|
if (!snapshot || typeof snapshot !== "object") return false;
|
|
if (snapshot.terminal === true || snapshot.sealed === true) return true;
|
|
if (snapshot.terminalEvidence?.available === true || snapshot.terminalEvidence?.source) return true;
|
|
const events = Array.isArray(snapshot.events) ? snapshot.events : [];
|
|
return events.some((event) => event?.terminal === true);
|
|
}
|
|
|
|
function codeAgentResultHasTerminalAuthority(result = null, traceId = null) {
|
|
if (!result || typeof result !== "object") return false;
|
|
if (result.terminal === true || result.sealed === true) return true;
|
|
if (result.error || result.blocker) return true;
|
|
if (codeAgentPayloadHasSealedFinalResponse(result)) return true;
|
|
if (agentRunTerminalTraceEvidence(result, traceId)) return true;
|
|
return Boolean(textValue(result.finishedAt ?? result.completedAt ?? result.endedAt));
|
|
}
|
|
|
|
function codeAgentRefreshErrorPayload(error, traceId, agentRun, fallbackCode) {
|
|
return {
|
|
code: error?.code ?? fallbackCode,
|
|
layer: error?.layer ?? "agentrun",
|
|
category: error?.category ?? (error?.code === "agentrun_timeout" ? "upstream-timeout" : "upstream-refresh-failed"),
|
|
retryable: error?.retryable !== false,
|
|
message: error?.message ?? "Code Agent turn status refresh degraded",
|
|
traceId,
|
|
runId: agentRun?.runId ?? error?.runId ?? null,
|
|
commandId: agentRun?.commandId ?? error?.commandId ?? null,
|
|
method: error?.method ?? null,
|
|
route: error?.route ?? error?.path ?? null,
|
|
timeoutMs: numberOrNull(error?.timeoutMs),
|
|
timeoutStage: error?.timeoutStage ?? null,
|
|
upstreamStatus: numberOrNull(error?.statusCode),
|
|
managerHost: error?.managerHost ?? null,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function normalizeTurnStatus(...values) {
|
|
for (const value of values) {
|
|
const text = textValue(value).toLowerCase().replace(/_/gu, "-");
|
|
if (!text) continue;
|
|
if (["accepted", "pending", "processing", "running", "busy", "creating", "queued", "in-flight"].includes(text)) return "running";
|
|
if (["completed", "done", "succeeded", "success"].includes(text)) return "completed";
|
|
if (["failed", "failure", "error", "stale", "thread-resume-failed", "aborted", "interrupted", "expired"].includes(text)) return "failed";
|
|
if (["cancelled", "canceled"].includes(text)) return "canceled";
|
|
if (["blocked", "timeout"].includes(text)) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function isTurnRunningStatus(status) {
|
|
return status === "running";
|
|
}
|
|
|
|
function codeAgentRunningStatus(value) {
|
|
const status = normalizeTurnStatus(value);
|
|
return isTurnRunningStatus(status) ? status : null;
|
|
}
|
|
|
|
function isTurnTerminalStatus(status) {
|
|
return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"));
|
|
}
|
|
|
|
export async function handleCodeAgentInspectHttp(request, response, url, options) {
|
|
const query = {
|
|
conversationId: safeConversationId(url.searchParams.get("conversationId")),
|
|
sessionId: safeSessionId(url.searchParams.get("sessionId")),
|
|
threadId: safeOpaqueId(url.searchParams.get("threadId")),
|
|
traceId: safeTraceId(url.searchParams.get("traceId"))
|
|
};
|
|
const sessionRegistry = options.sessionRegistry;
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
const manager = options.codexStdioManager;
|
|
const managerDescription = manager && typeof manager.describe === "function" ? manager.describe() : null;
|
|
const managerRecentSessions = Array.isArray(managerDescription?.recentSessions) ? managerDescription.recentSessions : [];
|
|
const directManagerSession = query.sessionId && manager && typeof manager.get === "function"
|
|
? manager.get(query.sessionId, { conversationId: query.conversationId })
|
|
: null;
|
|
const matchedManagerSession = directManagerSession ?? managerRecentSessions.find((session) => sessionMatchesInspectQuery(session, query)) ?? null;
|
|
const registryInspect = sessionRegistry && typeof sessionRegistry.inspect === "function"
|
|
? sessionRegistry.inspect(query)
|
|
: { ok: false, status: "unavailable", traceIds: [], session: null, conversationFacts: null };
|
|
const resultEvidence = await codeAgentInspectResultEvidence(query.traceId, options);
|
|
const session = matchedManagerSession ?? registryInspect.session ?? resultEvidence.session ?? null;
|
|
const conversationFacts = registryInspect.conversationFacts ?? resultEvidence.conversationFacts ?? null;
|
|
const requestedRunnerTrace = query.traceId ? traceStore.snapshot(query.traceId) : null;
|
|
const requestedTraceFound = Boolean(requestedRunnerTrace && requestedRunnerTrace.status !== "missing");
|
|
const traceIds = uniqueStrings([
|
|
requestedTraceFound ? query.traceId : null,
|
|
session?.currentTraceId,
|
|
session?.lastTraceId,
|
|
conversationFacts?.latestTraceId,
|
|
resultEvidence.latestTraceId,
|
|
...(Array.isArray(conversationFacts?.traceIds) ? conversationFacts.traceIds : []),
|
|
...(Array.isArray(registryInspect.traceIds) ? registryInspect.traceIds : [])
|
|
]);
|
|
const latestTraceId = traceIds[0] ?? null;
|
|
const runnerTrace = latestTraceId
|
|
? latestTraceId === query.traceId && requestedRunnerTrace
|
|
? requestedRunnerTrace
|
|
: traceStore.snapshot(latestTraceId)
|
|
: null;
|
|
const found = Boolean(registryInspect.ok || session || latestTraceId);
|
|
sendJson(response, found ? 200 : 404, {
|
|
ok: found,
|
|
action: "code-agent.chat.inspect",
|
|
status: found ? "found" : "not_found",
|
|
query,
|
|
session,
|
|
conversationFacts,
|
|
traceIds,
|
|
latestTraceId,
|
|
traceUrl: latestTraceId ? `/v1/agent/traces/${encodeURIComponent(latestTraceId)}` : null,
|
|
resultUrl: latestTraceId ? `/v1/agent/chat/result/${encodeURIComponent(latestTraceId)}` : null,
|
|
runnerTrace,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
});
|
|
}
|
|
|
|
async function codeAgentInspectResultEvidence(traceId, options = {}) {
|
|
if (!safeTraceId(traceId)) return { session: null, conversationFacts: null, latestTraceId: null };
|
|
const cached = options.codeAgentChatResults?.get?.(traceId) ?? null;
|
|
const persisted = cached ? null : await loadPersistedAgentRunResult(traceId, options);
|
|
const result = cached ?? persisted ?? null;
|
|
if (!result || typeof result !== "object") return { session: null, conversationFacts: null, latestTraceId: null };
|
|
const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null;
|
|
const resultSession = result.session && typeof result.session === "object" ? result.session : null;
|
|
const sessionReuse = result.sessionReuse && typeof result.sessionReuse === "object" ? result.sessionReuse : null;
|
|
const providerTrace = result.providerTrace && typeof result.providerTrace === "object" ? result.providerTrace : null;
|
|
const sessionId = safeSessionId(result.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? agentRun?.sessionId) || null;
|
|
const conversationId = safeConversationId(result.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? agentRun?.conversationId) || null;
|
|
const threadId = safeOpaqueId(result.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? providerTrace?.threadId ?? agentRun?.threadId) || null;
|
|
const status = textValue(resultSession?.status ?? result.status ?? agentRun?.status) || null;
|
|
const updatedAt = textValue(result.updatedAt ?? agentRun?.updatedAt ?? resultSession?.updatedAt) || null;
|
|
const session = sessionId || conversationId || threadId ? {
|
|
sessionId,
|
|
conversationId,
|
|
threadId,
|
|
status,
|
|
lastTraceId: traceId,
|
|
source: "code-agent-result",
|
|
agentRun: agentRun ? agentRunSessionEvidence(result).agentRun : null,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
} : null;
|
|
const conversationFacts = conversationId ? {
|
|
conversationId,
|
|
sessionId,
|
|
threadId,
|
|
latestTraceId: traceId,
|
|
traceIds: [traceId],
|
|
turnCount: 1,
|
|
source: "code-agent-result",
|
|
latestStatus: status,
|
|
updatedAt,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
} : null;
|
|
return { session, conversationFacts, latestTraceId: traceId };
|
|
}
|
|
|
|
function sessionMatchesInspectQuery(session, query) {
|
|
if (!session || typeof session !== "object") return false;
|
|
if (query.sessionId && session.sessionId === query.sessionId) return true;
|
|
if (query.threadId && session.threadId === query.threadId) return true;
|
|
if (query.conversationId && session.conversationId === query.conversationId) return true;
|
|
if (query.traceId && (session.currentTraceId === query.traceId || session.lastTraceId === query.traceId)) return true;
|
|
return false;
|
|
}
|
|
|
|
export async function handleCodeAgentCancelHttp(request, response, options) {
|
|
const body = await readBody(request, options.bodyLimitBytes);
|
|
let params = {};
|
|
|
|
try {
|
|
params = body ? JSON.parse(body) : {};
|
|
} catch (error) {
|
|
sendJson(response, 400, {
|
|
...createCodeAgentErrorPayload({
|
|
code: "parse_error",
|
|
message: "Invalid JSON body",
|
|
reason: error.message,
|
|
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
|
layer: "api",
|
|
retryable: true
|
|
})
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (!params || typeof params !== "object" || Array.isArray(params)) {
|
|
sendJson(response, 400, {
|
|
...createCodeAgentErrorPayload({
|
|
code: "invalid_params",
|
|
message: "Code Agent cancel body must be a JSON object",
|
|
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
|
layer: "api",
|
|
retryable: true
|
|
})
|
|
});
|
|
return;
|
|
}
|
|
|
|
const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId);
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
const snapshot = traceId ? traceStore.snapshot(traceId) : null;
|
|
const currentResult = traceId ? options.codeAgentChatResults?.get(traceId) ?? await loadPersistedAgentRunResult(traceId, options) : null;
|
|
if (currentResult?.agentRun?.commandId) {
|
|
const mismatch = codeAgentCancelScopeMismatch(params, currentResult);
|
|
if (mismatch) {
|
|
traceStore.append(traceId, {
|
|
type: "cancel",
|
|
status: "blocked",
|
|
label: "agentrun:cancel:scope_mismatch",
|
|
errorCode: "cancel_scope_mismatch",
|
|
message: `Cancel request ${mismatch.field} does not match the trace owner; AgentRun cancel was not forwarded.`,
|
|
requested: mismatch.requested,
|
|
expected: mismatch.expected,
|
|
waitingFor: "cancel-scope",
|
|
valuesPrinted: false
|
|
});
|
|
sendJson(response, 409, cancelBlockedPayload({
|
|
code: "cancel_scope_mismatch",
|
|
message: `取消请求的 ${mismatch.field} 与 trace 归属不一致，已拒绝转发 AgentRun cancel，避免误取消其他 session。`,
|
|
traceId,
|
|
conversationId: safeConversationId(params.conversationId) || safeConversationId(currentResult.conversationId) || null,
|
|
sessionId: safeSessionId(params.sessionId) || safeSessionId(currentResult.sessionId) || null,
|
|
runnerTrace: traceStore.snapshot(traceId),
|
|
status: "blocked"
|
|
}));
|
|
return;
|
|
}
|
|
await recordCodeAgentSessionInputFact({
|
|
params: {
|
|
...params,
|
|
sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId,
|
|
messageId: codeAgentTurnLifecycleFields(traceId, currentResult).userMessageId,
|
|
ownerUserId: options.actor?.id,
|
|
ownerRole: options.actor?.role
|
|
},
|
|
options,
|
|
traceId,
|
|
delivery: "cancel",
|
|
status: "promoted",
|
|
commandId: currentResult.agentRun.commandId
|
|
});
|
|
const payload = await cancelAgentRunChatTurn({ traceId, currentResult, options, traceStore });
|
|
if (payload) {
|
|
if (payload.alreadyTerminal === true || isTraceCommandTerminalStatus(payload.status)) {
|
|
await recordCodeAgentTerminalTurnStatusEffects({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, preserveLastTraceId: true });
|
|
} else {
|
|
await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" });
|
|
}
|
|
sendJson(response, 200, payload);
|
|
return;
|
|
}
|
|
}
|
|
const sessionId = safeSessionId(params.sessionId) || safeSessionId(snapshot?.sessionId);
|
|
const conversationId = safeConversationId(params.conversationId);
|
|
const manager = options.codexStdioManager;
|
|
|
|
if (!traceId) {
|
|
sendJson(response, 400, cancelBlockedPayload({
|
|
code: "cancel_trace_missing",
|
|
message: "traceId is required to cancel the current Code Agent request.",
|
|
traceId: "trc_unassigned",
|
|
conversationId,
|
|
sessionId
|
|
}));
|
|
return;
|
|
}
|
|
|
|
if (!manager || typeof manager.get !== "function" || typeof manager.cancel !== "function") {
|
|
traceStore.append(traceId, {
|
|
type: "cancel",
|
|
status: "unsupported",
|
|
label: "cancel:unsupported",
|
|
errorCode: "cancel_unsupported",
|
|
message: "Codex stdio cancel/interrupt backend is not available on this runtime.",
|
|
waitingFor: "session-control"
|
|
});
|
|
const runnerTrace = traceStore.snapshot(traceId);
|
|
sendJson(response, 501, cancelBlockedPayload({
|
|
code: "cancel_unsupported",
|
|
message: "当前后端不支持 Code Agent interrupt/cancel；本次按 unsupported/degraded 返回，不会静默假装已中断。",
|
|
traceId,
|
|
conversationId,
|
|
sessionId,
|
|
runnerTrace,
|
|
status: "degraded",
|
|
unsupported: true,
|
|
degraded: true
|
|
}));
|
|
return;
|
|
}
|
|
|
|
if (!sessionId) {
|
|
traceStore.append(traceId, {
|
|
type: "cancel",
|
|
status: "blocked",
|
|
label: "cancel:not_cancelable",
|
|
errorCode: "cancel_session_missing",
|
|
message: "Cancel request did not include a bound Codex stdio sessionId.",
|
|
waitingFor: "session-binding"
|
|
});
|
|
sendJson(response, 409, cancelBlockedPayload({
|
|
code: "cancel_session_missing",
|
|
message: "当前请求尚未暴露可取消的 Codex stdio sessionId；页面不能只隐藏 UI，已保留输入和 trace。",
|
|
traceId,
|
|
conversationId,
|
|
sessionId: null,
|
|
runnerTrace: traceStore.snapshot(traceId)
|
|
}));
|
|
return;
|
|
}
|
|
|
|
const currentSession = manager.get(sessionId, { conversationId }) ?? null;
|
|
if (!currentSession || !["busy", "creating"].includes(currentSession.status)) {
|
|
traceStore.append(traceId, {
|
|
type: "cancel",
|
|
status: "blocked",
|
|
label: "cancel:not_in_flight",
|
|
errorCode: "cancel_not_in_flight",
|
|
message: `Session ${sessionId} is not an in-flight Codex stdio request.`,
|
|
sessionId,
|
|
sessionStatus: currentSession?.status ?? "missing"
|
|
});
|
|
sendJson(response, 409, cancelBlockedPayload({
|
|
code: currentSession ? "cancel_not_in_flight" : "cancel_session_not_found",
|
|
message: currentSession
|
|
? `当前 session 状态为 ${currentSession.status}，没有可取消的 in-flight Codex stdio 请求。`
|
|
: `没有找到 sessionId=${sessionId} 的 Codex stdio session。`,
|
|
traceId,
|
|
conversationId,
|
|
sessionId,
|
|
session: currentSession,
|
|
runnerTrace: traceStore.snapshot(traceId)
|
|
}));
|
|
return;
|
|
}
|
|
|
|
const canceledSession = manager.cancel(sessionId, {
|
|
traceId,
|
|
conversationId,
|
|
reason: "user_cancel"
|
|
});
|
|
traceStore.append(traceId, {
|
|
type: "cancel",
|
|
status: "canceled",
|
|
label: "cancel:canceled",
|
|
message: "User canceled the current Codex stdio request.",
|
|
sessionId,
|
|
sessionStatus: canceledSession?.status ?? "canceled",
|
|
sessionLifecycleStatus: codeAgentSessionLifecycleSummary({ session: canceledSession, status: "canceled" }).status,
|
|
waitingFor: "user-retry",
|
|
terminal: true
|
|
});
|
|
const runnerTraceSnapshot = traceStore.snapshot(traceId);
|
|
const sessionSummary = codeAgentSessionLifecycleSummary({
|
|
session: canceledSession,
|
|
runnerTrace: runnerTraceSnapshot,
|
|
status: "canceled"
|
|
});
|
|
const runnerTrace = {
|
|
...runnerTraceSnapshot,
|
|
sessionLifecycleStatus: sessionSummary.status
|
|
};
|
|
const payload = {
|
|
accepted: true,
|
|
canceled: true,
|
|
status: "canceled",
|
|
conversationId: conversationId ?? canceledSession?.conversationId ?? null,
|
|
sessionId,
|
|
traceId,
|
|
session: canceledSession,
|
|
sessionLifecycleStatus: sessionSummary.status,
|
|
sessionLifecycle: sessionSummary,
|
|
sessionSummary,
|
|
runnerTrace,
|
|
lastTraceEvent: runnerTrace.lastEvent,
|
|
retryable: true,
|
|
error: {
|
|
code: "codex_stdio_canceled",
|
|
layer: "session",
|
|
category: "canceled",
|
|
retryable: true,
|
|
message: "user canceled current Code Agent request",
|
|
userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。",
|
|
traceId,
|
|
route: "/v1/agent/chat/cancel",
|
|
toolName: "codex-stdio.cancel"
|
|
},
|
|
userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。",
|
|
updatedAt: new Date().toISOString()
|
|
};
|
|
await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" });
|
|
recordCodeAgentConversationFact(payload, options);
|
|
options.codeAgentChatResults?.set(traceId, annotateOwner(payload, { ownerUserId: options.actor?.id, ownerRole: options.actor?.role }));
|
|
sendJson(response, 200, payload);
|
|
}
|
|
|
|
function recordCodeAgentConversationFact(payload = {}, options = {}) {
|
|
const registry = options.sessionRegistry;
|
|
const conversationId = safeConversationId(payload.conversationId);
|
|
if (!conversationId || typeof registry?.recordFact !== "function") return null;
|
|
try {
|
|
return registry.recordFact(conversationId, {
|
|
kind: payload.status === "completed" ? "runner_turn" : "runner_turn_terminal",
|
|
conversationId,
|
|
sessionId: payload.sessionId,
|
|
traceId: payload.traceId,
|
|
provider: payload.provider,
|
|
backend: payload.backend,
|
|
workspace: payload.workspace,
|
|
sandbox: payload.sandbox,
|
|
session: payload.session,
|
|
sessionMode: payload.sessionMode,
|
|
sessionReuse: payload.sessionReuse,
|
|
implementationType: payload.implementationType,
|
|
runner: payload.runner,
|
|
runnerTrace: payload.runnerTrace,
|
|
partialContext: payload.partialContext ?? payload.runnerTrace?.partialContext,
|
|
capabilityLevel: payload.capabilityLevel,
|
|
toolCalls: payload.toolCalls,
|
|
skills: payload.skills
|
|
}, { now: options.now });
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function recordCodeAgentTerminalTurnStatusEffects({ payload = {}, params = {}, options = {}, preserveLastTraceId = false } = {}) {
|
|
payload = codeAgentPayloadWithObservedTerminalStatus(payload, payload?.runnerTrace) ?? payload;
|
|
if (!payload || typeof payload !== "object" || !isTraceCommandTerminalStatus(payload.status)) return null;
|
|
if (payload.turnStatusTerminalEffects?.recorded === true) return payload.turnStatusTerminalEffects;
|
|
const billing = await finalizeCodeAgentBillingUsage({ payload, params, options });
|
|
recordCodeAgentConversationFact(payload, options);
|
|
const owner = await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload), preserveLastTraceId });
|
|
const billingSettled = codeAgentTerminalBillingSettled({ payload, params, options, billing });
|
|
const ownerRequired = Boolean(options.actor?.id && options.accessController?.recordAgentSessionOwner);
|
|
const ownerSettled = !ownerRequired || Boolean(owner);
|
|
const effects = {
|
|
recorded: billingSettled && ownerSettled,
|
|
pending: false,
|
|
billingSettled,
|
|
ownerSettled,
|
|
preserveLastTraceId: Boolean(preserveLastTraceId),
|
|
recordedAt: new Date().toISOString(),
|
|
valuesPrinted: false
|
|
};
|
|
payload.turnStatusTerminalEffects = effects;
|
|
const traceId = safeTraceId(payload.traceId ?? params.traceId);
|
|
if (traceId) void emitCodeAgentOtelSpan("projection_write", traceId, options.env ?? process.env, {
|
|
attributes: {
|
|
projection: "terminal-effects",
|
|
status: payload.status,
|
|
terminal: true,
|
|
billingSettled,
|
|
ownerSettled,
|
|
sessionId: safeSessionId(payload.sessionId ?? params.sessionId) || null,
|
|
turnId: codeAgentTurnLifecycleFields(traceId, payload).turnId,
|
|
runId: payload.agentRun?.runId ?? null,
|
|
commandId: payload.agentRun?.commandId ?? null
|
|
}
|
|
});
|
|
if (traceId) options.codeAgentChatResults?.set?.(traceId, payload);
|
|
return effects;
|
|
}
|
|
|
|
function scheduleCodeAgentTerminalTurnStatusEffects({ payload = {}, params = {}, options = {}, preserveLastTraceId = false } = {}) {
|
|
payload = codeAgentPayloadWithObservedTerminalStatus(payload, payload?.runnerTrace) ?? payload;
|
|
if (!payload || typeof payload !== "object" || !isTraceCommandTerminalStatus(payload.status)) return null;
|
|
const existing = payload.turnStatusTerminalEffects;
|
|
if (existing?.recorded === true || existing?.pending === true) return existing;
|
|
const traceId = safeTraceId(payload.traceId ?? params.traceId);
|
|
const pending = {
|
|
recorded: false,
|
|
pending: true,
|
|
billingSettled: false,
|
|
ownerSettled: false,
|
|
preserveLastTraceId: Boolean(preserveLastTraceId),
|
|
scheduledAt: new Date().toISOString(),
|
|
valuesPrinted: false
|
|
};
|
|
payload.turnStatusTerminalEffects = pending;
|
|
if (traceId) options.codeAgentChatResults?.set?.(traceId, payload);
|
|
setImmediate(() => {
|
|
void recordCodeAgentTerminalTurnStatusEffects({ payload, params, options, preserveLastTraceId }).catch((error) => {
|
|
const failed = {
|
|
recorded: false,
|
|
pending: false,
|
|
billingSettled: false,
|
|
ownerSettled: false,
|
|
preserveLastTraceId: Boolean(preserveLastTraceId),
|
|
errorCode: error?.code ?? "terminal_turn_status_effects_failed",
|
|
recordedAt: new Date().toISOString(),
|
|
valuesPrinted: false
|
|
};
|
|
payload.turnStatusTerminalEffects = failed;
|
|
if (traceId) {
|
|
options.codeAgentChatResults?.set?.(traceId, payload);
|
|
(options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, {
|
|
type: "turn-status",
|
|
status: "degraded",
|
|
label: "turn-status:terminal-effects-failed",
|
|
errorCode: failed.errorCode,
|
|
message: error?.message ?? "Terminal turn status side effects failed and will retry on the next poll.",
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
});
|
|
});
|
|
return pending;
|
|
}
|
|
|
|
function codeAgentTerminalBillingSettled({ payload = {}, params = {}, options = {}, billing = null } = {}) {
|
|
const reservation = params.userBillingReservation ?? payload.userBillingReservation;
|
|
const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : "";
|
|
if (!reservationId || !options.userBillingClient?.configured) return true;
|
|
return billing?.recorded === true || billing?.released === true || payload.billing?.recorded === true || payload.billing?.released === true;
|
|
}
|
|
|
|
export async function handleCodeAgentSteerHttp(request, response, options) {
|
|
const body = await readBody(request, options.bodyLimitBytes);
|
|
let params = {};
|
|
|
|
try {
|
|
params = body ? JSON.parse(body) : {};
|
|
} catch (error) {
|
|
sendJson(response, 400, {
|
|
...createCodeAgentErrorPayload({
|
|
code: "parse_error",
|
|
message: "Invalid JSON body",
|
|
reason: error.message,
|
|
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
|
layer: "api",
|
|
retryable: true,
|
|
route: "/v1/agent/chat/steer"
|
|
})
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (!params || typeof params !== "object" || Array.isArray(params)) {
|
|
sendJson(response, 400, {
|
|
...createCodeAgentErrorPayload({
|
|
code: "invalid_params",
|
|
message: "Code Agent steer body must be a JSON object",
|
|
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
|
layer: "api",
|
|
retryable: true,
|
|
route: "/v1/agent/chat/steer"
|
|
})
|
|
});
|
|
return;
|
|
}
|
|
|
|
const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId || params.targetTraceId);
|
|
const message = firstNonEmptyValue(params.message, params.prompt, params.text);
|
|
if (!traceId || !message) {
|
|
sendJson(response, 400, {
|
|
...createCodeAgentErrorPayload({
|
|
code: !traceId ? "steer_trace_missing" : "steer_message_missing",
|
|
message: !traceId ? "traceId is required to steer the current Code Agent turn." : "message is required to steer the current Code Agent turn.",
|
|
traceId: traceId || "trc_unassigned",
|
|
layer: "api",
|
|
retryable: true,
|
|
route: "/v1/agent/chat/steer"
|
|
})
|
|
});
|
|
return;
|
|
}
|
|
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
const currentResult = options.codeAgentChatResults?.get?.(traceId) ?? await loadPersistedAgentRunResult(traceId, options);
|
|
if (!canAccessOwnedResult(currentResult, options.actor)) {
|
|
sendJson(response, 403, {
|
|
ok: false,
|
|
status: "forbidden",
|
|
traceId,
|
|
error: { code: "code_agent_trace_forbidden", message: "Code Agent trace is not visible to the current actor" }
|
|
});
|
|
return;
|
|
}
|
|
if (!currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) {
|
|
sendJson(response, 404, {
|
|
ok: false,
|
|
status: "not_found",
|
|
traceId,
|
|
error: { code: "steer_trace_not_found", message: "No AgentRun-backed Code Agent turn was found for the requested traceId." },
|
|
route: "/v1/agent/chat/steer",
|
|
runnerTrace: traceStore.snapshot(traceId)
|
|
});
|
|
return;
|
|
}
|
|
const unsealedTerminalError = traceSnapshotTerminalFailureDiagnostic(traceStore.snapshot(traceId));
|
|
if (unsealedTerminalError) {
|
|
const terminalSync = await syncAgentRunTerminalBeforeSteer({ traceId, currentResult, params, options, traceStore });
|
|
const syncedResult = terminalSync?.payload ?? currentResult;
|
|
if (terminalSync?.terminal === true) {
|
|
sendJson(response, 409, {
|
|
ok: false,
|
|
accepted: false,
|
|
status: "blocked",
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
sessionId: safeSessionId(syncedResult.sessionId ?? params.sessionId) || null,
|
|
threadId: safeOpaqueId(syncedResult.threadId ?? params.threadId) || null,
|
|
agentRun: syncedResult.agentRun ?? currentResult.agentRun,
|
|
runnerTrace: terminalSync.runnerTrace ?? traceStore.snapshot(traceId),
|
|
terminalDiagnostic: unsealedTerminalError,
|
|
error: {
|
|
code: "steer_trace_terminal",
|
|
layer: "api",
|
|
category: "not_in_flight",
|
|
retryable: false,
|
|
message: `Trace ${traceId} is already terminal and cannot accept a steer command.`,
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
toolName: "agentrun.command.steer"
|
|
},
|
|
valuesPrinted: false
|
|
});
|
|
return;
|
|
}
|
|
traceStore.append(traceId, {
|
|
type: "backend",
|
|
status: "blocked",
|
|
label: "agentrun:steer:blocked-unsealed-terminal-error",
|
|
errorCode: terminalSync?.error?.code ?? unsealedTerminalError.failureKind ?? "steer_trace_unsealed_terminal_error",
|
|
message: terminalSync?.error?.message ?? "Trace has an unsealed terminal error diagnostic; HWLAB refused to steer a new prompt into the old trace.",
|
|
waitingFor: "terminal-projection-reconciliation",
|
|
terminalDiagnostic: unsealedTerminalError,
|
|
terminal: false,
|
|
valuesPrinted: false
|
|
});
|
|
sendJson(response, terminalSync?.error ? 503 : 409, {
|
|
ok: false,
|
|
accepted: false,
|
|
status: "blocked",
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null,
|
|
threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null,
|
|
agentRun: currentResult.agentRun,
|
|
runnerTrace: traceStore.snapshot(traceId),
|
|
terminalDiagnostic: unsealedTerminalError,
|
|
error: {
|
|
code: terminalSync?.error?.code ?? "steer_trace_unsealed_terminal_error",
|
|
layer: terminalSync?.error?.layer ?? "api",
|
|
category: "not_in_flight",
|
|
retryable: false,
|
|
message: terminalSync?.error?.message ?? "Trace has an unsealed terminal error diagnostic; start a new turn instead of steering into the old trace.",
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
toolName: "agentrun.command.steer"
|
|
},
|
|
valuesPrinted: false
|
|
});
|
|
return;
|
|
}
|
|
if (isTraceCommandTerminalStatus(currentResult.status)) {
|
|
sendJson(response, 409, {
|
|
ok: false,
|
|
accepted: false,
|
|
status: "blocked",
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null,
|
|
threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null,
|
|
agentRun: currentResult.agentRun,
|
|
runnerTrace: traceStore.snapshot(traceId),
|
|
error: {
|
|
code: "steer_trace_terminal",
|
|
layer: "api",
|
|
category: "not_in_flight",
|
|
retryable: false,
|
|
message: `Trace ${traceId} is already terminal and cannot accept a steer command.`,
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
toolName: "agentrun.command.steer"
|
|
},
|
|
valuesPrinted: false
|
|
});
|
|
return;
|
|
}
|
|
|
|
try {
|
|
await recordCodeAgentSessionInputFact({
|
|
params: {
|
|
...params,
|
|
sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId,
|
|
ownerUserId: options.actor?.id,
|
|
ownerRole: options.actor?.role
|
|
},
|
|
options,
|
|
traceId,
|
|
delivery: "steer",
|
|
status: "promoted",
|
|
commandId: currentResult.agentRun.commandId
|
|
});
|
|
const payload = await steerAgentRunChatTurn({
|
|
traceId,
|
|
currentResult,
|
|
params: {
|
|
...params,
|
|
message,
|
|
ownerUserId: options.actor?.id ?? params.ownerUserId,
|
|
ownerRole: options.actor?.role ?? params.ownerRole
|
|
},
|
|
options,
|
|
traceStore
|
|
});
|
|
sendJson(response, 202, payload);
|
|
setImmediate(() => {
|
|
void recordCodeAgentSessionOwner({ payload: currentResult, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "running" });
|
|
});
|
|
} catch (error) {
|
|
traceStore.append(traceId, {
|
|
type: "backend",
|
|
status: "failed",
|
|
label: "agentrun:steer:failed",
|
|
errorCode: error?.code ?? "agentrun_steer_failed",
|
|
message: error?.message ?? "AgentRun steer command failed.",
|
|
terminal: false,
|
|
valuesPrinted: false
|
|
});
|
|
sendJson(response, error?.statusCode === 404 ? 404 : 409, {
|
|
ok: false,
|
|
accepted: false,
|
|
status: "failed",
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null,
|
|
threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null,
|
|
agentRun: currentResult.agentRun,
|
|
runnerTrace: traceStore.snapshot(traceId),
|
|
error: {
|
|
code: error?.code ?? "agentrun_steer_failed",
|
|
layer: "agentrun",
|
|
category: "steer_failed",
|
|
retryable: true,
|
|
message: error?.message ?? "AgentRun steer command failed.",
|
|
traceId,
|
|
route: "/v1/agent/chat/steer",
|
|
toolName: "agentrun.command.steer"
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
async function syncAgentRunTerminalBeforeSteer({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) {
|
|
try {
|
|
const synced = await syncAgentRunChatResult({
|
|
traceId,
|
|
currentResult,
|
|
options: { ...options, deferAgentRunResultSync: false },
|
|
traceStore,
|
|
forceResultSync: true,
|
|
refreshEvents: true
|
|
});
|
|
const payload = codeAgentPayloadWithObservedTerminalStatus(synced?.result ?? null, synced?.runnerTrace ?? traceStore.snapshot(traceId)) ?? synced?.result ?? currentResult;
|
|
if (isTraceCommandTerminalStatus(payload?.status)) {
|
|
await recordCodeAgentTerminalTurnStatusEffects({
|
|
payload,
|
|
params: { ...params, traceId, ownerUserId: options.actor?.id ?? params.ownerUserId, ownerRole: options.actor?.role ?? params.ownerRole },
|
|
options,
|
|
preserveLastTraceId: true
|
|
});
|
|
return { terminal: true, payload, runnerTrace: synced?.runnerTrace ?? payload?.runnerTrace ?? traceStore.snapshot(traceId), valuesPrinted: false };
|
|
}
|
|
return { terminal: false, payload, runnerTrace: synced?.runnerTrace ?? payload?.runnerTrace ?? traceStore.snapshot(traceId), valuesPrinted: false };
|
|
} catch (error) {
|
|
return { terminal: false, error, runnerTrace: traceStore.snapshot(traceId), valuesPrinted: false };
|
|
}
|
|
}
|
|
|
|
function traceSnapshotTerminalFailureDiagnostic(snapshot = null) {
|
|
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
|
for (const event of [...events].reverse()) {
|
|
const payload = event?.payload && typeof event.payload === "object" ? event.payload : {};
|
|
const label = textValue(event?.label ?? event?.type);
|
|
const status = normalizeTurnStatus(event?.status ?? payload.status);
|
|
const failureKind = textValue(event?.failureKind ?? event?.errorCode ?? payload.failureKind ?? payload.errorCode);
|
|
const retrying = event?.willRetry === true || payload.willRetry === true;
|
|
const retryExhausted = event?.retryExhausted === true || payload.retryExhausted === true;
|
|
const explicitlyNonRetryable = event?.retryable === false || payload.retryable === false;
|
|
const hardFailure = retryExhausted || explicitlyNonRetryable || failureKind === "backend-timeout" || /^agentrun:error:/u.test(label);
|
|
if (retrying || !hardFailure || (status !== "failed" && !failureKind)) continue;
|
|
return {
|
|
label: label || null,
|
|
status: status || null,
|
|
failureKind: failureKind || null,
|
|
retryExhausted,
|
|
message: textValue(event?.message ?? payload.message) || null,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function recordCodeAgentSessionOwner({ payload = {}, params = {}, options = {}, status = "active", preserveLastTraceId = false } = {}) {
|
|
const ownerUserId = options.actor?.id ?? params.ownerUserId;
|
|
if (!ownerUserId || !options.accessController?.recordAgentSessionOwner) return null;
|
|
const traceId = safeTraceId(payload.traceId ?? params.traceId);
|
|
const session = payload.session && typeof payload.session === "object" ? payload.session : null;
|
|
const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null;
|
|
const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null;
|
|
const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null;
|
|
const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null;
|
|
return writeWorkbenchProjectionSession({
|
|
accessController: options.accessController,
|
|
runtimeStore: options.runtimeStore,
|
|
traceStore: options.traceStore ?? defaultCodeAgentTraceStore,
|
|
traceId,
|
|
ownerUserId,
|
|
ownerRole: options.actor?.role ?? params.ownerRole ?? null,
|
|
sessionId,
|
|
projectId: params.projectId ?? payload.projectId ?? null,
|
|
conversationId,
|
|
threadId,
|
|
status,
|
|
preserveLastTraceId,
|
|
session: codeAgentSessionOwnerEvidence(payload, params),
|
|
payload,
|
|
params
|
|
});
|
|
}
|
|
|
|
function codeAgentOwnerStatusForResult(result = {}) {
|
|
const sessionStatus = textValue(result?.session?.status ?? result?.sessionSummary?.status ?? result?.sessionLifecycleStatus);
|
|
if (sessionStatus === "thread-resume-failed") return "thread-resume-failed";
|
|
if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed";
|
|
if (result?.status === "completed") return "completed";
|
|
if (result?.status === "canceled" || result?.status === "cancelled") return "canceled";
|
|
return result?.status ?? "active";
|
|
}
|
|
|
|
function textValue(value) {
|
|
return String(value ?? "").trim();
|
|
}
|
|
|
|
function timestampIsoOrNow(value = null) {
|
|
const parsed = Date.parse(String(value ?? ""));
|
|
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString();
|
|
}
|
|
|
|
function timestampIsoOrNull(value = null) {
|
|
const parsed = Date.parse(String(value ?? ""));
|
|
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
|
}
|
|
|
|
function codeAgentSessionOwnerEvidence(payload = {}, params = {}) {
|
|
const traceId = payload.traceId ?? params.traceId ?? null;
|
|
const finalResponse = codeAgentFinalResponseEvidence(payload, traceId);
|
|
const traceSummary = codeAgentTraceSummaryEvidence(payload, traceId, finalResponse);
|
|
const messages = codeAgentConversationMessagesEvidence(payload, params, traceId, finalResponse);
|
|
const firstUserMessagePreview = messages.find((message) => message.role === "user")?.text?.slice(0, 240) ?? null;
|
|
const promptEvidence = codeAgentPromptMetadata(payload, params, traceId);
|
|
const traceResult = codeAgentTraceResultEvidence(payload, params, traceId, finalResponse, traceSummary);
|
|
return {
|
|
provider: payload.provider ?? null,
|
|
model: payload.model ?? null,
|
|
backend: payload.backend ?? null,
|
|
status: payload.status ?? null,
|
|
sessionMode: payload.sessionMode ?? payload.session?.sessionMode ?? null,
|
|
sessionLifecycleStatus: payload.sessionLifecycleStatus ?? null,
|
|
runnerKind: payload.runner?.kind ?? payload.runnerTrace?.runnerKind ?? null,
|
|
conversationId: payload.conversationId ?? params.conversationId ?? null,
|
|
sessionId: payload.sessionId ?? payload.session?.sessionId ?? payload.sessionReuse?.sessionId ?? params.sessionId ?? null,
|
|
threadId: payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? null,
|
|
traceId,
|
|
projectId: payload.projectId ?? params.projectId ?? null,
|
|
...codeAgentPromptFields(promptEvidence),
|
|
finalResponse,
|
|
traceSummary,
|
|
...(traceResult ? { traceResults: { [traceResult.traceId]: traceResult } } : {}),
|
|
...(messages.length ? { messages, messageCount: messages.length } : {}),
|
|
...(firstUserMessagePreview ? { firstUserMessagePreview } : {}),
|
|
...agentRunSessionEvidence(payload),
|
|
secretMaterialStored: false,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function codeAgentTraceResultEvidence(payload = {}, params = {}, traceId = null, finalResponse = null, traceSummary = null) {
|
|
const resolvedTraceId = safeTraceId(traceId);
|
|
if (!resolvedTraceId) return null;
|
|
const agentRun = agentRunSessionEvidence(payload).agentRun ?? null;
|
|
const userBillingReservation = codeAgentBillingReservationEvidence(payload.userBillingReservation ?? params.userBillingReservation);
|
|
const resultSession = payload.session && typeof payload.session === "object" ? payload.session : null;
|
|
const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null;
|
|
const conversationId = safeConversationId(payload.conversationId ?? agentRun?.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null;
|
|
const sessionId = safeSessionId(payload.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null;
|
|
const threadId = safeOpaqueId(payload.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? params.threadId ?? agentRun?.threadId) || null;
|
|
const promptEvidence = codeAgentPromptMetadata(payload, params, resolvedTraceId);
|
|
if (!agentRun && !finalResponse && !traceSummary) return null;
|
|
return {
|
|
traceId: resolvedTraceId,
|
|
status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? null,
|
|
conversationId,
|
|
sessionId,
|
|
threadId,
|
|
...codeAgentPromptFields(promptEvidence),
|
|
messageId: finalResponse?.messageId ?? payload.messageId ?? null,
|
|
createdAt: payload.createdAt ?? finalResponse?.createdAt ?? null,
|
|
updatedAt: payload.updatedAt ?? finalResponse?.updatedAt ?? null,
|
|
finalResponse,
|
|
traceSummary,
|
|
agentRun,
|
|
...(userBillingReservation ? { userBillingReservation } : {}),
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
function codeAgentPromptMetadata(payload = {}, params = {}, traceId = null) {
|
|
const prompt = payload?.prompt && typeof payload.prompt === "object" ? payload.prompt : null;
|
|
const rawPrompt = firstNonEmptyValue(
|
|
params.message,
|
|
params.prompt,
|
|
params.text,
|
|
payload.userMessage,
|
|
typeof payload.prompt === "string" ? payload.prompt : null
|
|
);
|
|
const fromText = codeAgentPromptMetadataFromText(rawPrompt, traceId, "code-agent-request", { fullText: Boolean(rawPrompt) });
|
|
return codeAgentPromptMetadataFromParts({
|
|
traceId,
|
|
source: prompt?.source ?? payload.promptSource ?? fromText?.prompt?.source ?? "code-agent-request",
|
|
promptId: firstNonEmptyValue(payload.promptId, prompt?.id, fromText?.promptId),
|
|
promptSha256: firstNonEmptyValue(payload.promptSha256, prompt?.sha256, fromText?.promptSha256),
|
|
promptSummary: firstNonEmptyValue(payload.promptSummary, prompt?.summary, fromText?.promptSummary),
|
|
promptPreview: firstNonEmptyValue(payload.promptPreview, prompt?.preview, fromText?.promptPreview),
|
|
textChars: Number.isInteger(prompt?.textChars) ? prompt.textChars : fromText?.prompt?.textChars
|
|
});
|
|
}
|
|
|
|
function codeAgentPromptMetadataFromText(value, traceId = null, source = "code-agent-request", options = {}) {
|
|
const promptText = String(value ?? "").trim();
|
|
if (!promptText) return null;
|
|
return codeAgentPromptMetadataFromParts({
|
|
traceId,
|
|
source,
|
|
promptSha256: options.fullText === true ? sha256Text(promptText) : undefined,
|
|
promptSummary: promptText,
|
|
promptPreview: promptText,
|
|
textChars: options.fullText === true ? promptText.length : undefined
|
|
});
|
|
}
|
|
|
|
function codeAgentPromptMetadataFromParts({ traceId = null, source = "code-agent-request", promptId = null, promptSha256 = null, promptSummary = null, promptPreview = null, textChars = undefined } = {}) {
|
|
const digest = firstNonEmptyValue(promptSha256);
|
|
const summary = clippedPromptText(firstNonEmptyValue(promptSummary, promptPreview), 240);
|
|
const preview = clippedPromptText(firstNonEmptyValue(promptPreview, promptSummary), 500);
|
|
const explicitId = firstNonEmptyValue(promptId);
|
|
if (!explicitId && !digest && !summary && !preview) return null;
|
|
const id = explicitId || codeAgentPromptId(traceId, digest);
|
|
const prompt = compactCodeAgentObject({
|
|
id,
|
|
sha256: digest || undefined,
|
|
summary,
|
|
preview,
|
|
textChars: Number.isInteger(textChars) ? textChars : undefined,
|
|
source,
|
|
valuesPrinted: false
|
|
});
|
|
return compactCodeAgentObject({
|
|
promptId: id,
|
|
promptSha256: digest || undefined,
|
|
promptSummary: summary,
|
|
promptPreview: preview,
|
|
prompt,
|
|
valuesPrinted: false
|
|
}) ?? null;
|
|
}
|
|
|
|
function codeAgentPromptFields(metadata = null) {
|
|
if (!metadata) return {};
|
|
return compactCodeAgentObject({
|
|
promptId: metadata.promptId,
|
|
promptSha256: metadata.promptSha256,
|
|
promptSummary: metadata.promptSummary,
|
|
promptPreview: metadata.promptPreview,
|
|
prompt: metadata.prompt
|
|
}) ?? {};
|
|
}
|
|
|
|
function codeAgentPromptId(traceId = null, digest = null) {
|
|
const traceSuffix = safeTraceId(traceId)?.slice(4).replace(/[^A-Za-z0-9_.:-]/gu, "_");
|
|
const digestSuffix = firstNonEmptyValue(digest)?.slice(0, 16);
|
|
const suffix = firstNonEmptyValue(traceSuffix, digestSuffix);
|
|
return suffix ? `prm_${suffix}` : undefined;
|
|
}
|
|
|
|
function clippedPromptText(value, limit) {
|
|
const promptText = String(value ?? "").trim();
|
|
if (!promptText) return undefined;
|
|
return promptText.length > limit ? `${promptText.slice(0, limit)}...` : promptText;
|
|
}
|
|
|
|
function sha256Text(value) {
|
|
return createHash("sha256").update(String(value)).digest("hex");
|
|
}
|
|
|
|
function compactCodeAgentObject(value) {
|
|
const result = {};
|
|
for (const [key, item] of Object.entries(value ?? {})) {
|
|
if (item === undefined || item === null) continue;
|
|
if (Array.isArray(item) && item.length === 0) continue;
|
|
if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue;
|
|
result[key] = item;
|
|
}
|
|
return Object.keys(result).length > 0 ? result : undefined;
|
|
}
|
|
|
|
function codeAgentBillingReservationEvidence(value = null) {
|
|
if (!value || typeof value !== "object") return null;
|
|
const reservationId = textValue(value.reservationId);
|
|
if (!reservationId) return null;
|
|
return {
|
|
reservationId,
|
|
resourceType: textValue(value.resourceType) || null,
|
|
planId: textValue(value.planId) || null,
|
|
estimatedCredits: numberOrNull(value.estimatedCredits),
|
|
estimatedTokens: numberOrNull(value.estimatedTokens),
|
|
expiresAt: textValue(value.expiresAt) || null,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
function codeAgentConversationMessagesEvidence(payload = {}, params = {}, traceId = null, finalResponse = null) {
|
|
const userText = boundedConversationMessageText(params.message ?? params.prompt ?? payload.userMessage ?? payload.prompt);
|
|
if (!userText) return [];
|
|
const resolvedTraceId = safeTraceId(traceId) || null;
|
|
const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, payload);
|
|
const session = payload.session && typeof payload.session === "object" ? payload.session : null;
|
|
const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null;
|
|
const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null;
|
|
const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null;
|
|
const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null;
|
|
const createdAt = payload.createdAt ?? payload.reply?.createdAt ?? new Date().toISOString();
|
|
const updatedAt = payload.updatedAt ?? createdAt;
|
|
const userMessage = {
|
|
id: lifecycle.userMessageId,
|
|
messageId: lifecycle.userMessageId,
|
|
role: "user",
|
|
title: "用户",
|
|
text: userText,
|
|
status: "sent",
|
|
traceId: resolvedTraceId,
|
|
turnId: lifecycle.turnId,
|
|
conversationId,
|
|
sessionId,
|
|
threadId,
|
|
createdAt,
|
|
updatedAt,
|
|
source: "code-agent-submit",
|
|
secretMaterialStored: false,
|
|
valuesRedacted: true
|
|
};
|
|
const agentText = boundedConversationMessageText(
|
|
finalResponse?.text
|
|
?? payload.reply?.content
|
|
?? payload.message?.content
|
|
?? payload.assistantText
|
|
?? payload.error?.userMessage
|
|
?? payload.error?.message
|
|
?? payload.blocker?.userMessage
|
|
?? payload.blocker?.message
|
|
?? payload.blocker?.summary
|
|
);
|
|
const agentStatus = codeAgentConversationAgentMessageStatus(payload);
|
|
const agentMessage = {
|
|
id: lifecycle.assistantMessageId,
|
|
messageId: lifecycle.assistantMessageId,
|
|
role: "agent",
|
|
title: agentStatus === "failed" ? "Code Agent 返回阻塞" : agentStatus === "running" ? "Code Agent 处理中" : "Code Agent 回复",
|
|
text: agentText || (agentStatus === "running" ? "" : "Code Agent 请求已结束，请查看 Trace 详情。"),
|
|
status: agentStatus,
|
|
traceId: resolvedTraceId,
|
|
turnId: lifecycle.turnId,
|
|
conversationId,
|
|
sessionId,
|
|
threadId,
|
|
createdAt: payload.reply?.createdAt ?? updatedAt,
|
|
updatedAt,
|
|
source: "code-agent-result",
|
|
...(payload.runnerTrace && typeof payload.runnerTrace === "object" ? { runnerTrace: codeAgentConversationRunnerTraceEvidence(payload.runnerTrace, resolvedTraceId) } : {}),
|
|
secretMaterialStored: false,
|
|
valuesRedacted: true
|
|
};
|
|
return [userMessage, agentMessage];
|
|
}
|
|
|
|
function boundedConversationMessageText(value, limit = 12000) {
|
|
const text = conversationText(value);
|
|
if (!text) return "";
|
|
return text.length > limit ? text.slice(0, limit) : text;
|
|
}
|
|
|
|
function conversationText(value) {
|
|
const extracted = conversationTextRaw(value).replace(/\r\n?/gu, "\n");
|
|
if (!extracted.trim() || extracted.trim() === "[object Object]") return "";
|
|
return extracted;
|
|
}
|
|
|
|
function conversationTextRaw(value) {
|
|
if (value === undefined || value === null) return "";
|
|
if (typeof value === "string") return value;
|
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
if (Array.isArray(value)) return value.map(conversationTextRaw).filter(Boolean).join("\n");
|
|
if (typeof value === "object") {
|
|
for (const key of ["text", "content", "message", "prompt", "value", "summary", "preview", "title"]) {
|
|
const text = conversationTextRaw(value[key]);
|
|
if (text) return text;
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function codeAgentMessageTraceSuffix(traceId) {
|
|
const text = safeTraceId(traceId) || `local_${Date.now().toString(36)}`;
|
|
return text.replace(/^trc_/u, "").replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 48) || "trace";
|
|
}
|
|
|
|
function codeAgentTurnLifecycleFields(traceId, source = {}) {
|
|
const resolvedTraceId = safeTraceId(source?.traceId ?? traceId) || safeTraceId(traceId) || null;
|
|
const turnId = safeTurnId(source?.turnId) || resolvedTraceId;
|
|
const traceSuffix = codeAgentMessageTraceSuffix(resolvedTraceId);
|
|
return {
|
|
turnId,
|
|
userMessageId: safeMessageId(source?.userMessageId ?? source?.userMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "user"),
|
|
assistantMessageId: safeMessageId(source?.assistantMessageId ?? source?.assistantMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "agent")
|
|
};
|
|
}
|
|
|
|
function codeAgentLifecycleMessageId(traceSuffix, role) {
|
|
return `msg_${traceSuffix}_${role}`;
|
|
}
|
|
|
|
function safeTurnId(value) {
|
|
const text = textValue(value);
|
|
return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null;
|
|
}
|
|
|
|
function safeMessageId(value) {
|
|
const text = textValue(value);
|
|
return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null;
|
|
}
|
|
|
|
function codeAgentConversationAgentMessageStatus(payload = {}) {
|
|
const status = textValue(payload.status ?? payload.agentRun?.terminalStatus ?? payload.commandState ?? payload.runStatus).toLowerCase();
|
|
if (codeAgentPayloadHasSealedFinalResponse(payload)) return "completed";
|
|
if (["completed", "done", "success"].includes(status)) return "completed";
|
|
if (["failed", "blocked", "error", "timeout", "canceled", "cancelled"].includes(status)) return "failed";
|
|
if (payload.error || payload.blocker) return "failed";
|
|
return "running";
|
|
}
|
|
|
|
function codeAgentPayloadHasSealedFinalResponse(payload = {}) {
|
|
if (!payload || typeof payload !== "object") return false;
|
|
if (payload.error || payload.blocker) return false;
|
|
if (!codeAgentFinalResponseText(payload)) return false;
|
|
const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : {};
|
|
return isTraceCommandTerminalStatus(payload.status) ||
|
|
isTraceCommandTerminalStatus(agentRun.status) ||
|
|
isTraceCommandTerminalStatus(agentRun.runStatus) ||
|
|
isTraceCommandTerminalStatus(agentRun.commandState) ||
|
|
isTraceCommandTerminalStatus(agentRun.terminalStatus);
|
|
}
|
|
|
|
function codeAgentConversationRunnerTraceEvidence(runnerTrace = {}, traceId = null) {
|
|
const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : [];
|
|
const lastEvent = runnerTrace.lastEvent && typeof runnerTrace.lastEvent === "object" ? runnerTrace.lastEvent : events.at(-1) ?? null;
|
|
const eventCount = Number.isFinite(Number(runnerTrace.eventCount)) ? Number(runnerTrace.eventCount) : events.length;
|
|
return {
|
|
traceId: safeTraceId(runnerTrace.traceId ?? traceId) || traceId || null,
|
|
status: runnerTrace.status ?? lastEvent?.status ?? null,
|
|
sessionId: runnerTrace.sessionId ?? null,
|
|
threadId: runnerTrace.threadId ?? null,
|
|
runId: runnerTrace.runId ?? lastEvent?.runId ?? null,
|
|
commandId: runnerTrace.commandId ?? lastEvent?.commandId ?? null,
|
|
eventCount,
|
|
lastEvent: lastEvent ? {
|
|
label: lastEvent.label ?? null,
|
|
status: lastEvent.status ?? null,
|
|
terminal: lastEvent.terminal === true,
|
|
createdAt: lastEvent.createdAt ?? null,
|
|
message: boundedConversationMessageText(lastEvent.message ?? lastEvent.text, 1000),
|
|
valuesPrinted: false
|
|
} : null,
|
|
eventsCompacted: true,
|
|
fullTraceLoaded: false,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function codeAgentFinalResponseEvidence(payload = {}, traceId = null) {
|
|
const text = codeAgentFinalResponseText(payload);
|
|
if (!text) return null;
|
|
return {
|
|
text,
|
|
textChars: text.length,
|
|
messageId: payload.finalResponse?.messageId ?? payload.reply?.messageId ?? payload.messageId ?? null,
|
|
role: payload.reply?.role ?? payload.message?.role ?? "assistant",
|
|
status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.finalResponse?.status ?? payload.status ?? null,
|
|
traceId,
|
|
createdAt: payload.finalResponse?.createdAt ?? payload.reply?.createdAt ?? payload.createdAt ?? null,
|
|
updatedAt: payload.finalResponse?.updatedAt ?? payload.updatedAt ?? null,
|
|
source: payload.finalResponse?.source ?? "code-agent-result",
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function codeAgentFinalResponseText(payload = {}) {
|
|
return conversationText(payload?.finalResponse?.text ?? payload?.finalResponse?.content ?? payload?.reply?.content ?? payload?.message?.content ?? payload?.assistantText);
|
|
}
|
|
|
|
function codeAgentTraceSummaryEvidence(payload = {}, traceId = null, finalResponse = null) {
|
|
const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null;
|
|
const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : [];
|
|
const terminalEvent = [...events].reverse().find((event) => event?.terminal === true) ?? runnerTrace?.lastEvent ?? null;
|
|
return {
|
|
traceId,
|
|
source: "code-agent-derived-session-evidence",
|
|
sourceEventCount: Number.isFinite(Number(runnerTrace?.eventCount)) ? Number(runnerTrace.eventCount) : events.length,
|
|
terminalStatus: payload.agentRun?.terminalStatus ?? payload.status ?? terminalEvent?.status ?? null,
|
|
lastEventLabel: terminalEvent?.label ?? null,
|
|
finalAssistantRow: finalResponse ? {
|
|
role: finalResponse.role,
|
|
status: finalResponse.status,
|
|
textChars: finalResponse.textChars,
|
|
textPreview: finalResponse.text.slice(0, 240),
|
|
messageId: finalResponse.messageId,
|
|
valuesPrinted: false
|
|
} : null,
|
|
agentRun: payload.agentRun ? {
|
|
runId: payload.agentRun.runId ?? null,
|
|
commandId: payload.agentRun.commandId ?? null,
|
|
attemptId: payload.agentRun.attemptId ?? null,
|
|
runnerId: payload.agentRun.runnerId ?? null,
|
|
jobName: payload.agentRun.jobName ?? null,
|
|
namespace: payload.agentRun.namespace ?? null,
|
|
lastSeq: payload.agentRun.lastSeq ?? null,
|
|
valuesPrinted: false
|
|
} : null,
|
|
updatedAt: payload.updatedAt ?? null,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function annotateOwner(payload, params = {}) {
|
|
if (!params.ownerUserId) return payload;
|
|
return {
|
|
...payload,
|
|
ownerUserId: params.ownerUserId,
|
|
ownerRole: params.ownerRole ?? null
|
|
};
|
|
}
|
|
|
|
function canAccessOwnedResult(result, actor) {
|
|
if (!result?.ownerUserId || !actor) return true;
|
|
return actor.role === "admin" || actor.id === result.ownerUserId;
|
|
}
|
|
|
|
function isCodeAgentResultCanceled(result) {
|
|
return result?.status === "canceled" || result?.canceled === true || result?.error?.code === "codex_stdio_canceled";
|
|
}
|
|
|
|
function codeAgentCancelScopeMismatch(params = {}, result = {}) {
|
|
return cancelScopeFieldMismatch("conversationId", safeConversationId(params.conversationId), cancelExpectedValues(safeConversationId,
|
|
result.conversationId,
|
|
result.agentRun?.conversationId,
|
|
result.session?.conversationId,
|
|
result.sessionReuse?.conversationId,
|
|
result.runnerTrace?.conversationId
|
|
)) ?? cancelScopeFieldMismatch("sessionId", safeSessionId(params.sessionId), cancelExpectedValues(safeSessionId,
|
|
result.sessionId,
|
|
result.session?.sessionId,
|
|
result.sessionReuse?.sessionId,
|
|
result.agentRun?.hwlabSessionId
|
|
)) ?? cancelScopeFieldMismatch("threadId", safeOpaqueId(params.threadId), cancelExpectedValues(safeOpaqueId,
|
|
result.threadId,
|
|
result.agentRun?.threadId,
|
|
result.session?.threadId,
|
|
result.sessionReuse?.threadId,
|
|
result.runnerTrace?.threadId
|
|
));
|
|
}
|
|
|
|
function cancelExpectedValues(normalize, ...values) {
|
|
return [...new Set(values.map((value) => normalize(value)).filter(Boolean))];
|
|
}
|
|
|
|
function cancelScopeFieldMismatch(field, requested, expectedValues) {
|
|
if (!requested || expectedValues.length === 0 || expectedValues.includes(requested)) return null;
|
|
return { field, requested, expected: expectedValues[0] };
|
|
}
|
|
|
|
async function syncAlreadyTerminalAgentRunBeforeCancel({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) {
|
|
if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null;
|
|
const currentTerminalStatus = currentResult.agentRun?.terminalStatus ?? currentResult.agentRun?.commandState ?? currentResult.agentRun?.commandStatus;
|
|
if (isTraceCommandTerminalStatus(currentTerminalStatus)) {
|
|
return alreadyTerminalCancelPayload({ traceId, payload: currentResult, params, options, traceStore });
|
|
}
|
|
try {
|
|
const synced = await syncAgentRunChatResult({
|
|
traceId,
|
|
currentResult,
|
|
options: { ...options, deferAgentRunResultSync: false },
|
|
traceStore,
|
|
forceResultSync: true,
|
|
refreshEvents: true
|
|
});
|
|
const payload = synced?.result ?? null;
|
|
const payloadTerminalStatus = payload?.agentRun?.terminalStatus ?? payload?.agentRun?.commandState ?? payload?.agentRun?.commandStatus;
|
|
if (!isTraceCommandTerminalStatus(payloadTerminalStatus)) return null;
|
|
return alreadyTerminalCancelPayload({ traceId, payload, params, options, traceStore });
|
|
} catch (error) {
|
|
traceStore.append(traceId, {
|
|
type: "cancel",
|
|
status: "degraded",
|
|
label: "agentrun:cancel:terminal-check-degraded",
|
|
errorCode: error?.code ?? "terminal_check_failed",
|
|
message: error?.message ?? "Failed to check AgentRun terminal state before forwarding cancel.",
|
|
waitingFor: "cancel-terminal-check",
|
|
valuesPrinted: false
|
|
});
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function alreadyTerminalCancelPayload({ traceId, payload = {}, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) {
|
|
traceStore.append(traceId, {
|
|
type: "cancel",
|
|
status: "blocked",
|
|
label: "agentrun:cancel:already-terminal",
|
|
message: "AgentRun command was already terminal; HWLAB did not forward cancel and kept the sealed terminal projection.",
|
|
runId: payload.agentRun?.runId ?? null,
|
|
commandId: payload.agentRun?.commandId ?? null,
|
|
terminalStatus: payload.agentRun?.terminalStatus ?? payload.status,
|
|
waitingFor: "already-terminal-projection",
|
|
valuesPrinted: false
|
|
});
|
|
await recordCodeAgentTerminalTurnStatusEffects({
|
|
payload,
|
|
params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role },
|
|
options,
|
|
preserveLastTraceId: true
|
|
});
|
|
return {
|
|
...payload,
|
|
cancelDisposition: {
|
|
status: "already_terminal",
|
|
forwarded: false,
|
|
route: "/v1/agent/chat/cancel",
|
|
message: "AgentRun command was already terminal; cancel was not forwarded.",
|
|
valuesPrinted: false
|
|
},
|
|
runnerTrace: traceStore.snapshot(traceId)
|
|
};
|
|
}
|
|
|
|
function cancelBlockedPayload({
|
|
code,
|
|
message,
|
|
traceId,
|
|
conversationId = null,
|
|
sessionId = null,
|
|
session = null,
|
|
runnerTrace = null,
|
|
status = "failed",
|
|
unsupported = false,
|
|
degraded = false
|
|
}) {
|
|
const sessionSummary = codeAgentSessionLifecycleSummary({
|
|
session,
|
|
runnerTrace,
|
|
status,
|
|
unsupported,
|
|
degraded,
|
|
error: { code, userMessage: message, message }
|
|
});
|
|
return {
|
|
accepted: false,
|
|
canceled: false,
|
|
status,
|
|
conversationId,
|
|
sessionId,
|
|
traceId,
|
|
session,
|
|
sessionLifecycleStatus: sessionSummary.status,
|
|
sessionLifecycle: sessionSummary,
|
|
sessionSummary,
|
|
unsupported,
|
|
degraded,
|
|
runnerTrace,
|
|
lastTraceEvent: runnerTrace?.lastEvent ?? null,
|
|
error: {
|
|
code,
|
|
layer: "session",
|
|
category: "cancel_blocked",
|
|
retryable: true,
|
|
userMessage: message,
|
|
message,
|
|
traceId,
|
|
route: "/v1/agent/chat/cancel",
|
|
toolName: "codex-stdio.cancel"
|
|
},
|
|
blocker: {
|
|
code,
|
|
layer: "session",
|
|
category: "cancel_blocked",
|
|
retryable: true,
|
|
summary: message,
|
|
userMessage: message,
|
|
traceId,
|
|
route: "/v1/agent/chat/cancel",
|
|
toolName: "codex-stdio.cancel"
|
|
}
|
|
};
|
|
}
|
|
|
|
export async function handleCodeAgentTraceHttp(request, response, url, options) {
|
|
const startedAt = nowMs();
|
|
const parts = url.pathname.split("/").filter(Boolean);
|
|
const traceId = decodeURIComponent(parts[3] ?? "");
|
|
const streamSegment = parts[4];
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
const pageOptions = codeAgentTracePageOptions(url);
|
|
if (!/^trc_[A-Za-z0-9_.:-]+$/u.test(traceId)) {
|
|
sendJson(response, 400, {
|
|
error: {
|
|
code: "invalid_trace_id",
|
|
message: "traceId must start with trc_ and contain only safe identifier characters"
|
|
}
|
|
});
|
|
return;
|
|
}
|
|
const requestOptions = codeAgentRequestScopedOptions(options);
|
|
const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "trace_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "trace"));
|
|
if (projectMismatch) {
|
|
sendJson(response, projectMismatch.statusCode, projectMismatch.body);
|
|
return;
|
|
}
|
|
const context = await measureCodeAgentHttpPhase(requestOptions, "trace_projection_read", () => readCodeAgentCompatProjection(traceId, requestOptions));
|
|
if (context.statusCode) {
|
|
sendJson(response, context.statusCode, context.body);
|
|
return;
|
|
}
|
|
|
|
if (streamSegment === "stream") {
|
|
sendTraceSse(response, traceStore, traceId, requestOptions);
|
|
return;
|
|
}
|
|
|
|
const body = await measureCodeAgentHttpPhase(requestOptions, "trace_paginate", () => {
|
|
return paginateTraceSnapshot(context.trace, pageOptions);
|
|
});
|
|
const statusCode = context.found ? 200 : 404;
|
|
void emitCodeAgentOtelSpan("trace_events_read", traceId, requestOptions.env ?? process.env, {
|
|
startTimeMs: startedAt,
|
|
attributes: {
|
|
"http.method": "GET",
|
|
"http.route": "/v1/agent/traces/:traceId",
|
|
"http.status_code": statusCode,
|
|
found: Boolean(context.found),
|
|
returnedEvents: Array.isArray(body?.events) ? body.events.length : 0,
|
|
sinceSeq: pageOptions.sinceSeq,
|
|
limit: pageOptions.limit,
|
|
fromSeq: body?.range?.fromSeq ?? null,
|
|
toSeq: body?.range?.toSeq ?? null,
|
|
totalEvents: body?.range?.total ?? body?.eventCount ?? null,
|
|
hasMore: Boolean(body?.hasMore),
|
|
fullTraceLoaded: Boolean(body?.fullTraceLoaded)
|
|
}
|
|
});
|
|
sendJson(response, statusCode, codeAgentCompatProjectionPayload(body, context));
|
|
}
|
|
|
|
function codeAgentTracePageOptions(url) {
|
|
const sinceSeq = nonNegativeInteger(url.searchParams.get("sinceSeq") ?? url.searchParams.get("since"), 0);
|
|
const requestedLimit = parsePositiveInteger(url.searchParams.get("limit"), DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT);
|
|
return {
|
|
sinceSeq,
|
|
limit: Math.min(requestedLimit, MAX_CODE_AGENT_TRACE_PAGE_LIMIT)
|
|
};
|
|
}
|
|
|
|
function paginateTraceSnapshot(snapshot, { sinceSeq, limit }) {
|
|
const sourceEvents = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
|
const indexedEvents = sourceEvents.map((event, index) => ({ event, seq: traceEventSequence(event, index) }));
|
|
const firstIndex = indexedEvents.findIndex((item) => item.seq > sinceSeq);
|
|
const startIndex = firstIndex >= 0 ? firstIndex : indexedEvents.length;
|
|
const pageItems = indexedEvents.slice(startIndex, startIndex + limit);
|
|
const events = pageItems.map((item) => item.event);
|
|
const fromSeq = pageItems.length > 0 ? pageItems[0].seq : null;
|
|
const toSeq = pageItems.length > 0 ? pageItems[pageItems.length - 1].seq : null;
|
|
const hasMore = startIndex + pageItems.length < indexedEvents.length;
|
|
const totalEvents = snapshot?.eventCount ?? indexedEvents.length;
|
|
return {
|
|
...snapshot,
|
|
events,
|
|
eventCount: totalEvents,
|
|
range: {
|
|
sinceSeq,
|
|
fromSeq,
|
|
toSeq,
|
|
limit,
|
|
returned: events.length,
|
|
total: totalEvents
|
|
},
|
|
truncated: hasMore,
|
|
hasMore,
|
|
nextSinceSeq: pageItems.length > 0 ? toSeq : sinceSeq,
|
|
fullTraceLoaded: !hasMore
|
|
};
|
|
}
|
|
|
|
function traceEventSequence(event, index) {
|
|
const seq = Number(event?.seq);
|
|
return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1;
|
|
}
|
|
|
|
function nonNegativeInteger(value, fallback) {
|
|
const parsed = Number.parseInt(value ?? "", 10);
|
|
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
|
}
|
|
|
|
function isTraceCommandTerminalStatus(status) {
|
|
return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"));
|
|
}
|
|
|
|
function traceSnapshotWithTerminalEvidence(snapshot, result, traceId, refreshError = null) {
|
|
const evidence = agentRunTerminalTraceEvidence(result, traceId);
|
|
if (snapshot?.status !== "missing") return traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError);
|
|
if (!evidence) return {
|
|
...snapshot,
|
|
ok: false,
|
|
traceStatus: "missing",
|
|
retention: traceRetentionSummary("missing")
|
|
};
|
|
return {
|
|
...snapshot,
|
|
ok: true,
|
|
status: "expired",
|
|
traceStatus: "expired",
|
|
projectionStatus: "projecting",
|
|
projectionHealth: "degraded",
|
|
persisted: true,
|
|
retention: traceRetentionSummary("expired"),
|
|
conversationId: evidence.conversationId,
|
|
sessionId: evidence.sessionId,
|
|
threadId: evidence.threadId,
|
|
...codeAgentPromptFields(evidence),
|
|
agentRun: evidence.agentRun,
|
|
finalResponse: evidence.finalResponse,
|
|
traceSummary: evidence.traceSummary,
|
|
terminalEvidence: {
|
|
available: true,
|
|
source: evidence.source,
|
|
refresh: refreshError ? {
|
|
attempted: true,
|
|
ok: false,
|
|
code: refreshError?.code ?? "agentrun_trace_refresh_failed",
|
|
message: refreshError?.message ?? "AgentRun trace events could not be refreshed; command result remains authoritative for final response.",
|
|
valuesPrinted: false
|
|
} : { attempted: false, ok: null },
|
|
conversationId: evidence.conversationId,
|
|
sessionId: evidence.sessionId,
|
|
threadId: evidence.threadId,
|
|
...codeAgentPromptFields(evidence),
|
|
agentRun: evidence.agentRun,
|
|
traceSummary: evidence.traceSummary,
|
|
finalResponse: evidence.finalResponse,
|
|
valuesPrinted: false
|
|
}
|
|
};
|
|
}
|
|
|
|
function traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError = null) {
|
|
if (!evidence) return snapshot;
|
|
return {
|
|
...snapshot,
|
|
conversationId: snapshot.conversationId ?? evidence.conversationId,
|
|
sessionId: snapshot.sessionId ?? evidence.sessionId,
|
|
threadId: snapshot.threadId ?? evidence.threadId,
|
|
...codeAgentPromptFields(evidence),
|
|
agentRun: snapshot.agentRun ?? evidence.agentRun,
|
|
finalResponse: snapshot.finalResponse ?? evidence.finalResponse,
|
|
traceSummary: snapshot.traceSummary ?? evidence.traceSummary,
|
|
terminalEvidence: snapshot.terminalEvidence ?? (refreshError ? {
|
|
available: true,
|
|
source: evidence.source,
|
|
refresh: {
|
|
attempted: true,
|
|
ok: false,
|
|
code: refreshError?.code ?? "agentrun_trace_refresh_failed",
|
|
message: refreshError?.message ?? "AgentRun trace events could not be refreshed; live events may be partial.",
|
|
valuesPrinted: false
|
|
},
|
|
valuesPrinted: false
|
|
} : undefined)
|
|
};
|
|
}
|
|
|
|
function agentRunTerminalTraceEvidence(result, traceId) {
|
|
if (!result || typeof result !== "object") return null;
|
|
const storedSummary = result.traceSummary && typeof result.traceSummary === "object" && result.traceSummary.source === "agentrun-command-result" ? result.traceSummary : null;
|
|
const finalResponse = agentRunTerminalFinalResponse(result, traceId);
|
|
const promptEvidence = codeAgentPromptMetadata(result, {}, traceId);
|
|
const agentRun = result.agentRun && typeof result.agentRun === "object" ? {
|
|
adapter: result.agentRun.adapter ?? null,
|
|
runId: result.agentRun.runId ?? null,
|
|
commandId: result.agentRun.commandId ?? null,
|
|
attemptId: result.agentRun.attemptId ?? null,
|
|
runnerId: result.agentRun.runnerId ?? null,
|
|
jobName: result.agentRun.jobName ?? null,
|
|
namespace: result.agentRun.namespace ?? null,
|
|
terminalStatus: result.agentRun.terminalStatus ?? null,
|
|
lastSeq: result.agentRun.lastSeq ?? null,
|
|
valuesPrinted: false
|
|
} : null;
|
|
const terminalStatus = normalizeTurnStatus(result.agentRun?.terminalStatus, result.agentRun?.commandState, result.agentRun?.status, result.agentRun?.runStatus);
|
|
if (!isTraceCommandTerminalStatus(terminalStatus)) return null;
|
|
if (!agentRun?.runId || !agentRun?.commandId || (!storedSummary && !finalResponse)) return null;
|
|
const traceSummary = {
|
|
traceId,
|
|
source: "agentrun-command-result",
|
|
sourceEventCount: numberOrNull(storedSummary?.sourceEventCount ?? storedSummary?.eventCount ?? result.runnerTrace?.eventCount),
|
|
renderedRowSummary: storedSummary?.renderedRowSummary ?? null,
|
|
noiseEventCount: numberOrNull(storedSummary?.noiseEventCount),
|
|
omittedNoiseCount: numberOrNull(storedSummary?.omittedNoiseCount),
|
|
terminalStatus: storedSummary?.terminalStatus ?? terminalStatus,
|
|
updatedAt: storedSummary?.updatedAt ?? result.updatedAt ?? null,
|
|
valuesPrinted: false
|
|
};
|
|
return {
|
|
source: "agentrun-command-result",
|
|
conversationId: safeConversationId(result.conversationId) || null,
|
|
sessionId: safeSessionId(result.sessionId) || null,
|
|
threadId: safeOpaqueId(result.threadId) || null,
|
|
...codeAgentPromptFields(promptEvidence),
|
|
agentRun,
|
|
finalResponse,
|
|
traceSummary
|
|
};
|
|
}
|
|
|
|
function agentRunTerminalFinalResponse(result, traceId) {
|
|
const stored = result.finalResponse && typeof result.finalResponse === "object" ? result.finalResponse : null;
|
|
const text = textValue(stored?.text ?? result.reply?.content ?? result.message?.content ?? result.assistantText);
|
|
if (!text) return null;
|
|
const storedTraceId = safeTraceId(stored?.traceId ?? result.traceId) || null;
|
|
if (storedTraceId && storedTraceId !== traceId) return null;
|
|
return {
|
|
text,
|
|
textChars: text.length,
|
|
messageId: stored?.messageId ?? result.reply?.messageId ?? result.messageId ?? null,
|
|
role: stored?.role ?? result.reply?.role ?? "assistant",
|
|
status: stored?.status ?? result.status ?? null,
|
|
traceId: traceId,
|
|
createdAt: stored?.createdAt ?? result.reply?.createdAt ?? result.createdAt ?? null,
|
|
updatedAt: stored?.updatedAt ?? result.updatedAt ?? null,
|
|
source: "agentrun-command-result",
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function traceRetentionSummary(status) {
|
|
return {
|
|
traceStatus: status,
|
|
liveTraceStore: status === "missing" ? "missing" : "expired-or-evicted",
|
|
policy: "live trace events are short-term storage; terminal final response and summary are resolved from AgentRun command result by traceId and commandId",
|
|
replacementEvidence: status === "missing" ? "resolve the trace through AgentRun command registry/result or report the trace as missing" : "use terminalEvidence.finalResponse, terminalEvidence.traceSummary, conversationId, sessionId, threadId, and agentRun ids",
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function numberOrNull(value) {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function sendTraceSse(response, traceStore, traceId, options = {}) {
|
|
response.writeHead(200, {
|
|
"content-type": "text/event-stream; charset=utf-8",
|
|
"cache-control": "no-store",
|
|
connection: "keep-alive",
|
|
"x-accel-buffering": "no"
|
|
});
|
|
const writeEvent = (eventName, payload) => {
|
|
response.write(`event: ${eventName}\n`);
|
|
response.write(`data: ${JSON.stringify(payload)}\n\n`);
|
|
};
|
|
writeEvent("snapshot", traceStore.snapshot(traceId));
|
|
const unsubscribe = traceStore.subscribe(traceId, (event, snapshot) => {
|
|
writeEvent("runnerTrace", { event, snapshot });
|
|
});
|
|
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_TRACE_HEARTBEAT_MS, 15000);
|
|
const heartbeat = setInterval(() => {
|
|
writeEvent("heartbeat", traceStore.snapshot(traceId));
|
|
}, heartbeatMs);
|
|
response.on("close", () => {
|
|
clearInterval(heartbeat);
|
|
unsubscribe();
|
|
});
|
|
}
|
|
|
|
export function createCodeAgentChatResultStore({ maxResults = 256 } = {}) {
|
|
const limit = positiveInteger(maxResults, 256);
|
|
const results = new Map();
|
|
function set(traceId, payload) {
|
|
const id = safeTraceId(traceId);
|
|
if (!id) return;
|
|
results.set(id, {
|
|
...payload,
|
|
traceId: id,
|
|
cachedAt: new Date().toISOString()
|
|
});
|
|
while (results.size > limit) {
|
|
const oldest = results.keys().next().value;
|
|
if (!oldest) break;
|
|
results.delete(oldest);
|
|
}
|
|
}
|
|
return {
|
|
set,
|
|
get: (traceId) => results.get(safeTraceId(traceId)) ?? null,
|
|
clear: () => results.clear()
|
|
};
|
|
}
|
|
|
|
function compactCodeAgentChatResultPayload(payload, options = {}) {
|
|
if (!payload || typeof payload !== "object") return payload;
|
|
const limit = resultTraceEventLimit(options);
|
|
const terminalEvidence = agentRunTerminalTraceEvidence(payload, payload.traceId);
|
|
const { userBillingReservation, ...publicPayload } = payload;
|
|
return {
|
|
...publicPayload,
|
|
...codeAgentTurnLifecycleFields(publicPayload.traceId, publicPayload),
|
|
...(terminalEvidence ? { terminalEvidence: terminalEvidencePayload(terminalEvidence) } : {}),
|
|
...(publicPayload.runnerTrace && typeof publicPayload.runnerTrace === "object"
|
|
? { runnerTrace: compactRunnerTraceForResult(publicPayload.runnerTrace, limit) }
|
|
: {})
|
|
};
|
|
}
|
|
|
|
function terminalEvidencePayload(evidence) {
|
|
return {
|
|
available: true,
|
|
source: evidence.source,
|
|
conversationId: evidence.conversationId,
|
|
sessionId: evidence.sessionId,
|
|
threadId: evidence.threadId,
|
|
...codeAgentPromptFields(evidence),
|
|
agentRun: evidence.agentRun,
|
|
traceSummary: evidence.traceSummary,
|
|
finalResponse: evidence.finalResponse,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function compactRunnerTraceForResult(runnerTrace, limit = DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT) {
|
|
if (!runnerTrace || typeof runnerTrace !== "object") return runnerTrace;
|
|
const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : [];
|
|
const total = Number.isInteger(runnerTrace.eventCount) ? runnerTrace.eventCount : events.length;
|
|
const effectiveLimit = Math.max(8, positiveInteger(limit, DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT));
|
|
if (events.length <= effectiveLimit) {
|
|
return {
|
|
...runnerTrace,
|
|
eventCount: total,
|
|
eventsCompacted: false
|
|
};
|
|
}
|
|
const headCount = Math.min(24, Math.max(1, Math.floor(effectiveLimit / 4)));
|
|
const tailCount = Math.max(1, effectiveLimit - headCount - 1);
|
|
const omitted = Math.max(0, events.length - headCount - tailCount);
|
|
const head = events.slice(0, headCount);
|
|
const tail = events.slice(events.length - tailCount);
|
|
const marker = {
|
|
traceId: runnerTrace.traceId ?? head[0]?.traceId ?? tail[0]?.traceId ?? null,
|
|
type: "trace",
|
|
stage: "trace",
|
|
status: "truncated",
|
|
label: "trace:compacted",
|
|
createdAt: tail[0]?.createdAt ?? runnerTrace.updatedAt ?? new Date().toISOString(),
|
|
message: `Result polling response compacted ${omitted} trace events; fetch /v1/agent/traces/${encodeURIComponent(runnerTrace.traceId ?? "")} for the full trace.`,
|
|
omittedEventCount: omitted,
|
|
retainedEventCount: head.length + tail.length,
|
|
totalEventCount: total,
|
|
valuesPrinted: false
|
|
};
|
|
const compactEvents = [...head, marker, ...tail];
|
|
return {
|
|
...runnerTrace,
|
|
eventCount: total,
|
|
events: compactEvents,
|
|
eventLabels: compactEvents.map(traceEventLabel).filter(Boolean),
|
|
eventsCompacted: true,
|
|
eventWindow: {
|
|
mode: "head-tail",
|
|
retained: compactEvents.length,
|
|
omitted,
|
|
total
|
|
}
|
|
};
|
|
}
|
|
|
|
function traceEventLabel(event) {
|
|
return typeof event === "string" ? event : event?.label;
|
|
}
|
|
|
|
function resultTraceEventLimit(options = {}) {
|
|
return positiveInteger(
|
|
options.env?.HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT,
|
|
DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT
|
|
);
|
|
}
|
|
|
|
function createCodeAgentM3HwlabApiRequestJson(options = {}) {
|
|
return async (targetUrl, request = {}) => {
|
|
const url = new URL(targetUrl);
|
|
const route = url.pathname;
|
|
if (route === M3_IO_CONTROL_ROUTE && request.method === "POST") {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
body: await handleM3IoControl(request.body ?? {}, {
|
|
runtimeStore: options.runtimeStore,
|
|
now: options.now,
|
|
env: options.env,
|
|
requestJson: options.m3IoRequestJson
|
|
})
|
|
};
|
|
}
|
|
if (route === M3_STATUS_ROUTE && request.method === "GET") {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
body: await describeM3StatusLive(options)
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
status: 404,
|
|
body: {
|
|
error: {
|
|
code: "not_found",
|
|
message: `Unsupported Code Agent M3 HWLAB API route ${route}`
|
|
}
|
|
},
|
|
error: `Unsupported Code Agent M3 HWLAB API route ${route}`
|
|
};
|
|
};
|
|
}
|
|
function recordCodeAgentProjectionMetrics(options = {}, { result = null, trace = null, projection = null, refreshError = null } = {}) {
|
|
const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
|
|
if (!performanceStore?.recordWorkbenchProjection || !projection) return;
|
|
const sourceLatestSeq = sourceLatestSeqFromResult(result);
|
|
const lastProjectedSeq = Number(projection.lastProjectedSeq ?? 0);
|
|
if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return;
|
|
performanceStore.recordWorkbenchProjection({
|
|
sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq,
|
|
lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0,
|
|
sourceLatestEventAt: sourceLatestAtFromResult(result, trace),
|
|
projectedLatestEventAt: projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null,
|
|
terminalSourceAt: terminalSourceAtFromResult(result),
|
|
terminalProjectedAt: projection.projectionStatus === "caught-up" ? projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null,
|
|
status: projection?.status ?? "unknown",
|
|
reason: projectionReason(projection, refreshError),
|
|
projectionStatus: projection.projectionStatus ?? "unknown",
|
|
source: "workbench_read_model"
|
|
});
|
|
}
|
|
|
|
function recordCodeAgentTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) {
|
|
const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
|
|
if (!performanceStore?.recordWorkbenchTurnRead) return;
|
|
performanceStore.recordWorkbenchTurnRead({
|
|
route: url?.pathname ?? "/v1/agent/turns/:id",
|
|
status: body?.status ?? statusClassLabel(statusCode),
|
|
degradedReason: turnReadDegradedReason(body),
|
|
durationMs: nowMs() - startedAt,
|
|
responseBytes: responseBodyBytes(body),
|
|
timedOut: statusCode === 504 || body?.error?.code === "timeout" || body?.refreshError?.code === "agentrun_timeout"
|
|
});
|
|
}
|
|
|
|
function turnReadDegradedReason(body = {}) {
|
|
if (body?.refreshError?.code) return body.refreshError.code;
|
|
if (body?.projection?.blocker?.code) return body.projection.blocker.code;
|
|
if (body?.blocker?.code) return body.blocker.code;
|
|
if (body?.error?.code) return body.error.code;
|
|
if (body?.projectionStatus === "projecting") return "projecting";
|
|
if (body?.projectionStatus === "blocked") return "projection_blocked";
|
|
return "none";
|
|
}
|
|
|
|
function responseBodyBytes(body = {}) {
|
|
try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; }
|
|
}
|
|
|
|
function statusClassLabel(value) {
|
|
const status = Number(value);
|
|
if (!Number.isFinite(status) || status <= 0) return "unknown";
|
|
return `${Math.floor(status / 100)}xx`;
|
|
}
|
|
|
|
function sourceLatestSeqFromResult(result = null) {
|
|
const candidates = [
|
|
result?.agentRun?.lastSeq,
|
|
result?.traceSummary?.agentRun?.lastSeq,
|
|
result?.traceSummary?.lastSeq,
|
|
result?.providerTrace?.lastSeq,
|
|
result?.runnerTrace?.lastEvent?.sourceSeq,
|
|
result?.runnerTrace?.lastEvent?.seq,
|
|
result?.runnerTrace?.eventCount
|
|
];
|
|
for (const value of candidates) {
|
|
const seq = Number(value);
|
|
if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq);
|
|
}
|
|
return NaN;
|
|
}
|
|
|
|
function sourceLatestAtFromResult(result = null, trace = null) {
|
|
return result?.traceSummary?.updatedAt
|
|
?? result?.agentRun?.updatedAt
|
|
?? result?.providerTrace?.updatedAt
|
|
?? result?.runnerTrace?.updatedAt
|
|
?? trace?.updatedAt
|
|
?? result?.updatedAt
|
|
?? null;
|
|
}
|
|
|
|
function terminalSourceAtFromResult(result = null) {
|
|
if (!result) return null;
|
|
const terminalStatus = result?.agentRun?.terminalStatus ?? result?.traceSummary?.terminalStatus ?? result?.status;
|
|
if (!isTraceCommandTerminalStatus(terminalStatus)) return null;
|
|
return result?.traceSummary?.updatedAt ?? result?.agentRun?.updatedAt ?? result?.updatedAt ?? null;
|
|
}
|
|
|
|
function projectionReason(projection = null, refreshError = null) {
|
|
if (refreshError) return refreshError.code ?? "refresh_error";
|
|
if (projection?.blocker) return projection.blocker.code ?? "projection_blocked";
|
|
if (projection?.projectionStatus === "projecting") return "projecting";
|
|
if (projection?.projectionStatus === "caught-up") return "none";
|
|
return projection?.projectionStatus ?? "unknown";
|
|
}
|
|
|
|
function nowMs() {
|
|
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
|
return Date.now();
|
|
}
|
|
|
|
function uniqueStrings(values) {
|
|
return [...new Set(values.map((value) => String(value ?? "").trim()).filter(Boolean))];
|
|
}
|