// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // Responsibility: AgentRun v0.1 adapter, incremental Workbench projection cursor sync, and upstream timing projection for Cloud API observability. import { randomUUID } from "node:crypto"; import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { decorateCodeAgentSession } from "./code-agent-session-lifecycle.ts"; import { parsePositiveInteger, safeConversationId, safeOpaqueId, safeSessionId, safeTraceId, text } from "./server-http-utils.ts"; import { currentBackendPerformanceContext } from "./backend-performance.ts"; import { DEFAULT_AGENTRUN_MGR_URL, agentRunJson, isAgentRunManagerInternalServiceHost, resolveAgentRunManagerUrl } from "./code-agent-agentrun-runtime.ts"; import { agentRunErrorWithPolling, agentRunEventCommandId, agentRunMetricStatusForError, agentRunPollingRetryableError, agentRunPollingRootCause, agentRunProjectionPollCount, agentRunProjectionPollingState, agentRunTerminalConsistencyGap, agentRunTerminalResultRefreshSatisfied, agentRunTerminalStatusFromEvents, agentRunTraceCursorSeq, emitAgentRunProjectionPollingSpan, fetchAgentRunEventsForTrace, measuredAgentRunEventsFetch, measuredAgentRunResultFetch, nonNegativeNumber, recordAgentRunEventsProjectionMetrics } from "./code-agent-agentrun-projection-polling.ts"; import { agentRunAlreadyTerminalCancelPayload, agentRunCancelBlockedPayload, agentRunCancelTerminalStatus } from "./code-agent-agentrun-cancel.ts"; import { DEFAULT_AGENTRUN_REPO_URL, agentRunAvailabilityBlocker, agentRunProviderIdOrNull, agentRunTraceMeta, backendForBackendProfile, modelForBackendProfile, providerForBackendProfile, requireAgentRunSourceCommit, resolveAgentRunBackendProfile, resolveAgentRunProviderId, resolveAgentRunRepoUrl } from "./code-agent-agentrun-profile.ts"; import { agentRunPromptEventFields, agentRunPromptFields, agentRunPromptMetadataFromParams, agentRunPromptMetadataFromSession, agentRunPromptMetadataFromText, messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts"; import * as agentRunResult from "./code-agent-agentrun-result.ts"; const ADAPTER_ID = "agentrun-v01"; const ADAPTER_IDS = new Set([ADAPTER_ID, "agentrun-v02"]); const ADAPTER_ALIASES = new Set(["agentrun", "agentrun-v0.1", "agentrun-v0.2"]); const DEFAULT_TENANT_ID = "hwlab"; const DEFAULT_PROJECT_ID = "pikasTech/HWLAB"; const DEFAULT_RUNNER_NAMESPACE = "agentrun-v01"; const DEFAULT_TIMEOUT_MS = 1_200_000; const AGENTRUN_RUNNER_KIND = "agentrun-v01-shared-runner"; const AGENTRUN_SESSION_MODE = "agentrun-v01-durable-session"; const AGENTRUN_IMPLEMENTATION_TYPE = "agentrun-v01-shared-execution-infra"; const AGENTRUN_CAPABILITY_LEVEL = "agentrun-v01-shared-code-agent-session"; const AGENTRUN_PROVIDER_TRACE_PROTOCOL = "agentrun-v01-jsonrpc"; const AGENTRUN_PROVIDER_TRACE_WIRE_API = "agentrun-v01-command-result"; const THREAD_CONTINUITY_POLICY = "hwlab-agentrun-v01-reuse-runner-thread"; const SESSION_POLICY_RUN_LOCAL = "hwlab-agentrun-v01-session-runner-reuse"; const TERMINAL_RUN_STATUSES = new Set(["completed", "failed", "blocked", "cancelled", "canceled"]); const AGENTRUN_DURABLE_TRANSIENT_ENV_RESERVED = new Set([ "AGENTRUN_API_KEY", "AUTH_PASSWORD", "CODEX_API_KEY", "CODEX_HOME", "GH_TOKEN", "GITHUB_TOKEN", "HOME", "OPENAI_API_KEY", "PROVIDER_TOKEN", "UNIDESK_AUTH_PASSWORD", "UNIDESK_PROVIDER_TOKEN", "UNIDESK_SSH_CLIENT_TOKEN" ]); const HWLAB_RESOURCE_GIT_BUNDLES = Object.freeze([ Object.freeze({ name: "hwlab-tools", subpath: "tools", target_path: "tools" }), Object.freeze({ name: "hwlab-agent-skills", subpath: "skills", target_path: ".agents/skills" }) ]); const HWLAB_RESOURCE_PROMPT_REFS = Object.freeze([ Object.freeze({ name: "hwlab-v02-runtime", path: "internal/agent/prompts/hwlab-v02-runtime.md", inject: "thread-start", required: true }) ]); const { isRetryableAgentRunTransportFailureKind, agentRunSessionEvidence, agentRunFailureRequiresFreshSession, agentRunFreshSessionFailure, providerCredentialSecretRef, agentRunProviderTrace, agentRunFailureAttribution, agentRunLongLivedSessionGate, agentRunToolCalls, agentRunReplyText, agentRunResultToCodeAgentPayload, agentRunCompletedFinalResponse, agentRunTraceTerminalAssistantText, isAgentRunAssistantTraceEvent, agentRunTerminalFailureFinalResponse, agentRunTerminalFailureTraceSummary, agentRunCompletedTraceSummary, partialAgentRunContext, agentRunResultTraceCreatedAt, decorateAgentRunRunningResult, appendAgentRunEventsToTrace, isForeignAgentRunCommandEvent, mapAgentRunEvent, agentRunProviderRetryEvent, providerRetryMessage, isRetryableProviderInterruptionPayload, normalizedFailureKind, numberOrNull, agentRunTraceEvent, agentRunManualEventType, isProtocolToolLifecycleMethod, agentRunToolCallPreviewSummary, jsonPreviewStringField, agentRunBackendStatusDetails, agentRunResourceSummary, compactObject, agentRunCommandExecutionEvent, nowMs, agentRunMapping, agentRunReusedMapping, agentRunResultRefs, withAgentRunRunnerJobCount, hwlabSessionIdForParams, scopedAgentRunSessionIdForParams, agentRunSessionId, agentRunEffectiveThreadId, agentRunSessionSummary, agentRunThreadReused, agentRunSessionReuseSummary, agentRunRunnerSummary, hostForUrl, waitingForForPhase, textPayload, requiredString, firstNonEmpty, nowIso, timestampIsoOrNow, redactUrl } = agentRunResult; export { agentRunSessionEvidence } from "./code-agent-agentrun-result.ts"; export function codeAgentAgentRunAdapterEnabled(env = process.env) { const value = String(env.HWLAB_CODE_AGENT_ADAPTER ?? env.HWLAB_CODE_AGENT_PROVIDER ?? "").trim().toLowerCase(); return ADAPTER_IDS.has(value) || ADAPTER_ALIASES.has(value); } export function describeAgentRunAdapterAvailability(env = process.env, options = {}) { const params = options.params ?? {}; const backendProfile = resolveAgentRunBackendProfile(env, params); const provider = providerForBackendProfile(backendProfile); const backend = `${ADAPTER_ID}/${backendProfile}`; const model = modelForBackendProfile(backendProfile, env); const blockers = []; let managerUrl = null; let repoUrl = null; let resourceBundleSourceCommit = null; try { managerUrl = resolveAgentRunManagerUrl(env); } catch (error) { blockers.push(agentRunAvailabilityBlocker(error, "manager-url")); } try { repoUrl = resolveAgentRunRepoUrl(env); } catch (error) { blockers.push(agentRunAvailabilityBlocker(error, "repo-url")); } try { resourceBundleSourceCommit = requireAgentRunSourceCommit(env); } catch (error) { blockers.push(agentRunAvailabilityBlocker(error, "resource-bundle-source-commit")); } let providerId = null; try { providerId = resolveAgentRunProviderId(env); } catch (error) { blockers.push(agentRunAvailabilityBlocker(error, "provider-id")); } const runnerNamespace = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE, env.AGENTRUN_RUNTIME_NAMESPACE, DEFAULT_RUNNER_NAMESPACE); const ready = blockers.length === 0 && Boolean(providerId && runnerNamespace && backendProfile && managerUrl && repoUrl); const managerHost = hostForUrl(managerUrl); const repoHost = hostForUrl(repoUrl); const status = ready ? "agentrun-ready" : "blocked"; const capabilityLevel = ready ? AGENTRUN_CAPABILITY_LEVEL : "blocked"; return { endpoint: "POST /v1/agent/chat", adapter: ADAPTER_ID, provider, model, backend, infrastructureBackend: backend, mode: ADAPTER_ID, schema: [ "conversationId", "sessionId", "messageId", "status", "createdAt", "updatedAt", "traceId", "provider", "model", "backend", "infrastructureBackend", "agentRun", "runner", "runnerTrace", "session", "sessionMode", "sessionReuse", "threadContinuityPolicy", "error.message", "error.code" ], runner: { kind: AGENTRUN_RUNNER_KIND, adapter: ADAPTER_ID, backend, provider, model, namespace: runnerNamespace, managerHost, repoHost, mode: ADAPTER_ID, sessionMode: AGENTRUN_SESSION_MODE, implementationType: AGENTRUN_IMPLEMENTATION_TYPE, status: ready ? "available" : "blocked", ready, capabilityLevel, longLivedSession: ready, durableSession: ready, durable: ready, codexStdio: false, delegatedToAgentRun: true, writeCapable: ready, readOnly: false, runnerLimitations: ready ? [] : ["agentrun-adapter-config-blocked"], secretMaterialRead: false, valuesRedacted: true }, agentRun: { adapter: ADAPTER_ID, backendProfile, providerId, runnerNamespace, managerUrl, managerHost, repoUrl, repoHost, resourceBundleSourceCommit, managerConfigured: Boolean(managerUrl), repoConfigured: Boolean(repoUrl), internalServiceDns: isAgentRunManagerInternalServiceHost(managerHost), ready, valuesPrinted: false }, status, readinessStatus: status, agentKind: "agentrun-v01-adapter", capabilityStatus: status, blocker: ready ? null : blockers[0]?.summary ?? "AgentRun adapter runtime configuration is blocked.", reason: ready ? null : blockers[0]?.code ?? "agentrun_adapter_blocked", summary: ready ? "HWLAB Code Agent delegates execution to AgentRun v0.1 shared infrastructure; hwlab-cloud-api keeps session/trace/API ownership and no longer manages a repo-owned runner control plane." : "HWLAB Code Agent is configured for AgentRun v0.1, but the adapter runtime configuration is incomplete or invalid.", missingEnv: [], secretRefs: [], egress: { ready, mode: "agentrun-managed-provider-egress", directPublicOpenAi: false, secretMaterialRead: false, valuesRedacted: true }, safety: { secretsRead: false, secretValuesPrinted: false, kubeconfigRead: false, providerCredentialsDelegatedToAgentRun: true, repoOwnedRunnerControlPlane: false, valuesRedacted: true }, capabilityLevel, workspace: repoUrl, sandbox: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SANDBOX, env.HWLAB_CODE_AGENT_CODEX_SANDBOX, "danger-full-access"), session: null, sessionStatus: null, sessionMode: AGENTRUN_SESSION_MODE, implementationType: AGENTRUN_IMPLEMENTATION_TYPE, runnerLimitations: ready ? [] : ["agentrun-adapter-config-blocked"], longLivedSessionGate: { status: ready ? "pass" : "blocked", provider, runnerKind: AGENTRUN_RUNNER_KIND, sessionMode: AGENTRUN_SESSION_MODE, implementationType: AGENTRUN_IMPLEMENTATION_TYPE, adapter: ADAPTER_ID, codexStdio: false, delegatedToAgentRun: true, blockers, secretMaterialRead: false, valuesRedacted: true }, blockers, blockerCodes: blockers.map((blocker) => blocker.code).filter(Boolean), ready, partialReady: false }; } export function initialAgentRunChatResult({ params = {}, options = {}, traceId }) { const env = options.env ?? process.env; const timestamp = nowIso(options.now); const backendProfile = resolveAgentRunBackendProfile(env, params); const promptEvidence = agentRunPromptMetadataFromParams(params, traceId); return { accepted: true, status: "running", shortConnection: true, traceId, ...codeAgentOtelTraceFields(traceId, env), projectId: firstNonEmpty(params.projectId) || null, conversationId: safeConversationId(params.conversationId) || null, sessionId: safeSessionId(params.sessionId) || agentRunSessionId(traceId), threadId: safeOpaqueId(params.threadId) || null, messageId: `msg_${safeTraceId(traceId)?.slice(4) || randomUUID()}`, ...agentRunAdmissionTimingFields(timestamp), provider: providerForBackendProfile(backendProfile), model: modelForBackendProfile(backendProfile, env), backend: backendForBackendProfile(backendProfile), infrastructureBackend: `agentrun-v01/${backendProfile}`, capabilityLevel: AGENTRUN_CAPABILITY_LEVEL, sessionMode: AGENTRUN_SESSION_MODE, implementationType: AGENTRUN_IMPLEMENTATION_TYPE, threadContinuityPolicy: THREAD_CONTINUITY_POLICY, ...agentRunPromptFields(promptEvidence), agentRun: { adapter: ADAPTER_ID, status: "pending-dispatch", backendProfile, providerId: resolveAgentRunProviderId(env), managerUrl: resolveAgentRunManagerUrl(env), repoUrl: resolveAgentRunRepoUrl(env), resourceBundleSourceCommit: requireAgentRunSourceCommit(env), valuesPrinted: false }, valuesPrinted: false }; } function agentRunAdmissionTimingFields(value) { const admittedAt = timestampIsoOrNow(value); const timing = { startedAt: admittedAt, lastEventAt: admittedAt, finishedAt: null, durationMs: null, observedAt: admittedAt, lastEventAgeMs: 0, valuesRedacted: true }; return { createdAt: admittedAt, updatedAt: admittedAt, timing, startedAt: admittedAt, lastEventAt: admittedAt, finishedAt: null, durationMs: null }; } export async function submitAgentRunChatTurn({ params = {}, options = {}, traceId, traceStore = defaultCodeAgentTraceStore, results }) { const env = options.env ?? process.env; const managerUrl = resolveAgentRunManagerUrl(env); const backendProfile = resolveAgentRunBackendProfile(env, params); const fetchImpl = options.fetchImpl ?? globalThis.fetch; const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); const startedAt = nowIso(options.now); const toolCapabilities = await resolveToolCapabilities({ params, options }); const promptEvidence = agentRunPromptMetadataFromParams(params, traceId); traceStore.ensure(traceId, agentRunTraceMeta(env, params)); traceStore.append(traceId, agentRunTraceEvent({ type: "request", status: "accepted", label: "agentrun:request:accepted", message: "HWLAB Code Agent request accepted by the AgentRun adapter; a warm runner is reused when valid, otherwise AgentRun durably admits command dispatch before this request returns.", waitingFor: "agentrun-run-reuse-check", adapter: ADAPTER_ID, managerHost: new URL(managerUrl).hostname, ...codeAgentOtelTraceFields(traceId, env), ...agentRunPromptEventFields(promptEvidence), valuesPrinted: false }, backendProfile), agentRunTraceMeta(env, params)); const forceFreshSession = agentRunFreshSessionRequested(params); if (forceFreshSession) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:reuse-skipped-fresh-session", message: "Stored Workbench continuation was marked stale; hwlab-cloud-api will create a fresh AgentRun session/run/thread for this turn instead of reusing a warm run.", sessionId: hwlabSessionIdForParams(params, traceId), backendProfile, waitingFor: "agentrun-session-reset", valuesPrinted: false }, backendProfile)); } const reusable = forceFreshSession ? null : await resolveReusableAgentRun({ params, options, env, managerUrl, fetchImpl, timeoutMs, backendProfile, traceId, traceStore }); if (reusable?.mapping) { const reusableParams = paramsWithAgentRunSessionThread(params, reusable.mapping.threadId); const commandInput = buildAgentRunCommandInput({ params: reusableParams, traceId, backendProfile, sessionId: reusable.sessionId }); let command = null; let commandId = null; try { command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: `/api/v1/runs/${encodeURIComponent(reusable.mapping.runId)}/commands`, method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId: reusable.mapping.runId, stage: "command-create-reused", terminalOnFailure: false }); commandId = requiredString(command?.id, "command.id"); } catch (error) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:reuse-command-failed", message: `AgentRun reused run ${reusable.mapping.runId} rejected the new command; hwlab-cloud-api will create a fresh run and ask AgentRun manager to admit its durable runner dispatch.`, errorCode: error?.code ?? "agentrun_reuse_command_failed", runId: reusable.mapping.runId, commandId: reusable.mapping.commandId ?? null, waitingFor: "agentrun-run-create", valuesPrinted: false }, backendProfile)); command = null; commandId = null; } if (command && commandId) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:command:created", message: `AgentRun command ${commandId} created on reused run ${reusable.mapping.runId}; the active runner will pick it up without creating another runner Job.`, runId: reusable.mapping.runId, commandId, backendProfile, waitingFor: "agentrun-result", valuesPrinted: false }, backendProfile)); const mapping = agentRunReusedMapping({ previous: reusable.mapping, run: reusable.run, command, traceId, startedAt, backendProfile, managerUrl, env }); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:runner-job:reused", message: `AgentRun runner Job ${mapping.jobName ?? "unknown"} remains active for this HWLAB session turn; no replacement Job was created.`, runId: mapping.runId, commandId: mapping.commandId, attemptId: mapping.attemptId, runnerId: mapping.runnerId, jobName: mapping.jobName, namespace: mapping.namespace, waitingFor: "agentrun-result", valuesPrinted: false }, mapping)); return decorateAgentRunRunningResult({ base: initialAgentRunChatResult({ params: reusableParams, options, traceId }), mapping, traceStore, traceId }); } } const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId); let sessionId = baseSessionId; let sessionReset = forceFreshSession; let finalDispatchParams = params; let run, command, mapping; for (let attempt = 0; attempt < 2; attempt += 1) { let attemptParams = params; if (sessionReset) { sessionId = newSessionIdAfterEviction(baseSessionId, traceId); attemptParams = paramsWithAgentRunSessionThread(params, null); const resetReason = firstNonEmpty(params.forceFreshAgentRunSessionReason, "session-store-evicted"); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:session-reset", message: "AgentRun continuation requires a fresh session (" + resetReason + "); hwlab-cloud-api is creating sessionId " + sessionId + " with threadId=null.", sessionId, previousSessionId: baseSessionId, resetReason, backendProfile, waitingFor: "agentrun-run-create", valuesPrinted: false, }, backendProfile)); } let ensuredSession = null; try { ensuredSession = await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }); } catch (error) { if (attempt === 0 && await shouldResetSessionAfterEviction("session-store-evicted", error?.message)) { sessionReset = true; continue; } throw error; } const ensuredSessionRecord = agentRunSessionRecordFromEnsure(ensuredSession); if (attempt === 0 && agentRunSessionThreadInvalidated(ensuredSessionRecord)) { sessionReset = true; traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:profile-session-reset", message: "AgentRun profile-scoped session " + sessionId + " is terminal after a thread/resume failure; hwlab-cloud-api will create a fresh profile sessionId instead of letting AgentRun manager backfill a stale threadId.", sessionId, previousSessionId: baseSessionId, backendProfile, waitingFor: "agentrun-session-reset", valuesPrinted: false }, backendProfile)); continue; } const profileThreadId = agentRunSessionThreadIdFromEnsure(ensuredSession, backendProfile); const dispatchParams = paramsWithAgentRunSessionThread(attemptParams, profileThreadId); if (safeOpaqueId(attemptParams.threadId) && !profileThreadId) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:profile-session-thread-reset", message: "AgentRun profile-scoped session " + sessionId + " has no threadId; hwlab-cloud-api will start a new thread instead of resuming a thread from another backend profile.", sessionId, backendProfile, waitingFor: "agentrun-run-create", valuesPrinted: false }, backendProfile)); } const runInput = buildAgentRunCreateRunInput({ params: dispatchParams, env, traceId, backendProfile, sessionId, toolCapabilities }); const runCreateStartedAt = Date.now(); run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create" }); const runId = requiredString(run?.id, "run.id"); void emitCodeAgentOtelSpan("agentrun_run_create", traceId, env, { startTimeMs: runCreateStartedAt, attributes: { runId, backendProfile, managerUrl } }); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:created", message: "AgentRun run " + runId + " created through internal k3s Service DNS.", runId, backendProfile, waitingFor: "agentrun-command-create", valuesPrinted: false, }, backendProfile)); const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }); const commandInput = buildAgentRunCommandInput({ params: dispatchParams, traceId, backendProfile, sessionId, dispatchInput }); const commandCreateStartedAt = Date.now(); command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create" }); const commandId = requiredString(command?.id, "command.id"); const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent); void emitCodeAgentOtelSpan("agentrun_command_create", traceId, env, { startTimeMs: commandCreateStartedAt, attributes: { runId, commandId, backendProfile, managerUrl } }); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:command:created", message: "AgentRun command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.", runId, commandId, dispatchIntentId: dispatchIntent.id, runnerJobId: dispatchIntent.runnerJobId, backendProfile, waitingFor: "agentrun-result", valuesPrinted: false, }, backendProfile)); mapping = agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: dispatchParams }); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:dispatch-intent:admitted", message: "AgentRun owns durable runner dispatch; HWLAB has no process-local runner Job creation window.", runId: mapping.runId, commandId: mapping.commandId, backendProfile, dispatchIntentId: dispatchIntent.id, dispatchIntentState: dispatchIntent.state, runnerJobId: dispatchIntent.runnerJobId, waitingFor: "agentrun-result", valuesPrinted: false }, mapping)); finalDispatchParams = dispatchParams; break; } return decorateAgentRunRunningResult({ base: initialAgentRunChatResult({ params: finalDispatchParams, options, traceId }), mapping, traceStore, traceId }); } async function agentRunDispatchJson({ fetchImpl, managerUrl, path, method = "GET", body = undefined, timeoutMs, env = process.env, traceStore = defaultCodeAgentTraceStore, traceId, backendProfile, runId = null, commandId = null, stage = "agentrun-dispatch", terminalOnFailure = true } = {}) { const policy = agentRunDispatchRetryPolicy(env); let retryAttempt = 0; for (;;) { try { return await agentRunJson(fetchImpl, managerUrl, path, { method, body, timeoutMs, env }); } catch (error) { const failureKind = agentRunDispatchFailureKind(error); const retryable = agentRunDispatchRetryable(error, failureKind); if (!retryable || retryAttempt >= policy.maxRetries) { traceStore.append(traceId, agentRunTraceEvent({ type: "error", eventType: "error", status: terminalOnFailure ? "failed" : "blocked", label: retryable ? `agentrun:dispatch-retry-exhausted:${failureKind}` : `agentrun:dispatch-failed:${failureKind}`, errorCode: failureKind, failureKind, retryable, willRetry: false, retryAttempt, retryMax: policy.maxRetries, retryExhausted: retryable && retryAttempt >= policy.maxRetries, runId, commandId, waitingFor: terminalOnFailure ? "agentrun-dispatch-failed" : "agentrun-dispatch-recovery", message: retryable ? `AgentRun dispatch transient ${failureKind}; retry ${retryAttempt}/${policy.maxRetries} exhausted at stage ${stage}. ${terminalOnFailure ? "Code Agent stopped this turn." : "HWLAB will continue with the next recovery path for this turn."}` : `AgentRun dispatch failed with non-retryable ${failureKind} at stage ${stage}. ${terminalOnFailure ? "Code Agent stopped this turn." : "HWLAB will continue with the next recovery path for this turn."}`, terminal: terminalOnFailure, valuesPrinted: false }, backendProfile)); throw Object.assign(error, { retryAttempt, retryMax: policy.maxRetries, failureKind, retryExhausted: retryable && retryAttempt >= policy.maxRetries }); } retryAttempt += 1; const retryDelayMs = agentRunDispatchRetryDelayMs(policy, retryAttempt); const nextRetryAt = new Date(Date.now() + retryDelayMs).toISOString(); traceStore.append(traceId, agentRunTraceEvent({ type: "provider_retry", eventType: "backend", status: "retrying", label: `agentrun:dispatch-retry:${failureKind}`, errorCode: failureKind, failureKind, retryable: true, willRetry: true, retryAttempt, retryMax: policy.maxRetries, retryDelayMs, nextRetryAt, runId, commandId, waitingFor: "agentrun-dispatch-retry", message: `AgentRun dispatch transient ${failureKind}; retry ${retryAttempt}/${policy.maxRetries} after ${retryDelayMs}ms at stage ${stage}.`, terminal: false, valuesPrinted: false }, backendProfile)); await sleepAgentRunDispatchRetry(retryDelayMs); } } } function agentRunDispatchRetryPolicy(env = process.env) { return { maxRetries: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX, 5), baseDelayMs: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_BASE_MS, 1000), maxDelayMs: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX_DELAY_MS, 30000) }; } function agentRunDispatchRetryDelayMs(policy, retryAttempt) { const base = Math.max(1, Number(policy.baseDelayMs ?? 1000)); const cap = Math.max(base, Number(policy.maxDelayMs ?? 30000)); return Math.min(cap, Math.trunc(base * Math.pow(2, Math.max(0, Number(retryAttempt ?? 1) - 1)))); } function agentRunDispatchRetryable(error, failureKind) { const status = Number(error?.statusCode ?? error?.status ?? error?.agentRunError?.statusCode ?? 0); if ([429, 502, 503, 504].includes(status)) return true; if (["provider-unavailable", "provider-stream-disconnected"].includes(failureKind)) return true; if (isRetryableAgentRunTransportFailureKind(failureKind)) return true; return /ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|fetch failed|network|timeout/iu.test(String(error?.message ?? "")); } function agentRunDispatchFailureKind(error) { return normalizedFailureKind(error?.agentRunError?.failureKind ?? error?.failureKind ?? error?.code ?? error?.name ?? (Number(error?.statusCode) ? `http-${error.statusCode}` : "agentrun-dispatch-error")); } function sleepAgentRunDispatchRetry(ms) { return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); } export async function syncAgentRunChatResult({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore, appendResultEvent = true, refreshEvents = true, forceResultSync = false, retryParams = null }) { const initial = currentResult ?? await loadPersistedAgentRunResult(traceId, options); if (isAgentRunUserCancelSealed(initial)) { const runnerTrace = initial.runnerTrace ?? traceStore.snapshot(traceId); return { result: initial, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true }; } const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial; if (isAgentRunUserCancelSealed(mapped)) { const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId); return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true }; } const env = options.env ?? process.env; const performanceStore = options.backendPerformanceStore ?? currentBackendPerformanceContext(); if (!forceResultSync && agentRunTerminalResultRefreshSatisfied(mapped, traceId)) { const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId); const polling = agentRunProjectionPollingState({ env, terminalFromEvents: "completed", rootCause: "terminal_result_sealed", terminalConsistencyGap: 0 }); emitAgentRunProjectionPollingSpan({ traceId, env, agentRun: mapped?.agentRun, polling, terminalRefreshSkipped: true }); return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, polling }; } if (!mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return { result: currentResult, runnerTrace: traceStore.snapshot(traceId), found: Boolean(currentResult), eventsRefreshed: false, resultSynced: false }; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); const projectionState = await loadWorkbenchProjectionStateForAgentRun(options.runtimeStore, { traceId, agentRun: mapped.agentRun }); const projectionRetryParams = retryParams ?? projectionState?.retryParams ?? null; const projectionPollCount = agentRunProjectionPollCount(projectionState); const fetchPlan = buildAgentRunProjectionEventsFetchPlan({ projectionState, agentRun: mapped.agentRun, pageLimit: env.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT }); const previousLastSeq = Number(fetchPlan.afterSeq ?? mapped.agentRun.lastSeq ?? 0); let eventsResponse = null; try { eventsResponse = refreshEvents ? await measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, lastSeq: fetchPlan.afterSeq, afterSeqOverride: fetchPlan.afterSeq, eventsPageLimit: fetchPlan.limit, traceSummary: mapped.traceSummary, pollCount: projectionPollCount, backoffMs: nonNegativeNumber(projectionState?.backoffMs), rootCause: projectionState?.rootCause ?? null, terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) } }) : null; } catch (error) { const polling = agentRunProjectionPollingState({ projectionState, env, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) }); await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { currentState: projectionState, result: mapped, agentRun: { ...mapped.agentRun, lastSeq: previousLastSeq }, eventsResponse: null, resultSyncState: projectionState?.resultSyncState ?? null, projectionStatus: "degraded", projectionHealth: "degraded", retryParams: projectionRetryParams, polling, error, now: options.now }); throw agentRunErrorWithPolling(error, polling); } if (eventsResponse) appendAgentRunEventsToTrace(traceStore, traceId, eventsResponse.events, mapped.agentRun); const nextLastSeq = eventsResponse ? agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq) : mapped.agentRun.lastSeq; const backendProfile = resolveAgentRunBackendProfile(env, { providerProfile: firstNonEmpty( projectionRetryParams?.providerProfile, projectionRetryParams?.codeAgentProviderProfile, projectionRetryParams?.backendProfile, mapped.agentRun.backendProfile ) }); recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse, previousLastSeq, nextLastSeq); const terminalFromEvents = agentRunTerminalStatusFromEvents(eventsResponse?.events); const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0)); const terminalConsistencyGap = agentRunTerminalConsistencyGap({ mapped, eventsResponse }); const projectionPolling = agentRunProjectionPollingState({ projectionState, env, pollCount: projectionPollCount, cursorAdvance, terminalFromEvents, rootCause: terminalFromEvents ? "terminal_from_events" : cursorAdvance > 0 ? "cursor_advanced" : "projection_no_progress", terminalConsistencyGap }); await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { currentState: projectionState, result: mapped, agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq }, eventsResponse, resultSyncState: terminalFromEvents ? "pending" : null, projectionStatus: terminalFromEvents ? "terminal" : null, polling: projectionPolling, retryParams: projectionRetryParams, now: options.now }); const deferResultSync = terminalFromEvents ? false : agentRunResultSyncDeferred({ forceResultSync, options, env }); if (!forceResultSync && (deferResultSync || !agentRunCommandResultSyncRequired(mapped, eventsResponse))) { const nextMapping = withAgentRunRunnerJobCount({ ...mapped.agentRun, lastSeq: nextLastSeq, status: mapped.agentRun.status ?? "running", runStatus: mapped.agentRun.runStatus ?? null, commandState: mapped.agentRun.commandState ?? null, terminalStatus: null, resultSyncState: terminalFromEvents ? "pending" : mapped.agentRun.resultSyncState ?? null, updatedAt: nowIso(options.now), valuesPrinted: false }); const payload = decorateAgentRunRunningResult({ base: { ...mapped, status: "running", agentRun: nextMapping, updatedAt: nowIso(options.now) }, mapping: nextMapping, traceStore, traceId }); options.codeAgentChatResults?.set?.(traceId, payload); return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, polling: projectionPolling }; } let result; try { result = await measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped, eventsResponse }); } catch (error) { const retryable = agentRunPollingRetryableError(error); const polling = agentRunProjectionPollingState({ projectionState, env, pollCount: projectionPollCount, cursorAdvance, terminalFromEvents, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap }); await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { currentState: projectionState, result: mapped, agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq }, eventsResponse, resultSyncState: retryable ? "pending" : agentRunMetricStatusForError(error) === "timeout" ? "timed_out" : "failed", projectionStatus: retryable ? "degraded" : terminalFromEvents ? "terminal" : null, projectionHealth: "degraded", retryParams: projectionRetryParams, polling, error, now: options.now }); throw agentRunErrorWithPolling(error, polling); } const nextMapping = withAgentRunRunnerJobCount({ ...mapped.agentRun, ...agentRunResultRefs(result), lastSeq: nextLastSeq, status: result?.status ?? mapped.agentRun.status ?? "running", runStatus: result?.runStatus ?? mapped.agentRun.runStatus ?? null, commandState: result?.commandState ?? mapped.agentRun.commandState ?? null, terminalStatus: result?.terminalStatus ?? mapped.agentRun.terminalStatus ?? null, updatedAt: nowIso(options.now), valuesPrinted: false }); const freshSessionRetry = await retryAgentRunTurnAfterFreshSessionFailure({ params: projectionRetryParams, options, traceId, traceStore, mapped, failedResult: result, failedMapping: nextMapping, env, managerUrl, fetchImpl, timeoutMs, backendProfile }); if (freshSessionRetry) { options.codeAgentChatResults?.set?.(traceId, freshSessionRetry.payload); await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { currentState: projectionState, result: freshSessionRetry.payload, agentRun: freshSessionRetry.mapping, eventsResponse: null, resultSyncState: null, projectionStatus: "projecting", retryParams: projectionRetryParams, now: options.now }); const retrySync = await syncAgentRunChatResult({ traceId, currentResult: freshSessionRetry.payload, options, traceStore, appendResultEvent, refreshEvents: true, forceResultSync, retryParams: projectionRetryParams }); if (retrySync?.result) { return { ...retrySync, found: true, eventsRefreshed: Boolean(eventsResponse) || Boolean(retrySync.eventsRefreshed), freshSessionRetried: true }; } return { result: freshSessionRetry.payload, runnerTrace: freshSessionRetry.payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, freshSessionRetried: true }; } const base = { ...mapped, agentRun: nextMapping, updatedAt: nowIso(options.now) }; const payload = agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent }); options.codeAgentChatResults?.set?.(traceId, payload); const resultPolling = agentRunProjectionPollingState({ projectionState, env, pollCount: projectionPollCount, cursorAdvance, terminalFromEvents: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? (nextMapping.terminalStatus ?? payload?.status) : terminalFromEvents, rootCause: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? "terminal_result_synced" : "result_synced", terminalConsistencyGap: agentRunTerminalConsistencyGap({ mapped: payload, eventsResponse, result }) }); await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { currentState: projectionState, result: payload, agentRun: nextMapping, eventsResponse, resultSyncState: "synced", projectionStatus: isAgentRunTerminalStatus(payload?.status) ? "terminal" : null, polling: resultPolling, retryParams: projectionRetryParams, now: options.now }); return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: true, polling: resultPolling }; } async function loadWorkbenchProjectionStateForAgentRun(runtimeStore, { traceId, agentRun = {} } = {}) { if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null; try { const result = await runtimeStore.getWorkbenchProjectionState({ traceId }); const state = result?.projectionState ?? null; if (!state) return null; if (text(state.runId ?? state.sourceRunId) !== text(agentRun.runId)) return null; if (text(state.commandId ?? state.sourceCommandId) !== text(agentRun.commandId)) return null; return state; } catch { return null; } } async function writeWorkbenchProjectionStateForAgentRun(runtimeStore, params = {}) { if (!runtimeStore || typeof runtimeStore.writeWorkbenchProjectionState !== "function") return null; const projectionState = buildWorkbenchProjectionStateUpdate(params); if (!projectionState.traceId || !projectionState.runId || !projectionState.commandId) return null; return await runtimeStore.writeWorkbenchProjectionState({ projectionState }); } function agentRunCommandResultSyncRequired(mapped = {}, eventsResponse = null) { const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {}; if (agentRunTerminalResultRefreshSatisfied(mapped, mapped?.traceId)) return true; if (isAgentRunTerminalStatus(agentRun.terminalStatus ?? agentRun.commandState)) return true; return Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events)); } function isAgentRunTerminalStatus(value) { const status = normalizeAgentRunStatus(value); return status === "timeout" || TERMINAL_RUN_STATUSES.has(status); } function normalizeAgentRunStatus(value) { const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); return status === "cancelled" ? "canceled" : status; } function isAgentRunUserCancelSealed(payload = null) { if (!payload || typeof payload !== "object") return false; const status = normalizeAgentRunStatus(payload.status); const cancelStatus = normalizeAgentRunStatus(payload.cancelStatus ?? payload.cancelDisposition?.status); const terminalStatus = normalizeAgentRunStatus(payload.agentRun?.terminalStatus ?? payload.agentRun?.commandState ?? payload.agentRun?.status); return payload.canceled === true || status === "canceled" || cancelStatus === "canceled" || terminalStatus === "canceled"; } async function resolveAgentRunTraceCommandMapping({ traceId, mapped, options = {} }) { const safeId = safeTraceId(traceId); const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null; if (!safeId || !agentRun?.runId) return mapped; const agentRunTraceId = safeTraceId(agentRun.traceId); const providerTraceId = safeTraceId(agentRun.providerTrace?.traceId); const traceSummaryCommandId = text(mapped?.traceSummary?.agentRun?.commandId); const commandId = text(agentRun.commandId); const traceFieldsMatch = agentRunTraceId === safeId && providerTraceId === safeId && (!traceSummaryCommandId || traceSummaryCommandId === commandId); if (traceFieldsMatch && (mapped.status === "running" || agentRunTerminalResultRefreshSatisfied(mapped, safeId))) return mapped; const env = options.env ?? process.env; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, agentRun.managerUrl); const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); const command = await findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId: agentRun.runId, traceId: safeId }); if (!command?.id) { throw Object.assign(new Error(`AgentRun command registry has no command for ${safeId} in run ${agentRun.runId}`), { code: "agentrun_trace_command_not_found", statusCode: 404, traceId: safeId, runId: agentRun.runId }); } if (command.id === commandId) { return { ...mapped, finalResponse: null, traceSummary: null, agentRun: { ...agentRun, traceId: safeId, lastSeq: 0, providerTrace: null, commandState: command.state ?? agentRun.commandState ?? null, updatedAt: nowIso(options.now), valuesPrinted: false } }; } return { ...mapped, finalResponse: null, traceSummary: null, agentRun: { ...agentRun, commandId: command.id, traceId: safeId, lastSeq: 0, providerTrace: null, commandState: command.state ?? agentRun.commandState ?? null, updatedAt: nowIso(options.now), valuesPrinted: false } }; } export async function refreshAgentRunTrace({ traceId, result = null, options = {}, traceStore = defaultCodeAgentTraceStore }) { const initial = result ?? await loadPersistedAgentRunResult(traceId, options); const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial; if (!mapped?.agentRun?.runId) return traceStore.snapshot(traceId); const env = options.env ?? process.env; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); const eventsResponse = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, traceSummary: mapped.traceSummary } }); const events = eventsResponse.events; appendAgentRunEventsToTrace(traceStore, traceId, events, mapped.agentRun); const lastSeq = agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq); if (lastSeq !== Number(mapped.agentRun.lastSeq ?? 0)) { options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun: { ...mapped.agentRun, lastSeq, updatedAt: nowIso(options.now) } }); } return traceStore.snapshot(traceId); } export async function cancelAgentRunChatTurn({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore }) { let mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options); if (!mapped?.agentRun?.commandId) return null; const env = options.env ?? process.env; const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); const cancelTimeoutMs = Math.min(timeoutMs, parsePositiveInteger(env.HWLAB_CODE_AGENT_CANCEL_TIMEOUT_MS, 5_000)); const cancelOptions = { ...options, env: { ...env, HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: String(cancelTimeoutMs), HWLAB_CODE_AGENT_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs) } }; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); try { const synced = await syncAgentRunChatResult({ traceId, currentResult: mapped, options: cancelOptions, traceStore, forceResultSync: true }); mapped = synced?.result ?? synced ?? mapped; } catch { // Cancel must still expose the command cancel disposition; result refresh failures are reported by the normal trace/result paths. } const terminalStatus = agentRunCancelTerminalStatus(mapped); if (terminalStatus) return agentRunAlreadyTerminalCancelPayload({ traceId, payload: mapped, terminalStatus, options: cancelOptions, traceStore }); const cancelPath = `/api/v1/commands/${encodeURIComponent(mapped.agentRun.commandId)}/cancel`; try { await agentRunJson(fetchImpl, managerUrl, cancelPath, { method: "POST", body: { reason: "hwlab-user-cancel", traceId }, timeoutMs: cancelTimeoutMs, env }); } catch (error) { return agentRunCancelBlockedPayload({ traceId, payload: mapped, error, cancelPath, options: cancelOptions, traceStore }); } traceStore.append(traceId, agentRunTraceEvent({ type: "cancel", status: "canceled", label: "agentrun:cancel:canceled", message: "HWLAB forwarded cancel to AgentRun command cancel API.", runId: mapped.agentRun.runId, commandId: mapped.agentRun.commandId, terminal: true, valuesPrinted: false }, mapped.agentRun)); const now = nowIso(options.now); const agentRun = { ...mapped.agentRun, status: "cancelled", commandState: "cancelled", terminalStatus: "cancelled", updatedAt: now }; const payload = { ...mapped, status: "canceled", canceled: true, updatedAt: now, agentRun, runnerTrace: traceStore.snapshot(traceId), error: { code: "agentrun_canceled", layer: "agentrun", category: "canceled", retryable: true, message: "AgentRun command was canceled by user request", userMessage: "当前 AgentRun 请求已取消；traceId/runId/commandId 已保留，可重试上一条消息。", traceId, route: "/v1/agent/chat/cancel", toolName: "agentrun.command.cancel" }, userMessage: "当前 AgentRun 请求已取消；traceId/runId/commandId 已保留，可重试上一条消息。" }; options.codeAgentChatResults?.set?.(traceId, payload); return payload; } export async function steerAgentRunChatTurn({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore }) { const mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options); if (!safeTraceId(traceId) || !mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return null; const message = firstNonEmpty(params.message, params.prompt, params.text); if (!message) { throw Object.assign(new Error("steer command requires non-empty message text"), { code: "steer_message_missing", statusCode: 400 }); } const env = options.env ?? process.env; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); const steerTraceId = safeTraceId(params.steerTraceId ?? params.steerId) || `trc_steer_${randomUUID().replace(/-/gu, "")}`; traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:steer:accepted", message: "HWLAB accepted an in-flight steer request and will create an AgentRun type=steer command on the existing run.", runId: mapped.agentRun.runId, commandId: mapped.agentRun.commandId, steerTraceId, waitingFor: "agentrun-steer-command-create", valuesPrinted: false }, mapped.agentRun)); const commandInput = buildAgentRunSteerCommandInput({ params: { ...params, message }, traceId, steerTraceId, mapped }); const command = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(mapped.agentRun.runId)}/commands`, { method: "POST", body: commandInput, timeoutMs, env }); const steerCommandId = requiredString(command?.id, "command.id"); const now = nowIso(options.now); const agentRun = { ...mapped.agentRun, lastSteerCommandId: steerCommandId, lastSteerTraceId: steerTraceId, updatedAt: now, valuesPrinted: false }; traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:steer:command-created", message: `AgentRun steer command ${steerCommandId} created on run ${mapped.agentRun.runId}; runner will apply it if the target Codex turn is still active.`, runId: mapped.agentRun.runId, commandId: mapped.agentRun.commandId, steerCommandId, targetCommandId: mapped.agentRun.commandId, steerTraceId, waitingFor: "agentrun-steer-apply", valuesPrinted: false }, agentRun)); const payload = { ok: true, accepted: true, status: "running", shortConnection: true, controlSemantics: "steer-active-turn-and-poll-target-trace", route: "/v1/agent/chat/steer", traceId, targetTraceId: traceId, steerTraceId, conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null, sessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null, threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null, resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, agentRun: { ...agentRun, steerCommandId, targetCommandId: mapped.agentRun.commandId, steerCommandState: command.state ?? null, valuesPrinted: false }, runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})), updatedAt: now, valuesPrinted: false }; options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun, runnerTrace: payload.runnerTrace, updatedAt: now }); return payload; } export async function loadPersistedAgentRunResult(traceId, options = {}) { const safeId = safeTraceId(traceId); if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null; const session = await loadAgentSessionByTraceId(safeId, options); const agentRun = agentRunSeedFromSession(session, safeId); if (!agentRun?.runId) return null; const backendProfile = firstNonEmpty(agentRun.backendProfile, "deepseek"); const status = firstNonEmpty(session?.status === "canceled" ? "canceled" : null, agentRun.terminalStatus, agentRun.commandState, agentRun.status, "running"); const promptEvidence = agentRunPromptMetadataFromSession(session, safeId); return { accepted: true, status, shortConnection: true, traceId: safeId, conversationId: safeConversationId(session?.conversationId) || agentRun.conversationId || null, sessionId: safeSessionId(session?.id) || agentRun.sessionId || null, threadId: safeOpaqueId(session?.threadId) || agentRun.threadId || null, messageId: `msg_${safeId.slice(4)}`, ...agentRunPromptFields(promptEvidence), createdAt: session?.startedAt ?? session?.updatedAt ?? nowIso(options.now), updatedAt: session?.updatedAt ?? nowIso(options.now), ownerUserId: session?.ownerUserId, ownerRole: session?.ownerRole, provider: providerForBackendProfile(backendProfile), model: modelForBackendProfile(backendProfile, options.env ?? process.env), backend: backendForBackendProfile(backendProfile), infrastructureBackend: `agentrun-v01/${backendProfile}`, capabilityLevel: AGENTRUN_CAPABILITY_LEVEL, sessionMode: AGENTRUN_SESSION_MODE, implementationType: AGENTRUN_IMPLEMENTATION_TYPE, ...(agentRun?.userBillingReservation ? { userBillingReservation: agentRun.userBillingReservation } : {}), agentRun: { ...withoutUserBillingReservation(agentRun), adapter: ADAPTER_ID, traceId: safeId, commandId: text(agentRun.commandId) || null, valuesPrinted: false }, valuesPrinted: false }; } async function loadAgentSessionByTraceId(traceId, options = {}) { const safeId = safeTraceId(traceId); if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null; const cache = options.codeAgentTraceSessionCache; if (cache instanceof Map) { if (cache.has(safeId)) return cache.get(safeId); const session = await options.accessController.getAgentSessionByTraceId(safeId); cache.set(safeId, session ?? null); return session ?? null; } return await options.accessController.getAgentSessionByTraceId(safeId); } function agentRunSeedFromSession(session, traceId) { const snapshot = session?.session && typeof session.session === "object" ? session.session : null; const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null; const traceResult = traceResults?.[traceId] && typeof traceResults[traceId] === "object" ? traceResults[traceId] : null; const traceAgentRun = traceResult?.agentRun && typeof traceResult.agentRun === "object" ? traceResult.agentRun : null; if (traceAgentRun?.runId) { return traceResult?.userBillingReservation ? { ...traceAgentRun, userBillingReservation: traceResult.userBillingReservation } : traceAgentRun; } const topLevelAgentRun = snapshot?.agentRun && typeof snapshot.agentRun === "object" ? snapshot.agentRun : null; return topLevelAgentRun?.runId ? topLevelAgentRun : null; } async function findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId, traceId }) { if (!runId || !safeTraceId(traceId)) return null; const commands = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands?afterSeq=0&limit=100`, { method: "GET", timeoutMs, env }); return (Array.isArray(commands?.items) ? commands.items : []).find((item) => agentRunCommandMatchesTrace(item, traceId)) ?? null; } function agentRunCommandMatchesTrace(command, traceId) { const payload = command?.payload && typeof command.payload === "object" ? command.payload : {}; return command?.idempotencyKey === traceId || payload.traceId === traceId; } function withoutUserBillingReservation(value = {}) { if (!value || typeof value !== "object") return value; const { userBillingReservation, ...rest } = value; return rest; } async function ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }) { const defaultPolicy = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SESSION_STORAGE, "persistent"); if (defaultPolicy !== "persistent") return null; try { const tenantId = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TENANT_ID, DEFAULT_TENANT_ID); const projectId = agentRunProjectIdForEnv(env); const expiresInDays = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_SESSION_TTL_DAYS, 30); const expiresAt = new Date(Date.now() + Math.max(1, expiresInDays) * 24 * 60 * 60 * 1000).toISOString(); return await agentRunJson(fetchImpl, managerUrl, "/api/v1/sessions", { method: "POST", timeoutMs: 15_000, env, body: { sessionId, tenantId, projectId, backendProfile, expiresAt, codexRolloutSubdir: "sessions" }, }); } catch (error) { const message = error?.message ?? String(error); if (/evicted/i.test(message)) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:session-storage-evicted", message: "AgentRun session " + sessionId + " storage was previously evicted; hwlab-cloud-api will create a fresh sessionId for this turn.", sessionId, backendProfile, waitingFor: "agentrun-session-reset", valuesPrinted: false, }, backendProfile)); throw error; } traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:session-storage-recover-warning", message: "AgentRun session " + sessionId + " pre-flight could not ensure PVC; falling back to legacy metadata-only session (" + message + ").", sessionId, backendProfile, valuesPrinted: false, }, backendProfile)); return null; } } function agentRunSessionRecordFromEnsure(value = null) { const record = value && typeof value === "object" ? value : null; if (!record) return null; const session = record.session && typeof record.session === "object" ? record.session : record; return session && typeof session === "object" ? session : null; } function agentRunSessionThreadIdFromEnsure(value = null, backendProfile = null) { const record = agentRunSessionRecordFromEnsure(value); if (agentRunSessionThreadInvalidated(record)) return null; const existingProfile = firstNonEmpty(record?.backendProfile, record?.metadata?.agentRunSessionProfile); if (existingProfile && backendProfile && existingProfile !== backendProfile) return null; return safeOpaqueId(record?.threadId) || null; } function agentRunSessionThreadInvalidated(record = null) { if (!record || typeof record !== "object") return false; const failureMessage = firstNonEmpty( record.failureMessage, record.blocker?.message, record.liveness?.terminalClassification?.reason, record.supervisor?.terminalClassification?.reason ); if (!agentRunFailureRequiresFreshSession(record.failureKind, failureMessage)) return false; const terminalStatus = normalizeAgentRunStatus(record.terminalStatus ?? record.status); const executionState = String(record.executionState ?? "").trim().toLowerCase(); return terminalStatus === "failed" || terminalStatus === "blocked" || executionState === "terminal"; } function paramsWithAgentRunSessionThread(params = {}, threadId = null) { const next = { ...(params ?? {}) }; const safeThreadId = safeOpaqueId(threadId); if (safeThreadId) next.threadId = safeThreadId; else delete next.threadId; return next; } function agentRunProjectIdForEnv(env = process.env) { return firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECT_ID, DEFAULT_PROJECT_ID); } async function shouldResetSessionAfterEviction(failureKind, failureMessage) { return agentRunFailureRequiresFreshSession(failureKind, failureMessage); } function agentRunFreshSessionRequested(params = {}) { const value = params?.forceFreshAgentRunSession ?? params?.agentRunFreshSession ?? params?.freshAgentRunSession; if (value === true) return true; return /^(1|true|yes|fresh)$/iu.test(String(value ?? "").trim()); } function newSessionIdAfterEviction(baseSessionId, traceId) { const profile = String(traceId).replace(/[^A-Za-z0-9]/gu, "").slice(0, 12) || "fresh"; return baseSessionId + "-reset-" + profile; } async function retryAgentRunTurnAfterFreshSessionFailure({ params = null, options = {}, traceId, traceStore = defaultCodeAgentTraceStore, mapped = {}, failedResult = {}, failedMapping = {}, env = process.env, managerUrl, fetchImpl, timeoutMs, backendProfile } = {}) { const failure = agentRunFreshSessionFailure(failedResult); if (!failure.requiresFreshSession) return null; if (mapped?.agentRun?.freshSessionRetryOf || failedMapping?.freshSessionRetryOf) return null; const promptText = messageAuthorityTextValue(params?.message ?? params?.prompt ?? params?.text ?? ""); if (!promptText) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "failed", label: "agentrun:session-reset:skipped", errorCode: "agentrun_retry_prompt_missing", message: "AgentRun session storage was evicted, but hwlab-cloud-api cannot retry the current turn because the original prompt text is unavailable in this sync path.", runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null, commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null, terminal: true, valuesPrinted: false }, failedMapping || mapped?.agentRun || backendProfile)); return null; } const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId); const sessionId = newSessionIdAfterEviction(baseSessionId, traceId); const retryParams = paramsWithAgentRunSessionThread(params, null); const startedAt = nowIso(options.now); const previous = { runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null, commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null, sessionId: failedMapping?.sessionId ?? mapped?.agentRun?.sessionId ?? baseSessionId, threadId: failedMapping?.threadId ?? mapped?.agentRun?.threadId ?? mapped?.threadId ?? null, failureKind: failure.code, failureMessage: failure.message, valuesPrinted: false }; traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:session-reset:retry-current-turn", message: "AgentRun session storage/thread resume failed; hwlab-cloud-api is retrying the same user turn with a fresh AgentRun session and threadId=null.", runId: previous.runId, commandId: previous.commandId, sessionId, previousSessionId: previous.sessionId, previousThreadId: previous.threadId, errorCode: failure.code, backendProfile, waitingFor: "agentrun-run-create", valuesPrinted: false }, failedMapping || mapped?.agentRun || backendProfile)); await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }); const toolCapabilities = await resolveToolCapabilities({ params: retryParams, options }); const runInput = buildAgentRunCreateRunInput({ params: retryParams, env, traceId, backendProfile, sessionId, toolCapabilities }); const run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create-after-session-reset" }); const runId = requiredString(run?.id, "run.id"); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:created-after-session-reset", message: "AgentRun fresh-session retry run " + runId + " created for the current user turn.", runId, backendProfile, waitingFor: "agentrun-command-create", valuesPrinted: false }, backendProfile)); const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }); const commandInput = buildAgentRunCommandInput({ params: retryParams, traceId, backendProfile, sessionId, dispatchInput }); const command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create-after-session-reset" }); const commandId = requiredString(command?.id, "command.id"); const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:command:created-after-session-reset", message: "AgentRun fresh-session retry command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.", runId, commandId, backendProfile, dispatchIntentId: dispatchIntent.id, runnerJobId: dispatchIntent.runnerJobId, waitingFor: "agentrun-result", valuesPrinted: false }, backendProfile)); const mapping = { ...agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: retryParams }), freshSessionRetryAttempt: 1, freshSessionRetryOf: previous, status: `dispatch-intent-${dispatchIntent.state}`, valuesPrinted: false }; traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:dispatch-intent:admitted-after-session-reset", message: "AgentRun owns the durable fresh-session runner dispatch; HWLAB has no process-local runner Job creation window.", runId: mapping.runId, commandId: mapping.commandId, backendProfile, dispatchIntentId: dispatchIntent.id, dispatchIntentState: dispatchIntent.state, runnerJobId: dispatchIntent.runnerJobId, waitingFor: "agentrun-result", valuesPrinted: false }, mapping)); const base = initialAgentRunChatResult({ params: retryParams, options, traceId }); const payload = decorateAgentRunRunningResult({ base: { ...base, freshSessionRetryOf: previous }, mapping, traceStore, traceId }); return { payload, mapping }; } function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId, toolCapabilities = null }) { const commitId = requireAgentRunSourceCommit(env); const runtimeAuthority = requireAgentRunRuntimeAuthority(env); const threadId = safeOpaqueId(params.threadId); const hwlabProjectId = firstNonEmpty(params.projectId); rejectRemovedResourceWorkspaceFiles(params.workspaceFiles ?? params.resourceWorkspaceFiles); const resourceBundleRef = { kind: "gitbundle", repoUrl: resolveAgentRunRepoUrl(env), commitId, submodules: false, lfs: false, bundles: HWLAB_RESOURCE_GIT_BUNDLES, promptRefs: HWLAB_RESOURCE_PROMPT_REFS, }; return { tenantId: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TENANT_ID, DEFAULT_TENANT_ID), projectId: agentRunProjectIdForEnv(env), workspaceRef: { kind: "opaque", repo: DEFAULT_PROJECT_ID, branch: firstNonEmpty(env.HWLAB_BOOT_REF, env.HWLAB_CODE_AGENT_AGENTRUN_BRANCH, defaultAgentRunBranchForRuntimeLane(runtimeAuthority.lane), "v0.2"), runtimeNamespace: runtimeAuthority.namespace, hwlabTraceId: traceId, valuesPrinted: false }, sessionRef: { sessionId: sessionId ?? scopedAgentRunSessionIdForParams(params, traceId), ...(safeConversationId(params.conversationId) ? { conversationId: safeConversationId(params.conversationId) } : {}), ...(threadId ? { threadId } : {}), metadata: { adapter: ADAPTER_ID, hwlabTraceId: traceId, otelTraceId: codeAgentOtelTraceContext(traceId).traceId, traceparent: codeAgentOtelTraceContext(traceId).traceparent, hwlabApi: "/v1/agent/chat", hwlabProjectId, hwlabSessionId: safeSessionId(params.sessionId) || null, threadContinuityPolicy: THREAD_CONTINUITY_POLICY, sessionPolicy: SESSION_POLICY_RUN_LOCAL, agentRunSessionProfile: backendProfile, agentRunSessionPolicy: "hwlab-session-scoped", valuesPrinted: false } }, resourceBundleRef, providerId: resolveAgentRunProviderId(env), backendProfile, executionPolicy: { sandbox: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SANDBOX, env.HWLAB_CODE_AGENT_CODEX_SANDBOX, "danger-full-access"), approval: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_APPROVAL, "never"), timeoutMs: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_TIMEOUT_MS, parsePositiveInteger(env.HWLAB_CODE_AGENT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS)), network: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_NETWORK, "enabled"), secretScope: { allowCredentialEcho: false, providerCredentials: [providerCredentialSecretRef(backendProfile, env)], toolCredentials: agentRunToolCredentials(env, toolCapabilities) } }, traceSink: { kind: "hwlab-cloud-api", traceId, ...codeAgentOtelTraceFields(traceId, env), resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, valuesPrinted: false } }; } function rejectRemovedResourceWorkspaceFiles(value) { if (value == null || value === false || value === "") return []; throw adapterError("legacy_workspace_files_removed", "workspaceFiles/resourceWorkspaceFiles are removed; use AgentRun ResourceBundleRef kind=gitbundle with bundles[]"); } function adapterError(code, message, details = {}) { return Object.assign(new Error(message), { code, statusCode: 400, details }); } function buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId, dispatchInput = null }) { const prompt = messageAuthorityTextValue(params.message ?? params.prompt ?? ""); const promptEvidence = agentRunPromptMetadataFromText(prompt, traceId, "agentrun-command-payload", { fullText: true }); const threadId = safeOpaqueId(params.threadId); return { type: "turn", payload: { prompt, message: prompt, ...agentRunPromptEventFields(promptEvidence), ...(promptEvidence?.prompt ? { promptMetadata: promptEvidence.prompt } : {}), traceId, projectId: firstNonEmpty(params.projectId) || null, conversationId: safeConversationId(params.conversationId) || null, sessionId: sessionId ?? scopedAgentRunSessionIdForParams(params, traceId), hwlabSessionId: safeSessionId(params.sessionId) || null, threadId: threadId || null, threadContinuityPolicy: THREAD_CONTINUITY_POLICY, sessionPolicy: SESSION_POLICY_RUN_LOCAL, providerProfile: backendProfile, source: "hwlab-cloud-api", valuesPrinted: false }, ...(dispatchInput ? { dispatch: { kind: "kubernetes-job", input: dispatchInput } } : {}), idempotencyKey: traceId }; } function buildAgentRunSteerCommandInput({ params, traceId, steerTraceId, mapped }) { const prompt = messageAuthorityTextValue(params.message ?? params.prompt ?? params.text ?? ""); const promptEvidence = agentRunPromptMetadataFromText(prompt, steerTraceId, "agentrun-steer-command-payload", { fullText: true }); return { type: "steer", payload: { prompt, message: prompt, text: prompt, ...agentRunPromptEventFields(promptEvidence), ...(promptEvidence?.prompt ? { promptMetadata: promptEvidence.prompt } : {}), traceId: steerTraceId, targetTraceId: traceId, targetCommandId: mapped.agentRun.commandId, conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null, sessionId: mapped.agentRun.sessionId ?? safeSessionId(mapped.sessionId ?? params.sessionId) ?? null, hwlabSessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null, threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null, source: "hwlab-cloud-api", valuesPrinted: false }, idempotencyKey: steerTraceId }; } function buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }) { const runtimeAuthority = requireAgentRunRuntimeAuthority(env); const baseTransient = buildAgentRunTransientEnv(env, { providerProfile: backendProfile, parentTraceId: traceId }); assertAgentRunTransientRuntimeAuthority(baseTransient, runtimeAuthority); assertAgentRunDurableTransientEnv(baseTransient); return { idempotencyKey: traceId, ...(baseTransient.length > 0 ? { transientEnv: baseTransient } : {}) }; } function assertAgentRunDurableTransientEnv(entries) { const seen = new Set(); for (const entry of Array.isArray(entries) ? entries : []) { const name = firstNonEmpty(entry?.name); const value = firstNonEmpty(entry?.value); if (!/^[A-Z_][A-Z0-9_]*$/u.test(name) || !value || seen.has(name)) { throw adapterError("agentrun_durable_dispatch_transient_env_invalid", "AgentRun durable dispatch transient environment must contain unique non-empty environment entries."); } if (entry?.sensitive === true) { throw adapterError("agentrun_durable_dispatch_secret_forbidden", "AgentRun durable dispatch input must not persist sensitive transient environment values."); } if (agentRunReservedTransientEnvName(name)) { throw adapterError("agentrun_durable_dispatch_transient_env_reserved", `AgentRun durable dispatch transient environment cannot override runtime-owned ${name}.`); } seen.add(name); } } function agentRunReservedTransientEnvName(name) { return name.startsWith("AGENTRUN_") || name.startsWith("KUBERNETES_") || AGENTRUN_DURABLE_TRANSIENT_ENV_RESERVED.has(name) || new Set([ "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "GIT_SSH_COMMAND", "OTEL_SERVICE_NAME", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" ]).has(name); } function requireAgentRunDurableDispatchIntent(value) { const intent = value && typeof value === "object" && !Array.isArray(value) ? value : null; const admittedStates = new Set(["pending", "dispatching", "retry", "dispatched"]); const terminalStates = new Set(["failed", "cancelled"]); const id = typeof intent?.id === "string" ? intent.id.trim() : ""; const state = typeof intent?.state === "string" ? intent.state.trim() : ""; const runnerJobId = typeof intent?.runnerJobId === "string" ? intent.runnerJobId.trim() : ""; const attemptCount = Number(intent?.attemptCount ?? 0); if ( !intent || intent.durable !== true || (!admittedStates.has(state) && !terminalStates.has(state)) || !/^dispatch_[A-Za-z0-9_.:-]+$/u.test(id) || !/^rjob_[A-Za-z0-9_.:-]+$/u.test(runnerJobId) || !Object.hasOwn(intent, "attemptCount") || !Number.isInteger(attemptCount) || attemptCount < 0 ) { throw Object.assign(new Error("AgentRun command response did not confirm a durable dispatch intent."), { code: "agentrun_dispatch_intent_unconfirmed", statusCode: 502, valuesPrinted: false }); } if (terminalStates.has(state)) { throw Object.assign(new Error(`AgentRun durable dispatch intent ${id} is already ${state}; the turn was not admitted as running.`), { code: `agentrun_dispatch_intent_${state}`, statusCode: state === "failed" ? 503 : 409, dispatchIntentId: id, runnerJobId, attemptCount, valuesPrinted: false }); } return { id, state, runnerJobId, attemptCount, durable: true, valuesPrinted: false }; } function buildAgentRunTransientEnv(env = process.env, options = {}) { const providerProfile = firstNonEmpty(options.providerProfile); const parentTraceId = firstNonEmpty(options.parentTraceId); const entries = []; for (const name of [ "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP" ]) { const value = firstNonEmpty(env[name]); if (value) entries.push({ name, value, sensitive: false }); } if (providerProfile) entries.push({ name: "HWLAB_CODE_AGENT_PROVIDER_PROFILE", value: providerProfile, sensitive: false }); if (parentTraceId) entries.push({ name: "HWLAB_CODE_AGENT_PARENT_TRACE_ID", value: parentTraceId, sensitive: false }); return entries; } function requireAgentRunRuntimeAuthority(env = process.env) { const explicitLane = normalizeAgentRunRuntimeLane(firstNonEmpty( env.HWLAB_RUNTIME_LANE, env.HWLAB_GITOPS_LANE, env.HWLAB_GITOPS_PROFILE, env.HWLAB_ENVIRONMENT, env.HWLAB_BOOT_REF )); const namespace = firstNonEmpty( env.HWLAB_RUNTIME_NAMESPACE, env.POD_NAMESPACE, env.HWLAB_NAMESPACE, namespaceForAgentRunRuntimeLane(explicitLane) ); if (!namespace) { throw agentRunRuntimeAuthorityError("agentrun_runtime_namespace_required", "AgentRun dispatch requires HWLAB_RUNTIME_NAMESPACE/POD_NAMESPACE/HWLAB_NAMESPACE or a supported runtime lane"); } const namespaceLane = agentRunRuntimeLaneForNamespace(namespace); if (!namespaceLane) { throw agentRunRuntimeAuthorityError("agentrun_runtime_namespace_unsupported", `unsupported AgentRun HWLAB runtime namespace: ${namespace}`); } if (explicitLane && explicitLane !== namespaceLane) { throw agentRunRuntimeAuthorityError("agentrun_runtime_authority_mismatch", `AgentRun HWLAB runtime lane ${explicitLane} does not match namespace ${namespace} (${namespaceLane})`); } assertAgentRunEndpointNamespace(env.HWLAB_RUNTIME_API_URL, namespace, "HWLAB_RUNTIME_API_URL"); assertAgentRunEndpointNamespace(env.HWLAB_RUNTIME_WEB_URL, namespace, "HWLAB_RUNTIME_WEB_URL"); return { namespace, lane: namespaceLane }; } function assertAgentRunTransientRuntimeAuthority(entries, runtimeAuthority) { const values = new Map((Array.isArray(entries) ? entries : []).map((entry) => [entry?.name, firstNonEmpty(entry?.value)])); if (values.get("HWLAB_RUNTIME_NAMESPACE") !== runtimeAuthority.namespace) { throw agentRunRuntimeAuthorityError("agentrun_transient_namespace_mismatch", `AgentRun transient HWLAB_RUNTIME_NAMESPACE must be ${runtimeAuthority.namespace}`); } if (normalizeAgentRunRuntimeLane(values.get("HWLAB_RUNTIME_LANE")) !== runtimeAuthority.lane) { throw agentRunRuntimeAuthorityError("agentrun_transient_lane_mismatch", `AgentRun transient HWLAB_RUNTIME_LANE must be ${runtimeAuthority.lane}`); } for (const name of ["HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL"]) { const value = values.get(name); if (!value) { throw agentRunRuntimeAuthorityError("agentrun_transient_endpoint_required", `AgentRun transient ${name} is required`); } assertAgentRunEndpointNamespace(value, runtimeAuthority.namespace, name); } } function assertAgentRunEndpointNamespace(value, namespace, fieldName) { const endpointNamespace = agentRunRuntimeNamespaceForEndpoint(value); if (endpointNamespace && endpointNamespace !== namespace) { throw agentRunRuntimeAuthorityError("agentrun_runtime_endpoint_mismatch", `${fieldName} points at ${endpointNamespace}, not ${namespace}`); } } function agentRunRuntimeNamespaceForEndpoint(value) { const match = String(value ?? "").match(/\.((?:hwlab-[a-z0-9-]+))\.svc\.cluster\.local(?::|\/|$)/iu); return match?.[1] ?? null; } function agentRunRuntimeLaneForNamespace(namespace) { const value = String(namespace ?? "").trim().toLowerCase(); if (value === "hwlab-prod") return "prod"; if (value === "hwlab-dev") return "dev"; const versionMatch = value.match(/^hwlab-v(\d+)$/u); if (versionMatch) return versionAgentRunRuntimeLane(versionMatch[1]); return null; } function normalizeAgentRunRuntimeLane(value) { const profile = String(value ?? "").trim().toLowerCase(); if (!profile) return null; if (profile === "prod" || profile === "production") return "prod"; if (profile === "dev" || profile === "development" || profile === "local") return "dev"; const dotted = profile.match(/^v?0+\.(\d+)$/u); if (dotted) return versionAgentRunRuntimeLane(dotted[1]); const compact = profile.match(/^v?0*(\d+)$/u); if (compact) return versionAgentRunRuntimeLane(compact[1]); throw agentRunRuntimeAuthorityError("agentrun_runtime_lane_unsupported", `unsupported AgentRun HWLAB runtime lane: ${String(value).trim()}`); } function versionAgentRunRuntimeLane(value) { const version = Number.parseInt(value, 10); if (!Number.isFinite(version) || version <= 0) return null; return `v${String(version).padStart(2, "0")}`; } function namespaceForAgentRunRuntimeLane(lane) { if (lane === "prod") return "hwlab-prod"; if (lane === "dev") return "hwlab-dev"; if (lane) return `hwlab-${lane}`; return null; } function defaultAgentRunBranchForRuntimeLane(lane) { const match = String(lane ?? "").match(/^v(\d+)$/u); if (!match) return null; return `v0.${Number.parseInt(match[1], 10)}`; } function agentRunRuntimeAuthorityError(code, message) { return Object.assign(new Error(message), { code, statusCode: 500, valuesPrinted: false }); } function agentRunToolCredentials(env = process.env, toolCapabilities = null) { const namespace = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TOOL_SECRET_NAMESPACE, env.HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE, DEFAULT_RUNNER_NAMESPACE); const githubSecretName = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_GITHUB_TOOL_SECRET_NAME, "agentrun-v01-tool-github-pr"); const githubSecretKey = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_GITHUB_TOOL_SECRET_KEY, "GH_TOKEN"); const unideskSecretName = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_UNIDESK_SSH_TOOL_SECRET_NAME, "agentrun-v01-tool-unidesk-ssh"); const credentials = []; if (toolCapabilityAllowed(toolCapabilities, "github_pr")) credentials.push({ tool: "github", purpose: "pull-request", secretRef: { namespace, name: githubSecretName, keys: [githubSecretKey] }, projection: { kind: "env", envName: githubSecretKey, secretKey: githubSecretKey } }); if (toolCapabilityAllowed(toolCapabilities, "unidesk_ssh")) credentials.push({ tool: "unidesk-ssh", purpose: "ssh-passthrough", secretRef: { namespace, name: unideskSecretName, keys: ["UNIDESK_SSH_CLIENT_TOKEN"] }, projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN", secretKey: "UNIDESK_SSH_CLIENT_TOKEN" } }); return credentials; } async function resolveToolCapabilities({ params = {}, options = {} } = {}) { const ownerUserId = text(params.ownerUserId); const accessController = options.accessController; if (!ownerUserId || typeof accessController?.codeAgentToolCapabilitiesForOwner !== "function") return null; return await accessController.codeAgentToolCapabilitiesForOwner(ownerUserId); } function toolCapabilityAllowed(toolCapabilities = null, toolId) { if (!toolCapabilities || !toolCapabilities.tools) return true; return toolCapabilities.tools?.[toolId]?.allowed === true; } async function resolveReusableAgentRun({ params = {}, options = {}, env = process.env, managerUrl, fetchImpl, timeoutMs, backendProfile, traceId, traceStore }) { const hwlabSessionId = hwlabSessionIdForParams(params, traceId); if (!safeSessionId(hwlabSessionId) || hwlabSessionId === agentRunSessionId(traceId) || typeof options.accessController?.getAgentSession !== "function") { return null; } const session = await options.accessController.getAgentSession(hwlabSessionId); if (!canReuseAgentRunSessionForOwner(session, params, options)) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:reuse-owner-skipped", message: "Stored AgentRun session is not visible to the current actor; a fresh run with manager-owned durable dispatch will be admitted for this request.", sessionId: hwlabSessionId, waitingFor: "agentrun-run-create", valuesPrinted: false }, backendProfile)); return null; } const mapping = session?.session?.agentRun; if (!isReusableAgentRunMapping(mapping, backendProfile)) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:reuse-skipped", message: "No reusable AgentRun run was found for this HWLAB session; a fresh run with manager-owned durable dispatch will be admitted.", sessionId: hwlabSessionId, waitingFor: "agentrun-run-create", valuesPrinted: false }, backendProfile)); return null; } let run = null; try { run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: `/api/v1/runs/${encodeURIComponent(mapping.runId)}`, method: "GET", timeoutMs, env, traceStore, traceId, backendProfile, runId: mapping.runId, stage: "run-reuse-check", terminalOnFailure: false }); } catch (error) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:reuse-blocked", message: `Stored AgentRun run ${mapping.runId} could not be read from AgentRun manager; hwlab-cloud-api will create a fresh run.`, errorCode: error?.code ?? error?.agentRunError?.failureKind ?? "agentrun_reuse_check_failed", runId: mapping.runId, commandId: mapping.commandId ?? null, runnerId: mapping.runnerId ?? null, jobName: mapping.jobName ?? null, namespace: mapping.namespace ?? null, sessionId: hwlabSessionId, agentRunSessionId: mapping.sessionId ?? null, waitingFor: "agentrun-run-create", valuesPrinted: false }, backendProfile)); return null; } const blocker = agentRunReuseBlocker(run, mapping); if (blocker) { traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:reuse-blocked", message: `Stored AgentRun run ${mapping.runId} cannot be reused (${blocker}); hwlab-cloud-api will create a fresh run while preserving the session-scoped AgentRun PVC.`, runId: mapping.runId, commandId: mapping.commandId ?? null, runnerId: mapping.runnerId ?? null, jobName: mapping.jobName ?? null, namespace: mapping.namespace ?? null, sessionId: hwlabSessionId, agentRunSessionId: mapping.sessionId ?? null, reuseBlocker: blocker, runStatus: run?.terminalStatus ?? run?.status ?? mapping.runStatus ?? null, leaseExpiresAt: run?.leaseExpiresAt ?? null, waitingFor: "agentrun-run-create", valuesPrinted: false }, backendProfile)); return null; } const agentRunScopedSessionId = safeSessionId(run?.sessionRef?.sessionId) || safeSessionId(mapping.sessionId) || scopedAgentRunSessionIdForParams(params, traceId); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:run:reuse-enabled", message: `Stored AgentRun run ${mapping.runId} is still claimed by an active runner; hwlab-cloud-api will append the next command without creating another runner Job.`, runId: mapping.runId, commandId: mapping.commandId ?? null, runnerId: mapping.runnerId ?? null, jobName: mapping.jobName ?? null, namespace: mapping.namespace ?? null, sessionId: hwlabSessionId, agentRunSessionId: agentRunScopedSessionId, runStatus: run?.status ?? mapping.runStatus ?? null, claimedBy: run?.claimedBy ?? null, leaseExpiresAt: run?.leaseExpiresAt ?? null, waitingFor: "agentrun-command-create", valuesPrinted: false }, backendProfile)); return { mapping, run, sessionId: agentRunScopedSessionId }; } function isReusableAgentRunMapping(mapping, backendProfile) { return Boolean( mapping && typeof mapping === "object" && mapping.adapter === ADAPTER_ID && mapping.reuseEligible !== false && mapping.runId && mapping.jobName && (!mapping.backendProfile || mapping.backendProfile === backendProfile) ); } function canReuseAgentRunSessionForOwner(session, params = {}, options = {}) { if (!session) return false; const ownerUserId = options.actor?.id ?? params.ownerUserId ?? null; const ownerRole = options.actor?.role ?? params.ownerRole ?? null; if (ownerRole === "admin") return true; if (!session.ownerUserId) return true; return Boolean(ownerUserId && session.ownerUserId === ownerUserId); } function agentRunReuseBlocker(run = {}, mapping = {}) { const status = String(run?.terminalStatus ?? run?.status ?? mapping.runStatus ?? "").toLowerCase(); if (TERMINAL_RUN_STATUSES.has(status)) return `run_status_${status}`; if (!run?.claimedBy) return "runner_not_claimed"; if (runLeaseExpired(run.leaseExpiresAt)) return "runner_reuse_window_expired"; return null; } function runLeaseExpired(leaseExpiresAt) { const expiresMs = Date.parse(String(leaseExpiresAt ?? "")); return !Number.isFinite(expiresMs) || expiresMs <= Date.now(); }