2941 lines
135 KiB
TypeScript
2941 lines
135 KiB
TypeScript
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// Responsibility: AgentRun v0.1 adapter, incremental Workbench projection cursor sync, and upstream timing projection for Cloud API observability.
|
|
|
|
import { randomUUID } from "node:crypto";
|
|
|
|
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
|
import { decorateCodeAgentSession } from "./code-agent-session-lifecycle.ts";
|
|
import {
|
|
parsePositiveInteger,
|
|
safeConversationId,
|
|
safeOpaqueId,
|
|
safeSessionId,
|
|
safeTraceId,
|
|
text
|
|
} from "./server-http-utils.ts";
|
|
import { currentBackendPerformanceContext } from "./backend-performance.ts";
|
|
import {
|
|
DEFAULT_AGENTRUN_MGR_URL,
|
|
agentRunJson,
|
|
isAgentRunManagerInternalServiceHost,
|
|
resolveAgentRunManagerUrl
|
|
} from "./code-agent-agentrun-runtime.ts";
|
|
import {
|
|
agentRunAlreadyTerminalCancelPayload,
|
|
agentRunCancelBlockedPayload,
|
|
agentRunCancelTerminalStatus
|
|
} from "./code-agent-agentrun-cancel.ts";
|
|
import {
|
|
DEFAULT_AGENTRUN_REPO_URL,
|
|
agentRunAvailabilityBlocker,
|
|
agentRunProviderIdOrNull,
|
|
agentRunTraceMeta,
|
|
backendForBackendProfile,
|
|
modelForBackendProfile,
|
|
providerForBackendProfile,
|
|
requireAgentRunSourceCommit,
|
|
resolveAgentRunBackendProfile,
|
|
resolveAgentRunProviderId,
|
|
resolveAgentRunRepoUrl
|
|
} from "./code-agent-agentrun-profile.ts";
|
|
import {
|
|
agentRunPromptEventFields,
|
|
agentRunPromptFields,
|
|
agentRunPromptMetadataFromParams,
|
|
agentRunPromptMetadataFromSession,
|
|
agentRunPromptMetadataFromText,
|
|
messageAuthorityTextValue
|
|
} from "./code-agent-agentrun-prompt.ts";
|
|
import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts";
|
|
import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
|
|
|
const ADAPTER_ID = "agentrun-v01";
|
|
const DEFAULT_TENANT_ID = "hwlab";
|
|
const DEFAULT_PROJECT_ID = "pikasTech/HWLAB";
|
|
const DEFAULT_RUNNER_NAMESPACE = "agentrun-v01";
|
|
const DEFAULT_TIMEOUT_MS = 1_200_000;
|
|
const AGENTRUN_RUNNER_KIND = "agentrun-v01-shared-runner";
|
|
const AGENTRUN_SESSION_MODE = "agentrun-v01-durable-session";
|
|
const AGENTRUN_IMPLEMENTATION_TYPE = "agentrun-v01-shared-execution-infra";
|
|
const AGENTRUN_CAPABILITY_LEVEL = "agentrun-v01-shared-code-agent-session";
|
|
const AGENTRUN_PROVIDER_TRACE_PROTOCOL = "agentrun-v01-jsonrpc";
|
|
const AGENTRUN_PROVIDER_TRACE_WIRE_API = "agentrun-v01-command-result";
|
|
const THREAD_CONTINUITY_POLICY = "hwlab-agentrun-v01-reuse-runner-thread";
|
|
const SESSION_POLICY_RUN_LOCAL = "hwlab-agentrun-v01-session-runner-reuse";
|
|
const TERMINAL_RUN_STATUSES = new Set(["completed", "failed", "blocked", "cancelled", "canceled"]);
|
|
const HWLAB_RESOURCE_GIT_BUNDLES = Object.freeze([
|
|
Object.freeze({ name: "hwlab-tools", subpath: "tools", target_path: "tools" }),
|
|
Object.freeze({ name: "hwlab-agent-skills", subpath: "skills", target_path: ".agents/skills" })
|
|
]);
|
|
const HWLAB_RESOURCE_PROMPT_REFS = Object.freeze([
|
|
Object.freeze({ name: "hwlab-v02-runtime", path: "internal/agent/prompts/hwlab-v02-runtime.md", inject: "thread-start", required: true })
|
|
]);
|
|
|
|
export function codeAgentAgentRunAdapterEnabled(env = process.env) {
|
|
const value = String(env.HWLAB_CODE_AGENT_ADAPTER ?? env.HWLAB_CODE_AGENT_PROVIDER ?? "").trim().toLowerCase();
|
|
return value === ADAPTER_ID || value === "agentrun" || value === "agentrun-v0.1";
|
|
}
|
|
|
|
export function describeAgentRunAdapterAvailability(env = process.env, options = {}) {
|
|
const params = options.params ?? {};
|
|
const backendProfile = resolveAgentRunBackendProfile(env, params);
|
|
const provider = providerForBackendProfile(backendProfile);
|
|
const backend = `${ADAPTER_ID}/${backendProfile}`;
|
|
const model = modelForBackendProfile(backendProfile, env);
|
|
const blockers = [];
|
|
let managerUrl = null;
|
|
let repoUrl = null;
|
|
let resourceBundleSourceCommit = null;
|
|
|
|
try {
|
|
managerUrl = resolveAgentRunManagerUrl(env);
|
|
} catch (error) {
|
|
blockers.push(agentRunAvailabilityBlocker(error, "manager-url"));
|
|
}
|
|
try {
|
|
repoUrl = resolveAgentRunRepoUrl(env);
|
|
} catch (error) {
|
|
blockers.push(agentRunAvailabilityBlocker(error, "repo-url"));
|
|
}
|
|
try {
|
|
resourceBundleSourceCommit = requireAgentRunSourceCommit(env);
|
|
} catch (error) {
|
|
blockers.push(agentRunAvailabilityBlocker(error, "resource-bundle-source-commit"));
|
|
}
|
|
let providerId = null;
|
|
try {
|
|
providerId = resolveAgentRunProviderId(env);
|
|
} catch (error) {
|
|
blockers.push(agentRunAvailabilityBlocker(error, "provider-id"));
|
|
}
|
|
|
|
const runnerNamespace = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE, env.AGENTRUN_RUNTIME_NAMESPACE, DEFAULT_RUNNER_NAMESPACE);
|
|
const ready = blockers.length === 0 && Boolean(providerId && runnerNamespace && backendProfile && managerUrl && repoUrl);
|
|
const managerHost = hostForUrl(managerUrl);
|
|
const repoHost = hostForUrl(repoUrl);
|
|
const status = ready ? "agentrun-ready" : "blocked";
|
|
const capabilityLevel = ready ? AGENTRUN_CAPABILITY_LEVEL : "blocked";
|
|
|
|
return {
|
|
endpoint: "POST /v1/agent/chat",
|
|
adapter: ADAPTER_ID,
|
|
provider,
|
|
model,
|
|
backend,
|
|
infrastructureBackend: backend,
|
|
mode: ADAPTER_ID,
|
|
schema: [
|
|
"conversationId",
|
|
"sessionId",
|
|
"messageId",
|
|
"status",
|
|
"createdAt",
|
|
"updatedAt",
|
|
"traceId",
|
|
"provider",
|
|
"model",
|
|
"backend",
|
|
"infrastructureBackend",
|
|
"agentRun",
|
|
"runner",
|
|
"runnerTrace",
|
|
"session",
|
|
"sessionMode",
|
|
"sessionReuse",
|
|
"threadContinuityPolicy",
|
|
"error.message",
|
|
"error.code"
|
|
],
|
|
runner: {
|
|
kind: AGENTRUN_RUNNER_KIND,
|
|
adapter: ADAPTER_ID,
|
|
backend,
|
|
provider,
|
|
model,
|
|
namespace: runnerNamespace,
|
|
managerHost,
|
|
repoHost,
|
|
mode: ADAPTER_ID,
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
status: ready ? "available" : "blocked",
|
|
ready,
|
|
capabilityLevel,
|
|
longLivedSession: ready,
|
|
durableSession: ready,
|
|
durable: ready,
|
|
codexStdio: false,
|
|
delegatedToAgentRun: true,
|
|
writeCapable: ready,
|
|
readOnly: false,
|
|
runnerLimitations: ready ? [] : ["agentrun-adapter-config-blocked"],
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true
|
|
},
|
|
agentRun: {
|
|
adapter: ADAPTER_ID,
|
|
backendProfile,
|
|
providerId,
|
|
runnerNamespace,
|
|
managerUrl,
|
|
managerHost,
|
|
repoUrl,
|
|
repoHost,
|
|
resourceBundleSourceCommit,
|
|
managerConfigured: Boolean(managerUrl),
|
|
repoConfigured: Boolean(repoUrl),
|
|
internalServiceDns: isAgentRunManagerInternalServiceHost(managerHost),
|
|
ready,
|
|
valuesPrinted: false
|
|
},
|
|
status,
|
|
readinessStatus: status,
|
|
agentKind: "agentrun-v01-adapter",
|
|
capabilityStatus: status,
|
|
blocker: ready ? null : blockers[0]?.summary ?? "AgentRun adapter runtime configuration is blocked.",
|
|
reason: ready ? null : blockers[0]?.code ?? "agentrun_adapter_blocked",
|
|
summary: ready
|
|
? "HWLAB Code Agent delegates execution to AgentRun v0.1 shared infrastructure; hwlab-cloud-api keeps session/trace/API ownership and no longer manages a repo-owned runner control plane."
|
|
: "HWLAB Code Agent is configured for AgentRun v0.1, but the adapter runtime configuration is incomplete or invalid.",
|
|
missingEnv: [],
|
|
secretRefs: [],
|
|
egress: {
|
|
ready,
|
|
mode: "agentrun-managed-provider-egress",
|
|
directPublicOpenAi: false,
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true
|
|
},
|
|
safety: {
|
|
secretsRead: false,
|
|
secretValuesPrinted: false,
|
|
kubeconfigRead: false,
|
|
providerCredentialsDelegatedToAgentRun: true,
|
|
repoOwnedRunnerControlPlane: false,
|
|
valuesRedacted: true
|
|
},
|
|
capabilityLevel,
|
|
workspace: repoUrl,
|
|
sandbox: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SANDBOX, env.HWLAB_CODE_AGENT_CODEX_SANDBOX, "danger-full-access"),
|
|
session: null,
|
|
sessionStatus: null,
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
runnerLimitations: ready ? [] : ["agentrun-adapter-config-blocked"],
|
|
longLivedSessionGate: {
|
|
status: ready ? "pass" : "blocked",
|
|
provider,
|
|
runnerKind: AGENTRUN_RUNNER_KIND,
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
adapter: ADAPTER_ID,
|
|
codexStdio: false,
|
|
delegatedToAgentRun: true,
|
|
blockers,
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true
|
|
},
|
|
blockers,
|
|
blockerCodes: blockers.map((blocker) => blocker.code).filter(Boolean),
|
|
ready,
|
|
partialReady: false
|
|
};
|
|
}
|
|
|
|
export function initialAgentRunChatResult({ params = {}, options = {}, traceId }) {
|
|
const env = options.env ?? process.env;
|
|
const timestamp = nowIso(options.now);
|
|
const backendProfile = resolveAgentRunBackendProfile(env, params);
|
|
const promptEvidence = agentRunPromptMetadataFromParams(params, traceId);
|
|
return {
|
|
accepted: true,
|
|
status: "running",
|
|
shortConnection: true,
|
|
traceId,
|
|
...codeAgentOtelTraceFields(traceId, env),
|
|
projectId: firstNonEmpty(params.projectId) || null,
|
|
conversationId: safeConversationId(params.conversationId) || null,
|
|
sessionId: safeSessionId(params.sessionId) || agentRunSessionId(traceId),
|
|
threadId: safeOpaqueId(params.threadId) || null,
|
|
messageId: `msg_${safeTraceId(traceId)?.slice(4) || randomUUID()}`,
|
|
...agentRunAdmissionTimingFields(timestamp),
|
|
provider: providerForBackendProfile(backendProfile),
|
|
model: modelForBackendProfile(backendProfile, env),
|
|
backend: backendForBackendProfile(backendProfile),
|
|
infrastructureBackend: `agentrun-v01/${backendProfile}`,
|
|
capabilityLevel: AGENTRUN_CAPABILITY_LEVEL,
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
threadContinuityPolicy: THREAD_CONTINUITY_POLICY,
|
|
...agentRunPromptFields(promptEvidence),
|
|
agentRun: {
|
|
adapter: ADAPTER_ID,
|
|
status: "pending-dispatch",
|
|
backendProfile,
|
|
providerId: resolveAgentRunProviderId(env),
|
|
managerUrl: resolveAgentRunManagerUrl(env),
|
|
repoUrl: resolveAgentRunRepoUrl(env),
|
|
resourceBundleSourceCommit: requireAgentRunSourceCommit(env),
|
|
valuesPrinted: false
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunAdmissionTimingFields(value) {
|
|
const admittedAt = timestampIsoOrNow(value);
|
|
const timing = {
|
|
startedAt: admittedAt,
|
|
lastEventAt: admittedAt,
|
|
finishedAt: null,
|
|
durationMs: null,
|
|
observedAt: admittedAt,
|
|
lastEventAgeMs: 0,
|
|
valuesRedacted: true
|
|
};
|
|
return {
|
|
createdAt: admittedAt,
|
|
updatedAt: admittedAt,
|
|
timing,
|
|
startedAt: admittedAt,
|
|
lastEventAt: admittedAt,
|
|
finishedAt: null,
|
|
durationMs: null
|
|
};
|
|
}
|
|
|
|
export async function submitAgentRunChatTurn({ params = {}, options = {}, traceId, traceStore = defaultCodeAgentTraceStore, results }) {
|
|
const env = options.env ?? process.env;
|
|
const managerUrl = resolveAgentRunManagerUrl(env);
|
|
const backendProfile = resolveAgentRunBackendProfile(env, params);
|
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
|
const startedAt = nowIso(options.now);
|
|
const ownerApiKey = await resolveOwnerApiKey({ params, options, now: () => nowIso(options.now) });
|
|
const toolCapabilities = await resolveToolCapabilities({ params, options });
|
|
const promptEvidence = agentRunPromptMetadataFromParams(params, traceId);
|
|
|
|
traceStore.ensure(traceId, agentRunTraceMeta(env, params));
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "request",
|
|
status: "accepted",
|
|
label: "agentrun:request:accepted",
|
|
message: "HWLAB Code Agent request accepted by the AgentRun v0.1 adapter; hwlab-cloud-api will reuse a warm session run when its runner lease is valid, otherwise create a fresh run and runner Job outside the browser admission path.",
|
|
waitingFor: "agentrun-run-reuse-check",
|
|
adapter: ADAPTER_ID,
|
|
managerHost: new URL(managerUrl).hostname,
|
|
...codeAgentOtelTraceFields(traceId, env),
|
|
...agentRunPromptEventFields(promptEvidence),
|
|
valuesPrinted: false
|
|
}, backendProfile), agentRunTraceMeta(env, params));
|
|
|
|
const reusable = await resolveReusableAgentRun({ params, options, env, managerUrl, fetchImpl, timeoutMs, backendProfile, traceId, traceStore });
|
|
if (reusable?.mapping) {
|
|
const reusableParams = paramsWithAgentRunSessionThread(params, reusable.mapping.threadId);
|
|
const commandInput = buildAgentRunCommandInput({ params: reusableParams, traceId, backendProfile, sessionId: reusable.sessionId });
|
|
let command = null;
|
|
let commandId = null;
|
|
try {
|
|
command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: `/api/v1/runs/${encodeURIComponent(reusable.mapping.runId)}/commands`,
|
|
method: "POST",
|
|
body: commandInput,
|
|
timeoutMs,
|
|
env,
|
|
traceStore,
|
|
traceId,
|
|
backendProfile,
|
|
runId: reusable.mapping.runId,
|
|
stage: "command-create-reused",
|
|
terminalOnFailure: false
|
|
});
|
|
commandId = requiredString(command?.id, "command.id");
|
|
} catch (error) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:run:reuse-command-failed",
|
|
message: `AgentRun reused run ${reusable.mapping.runId} rejected the new command; hwlab-cloud-api will create a fresh run/runner Job for this turn.`,
|
|
errorCode: error?.code ?? "agentrun_reuse_command_failed",
|
|
runId: reusable.mapping.runId,
|
|
commandId: reusable.mapping.commandId ?? null,
|
|
waitingFor: "agentrun-run-create",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
command = null;
|
|
commandId = null;
|
|
}
|
|
if (command && commandId) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:command:created",
|
|
message: `AgentRun command ${commandId} created on reused run ${reusable.mapping.runId}; the active runner will pick it up without creating another runner Job.`,
|
|
runId: reusable.mapping.runId,
|
|
commandId,
|
|
backendProfile,
|
|
waitingFor: "agentrun-result",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
const runnerJob = null;
|
|
const mapping = agentRunReusedMapping({ previous: reusable.mapping, run: reusable.run, command, runnerJob, traceId, startedAt, backendProfile, managerUrl, env });
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: runnerJob ? "agentrun:runner-job:ensured" : "agentrun:runner-job:reused",
|
|
message: runnerJob
|
|
? `AgentRun runner Job ${mapping.jobName ?? "unknown"} ensured for reused run ${mapping.runId}; this keeps the persistent session resumable after pod replacement.`
|
|
: `AgentRun runner Job ${mapping.jobName ?? "unknown"} remains active for this HWLAB session turn; no replacement Job was created.`,
|
|
runId: mapping.runId,
|
|
commandId: mapping.commandId,
|
|
attemptId: mapping.attemptId,
|
|
runnerId: mapping.runnerId,
|
|
jobName: mapping.jobName,
|
|
namespace: mapping.namespace,
|
|
waitingFor: "agentrun-result",
|
|
valuesPrinted: false
|
|
}, mapping));
|
|
return decorateAgentRunRunningResult({ base: initialAgentRunChatResult({ params: reusableParams, options, traceId }), mapping, traceStore, traceId });
|
|
}
|
|
}
|
|
|
|
const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId);
|
|
let sessionId = baseSessionId;
|
|
let sessionReset = false;
|
|
let finalDispatchParams = params;
|
|
let run, command, mapping;
|
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
let attemptParams = params;
|
|
if (sessionReset) {
|
|
sessionId = newSessionIdAfterEviction(baseSessionId, traceId);
|
|
attemptParams = paramsWithAgentRunSessionThread(params, null);
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend", status: "running",
|
|
label: "agentrun:session-reset",
|
|
message: "AgentRun session storage was evicted; hwlab-cloud-api is creating a fresh sessionId " + sessionId + " with threadId=null.",
|
|
sessionId, previousSessionId: baseSessionId, backendProfile, waitingFor: "agentrun-run-create", valuesPrinted: false,
|
|
}, backendProfile));
|
|
}
|
|
let ensuredSession = null;
|
|
try {
|
|
ensuredSession = await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore });
|
|
} catch (error) {
|
|
if (attempt === 0 && await shouldResetSessionAfterEviction("session-store-evicted", error?.message)) {
|
|
sessionReset = true;
|
|
continue;
|
|
}
|
|
throw error;
|
|
}
|
|
const ensuredSessionRecord = agentRunSessionRecordFromEnsure(ensuredSession);
|
|
if (attempt === 0 && agentRunSessionThreadInvalidated(ensuredSessionRecord)) {
|
|
sessionReset = true;
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:profile-session-reset",
|
|
message: "AgentRun profile-scoped session " + sessionId + " is terminal after a thread/resume failure; hwlab-cloud-api will create a fresh profile sessionId instead of letting AgentRun manager backfill a stale threadId.",
|
|
sessionId,
|
|
previousSessionId: baseSessionId,
|
|
backendProfile,
|
|
waitingFor: "agentrun-session-reset",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
continue;
|
|
}
|
|
const profileThreadId = agentRunSessionThreadIdFromEnsure(ensuredSession, backendProfile);
|
|
const dispatchParams = paramsWithAgentRunSessionThread(attemptParams, profileThreadId);
|
|
if (safeOpaqueId(attemptParams.threadId) && !profileThreadId) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:profile-session-thread-reset",
|
|
message: "AgentRun profile-scoped session " + sessionId + " has no threadId; hwlab-cloud-api will start a new thread instead of resuming a thread from another backend profile.",
|
|
sessionId,
|
|
backendProfile,
|
|
waitingFor: "agentrun-run-create",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
}
|
|
const runInput = buildAgentRunCreateRunInput({ params: dispatchParams, env, traceId, backendProfile, sessionId, toolCapabilities });
|
|
const runCreateStartedAt = Date.now();
|
|
run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create" });
|
|
const runId = requiredString(run?.id, "run.id");
|
|
void emitCodeAgentOtelSpan("agentrun_run_create", traceId, env, { startTimeMs: runCreateStartedAt, attributes: { runId, backendProfile, managerUrl } });
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend", status: "running",
|
|
label: "agentrun:run:created",
|
|
message: "AgentRun run " + runId + " created through internal k3s Service DNS.",
|
|
runId, backendProfile, waitingFor: "agentrun-command-create", valuesPrinted: false,
|
|
}, backendProfile));
|
|
const commandInput = buildAgentRunCommandInput({ params: dispatchParams, traceId, backendProfile, sessionId });
|
|
const commandCreateStartedAt = Date.now();
|
|
command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create" });
|
|
const commandId = requiredString(command?.id, "command.id");
|
|
void emitCodeAgentOtelSpan("agentrun_command_create", traceId, env, { startTimeMs: commandCreateStartedAt, attributes: { runId, commandId, backendProfile, managerUrl } });
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend", status: "running",
|
|
label: "agentrun:command:created",
|
|
message: "AgentRun command " + commandId + " created; hwlab-cloud-api will start a runner Job explicitly without relying on scheduler automation.",
|
|
runId, commandId, backendProfile, waitingFor: "agentrun-runner-job-create", valuesPrinted: false,
|
|
}, backendProfile));
|
|
const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities, backendProfile });
|
|
mapping = agentRunMapping({ env, managerUrl, backendProfile, run, command, runnerJob: null, traceId, startedAt, params: dispatchParams });
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:runner-job:queued",
|
|
message: "AgentRun run/command admission is durable; hwlab-cloud-api will create the runner Job asynchronously so the browser request is not blocked by Kubernetes Job creation.",
|
|
runId: mapping.runId,
|
|
commandId: mapping.commandId,
|
|
backendProfile,
|
|
waitingFor: "agentrun-runner-job-create",
|
|
valuesPrinted: false
|
|
}, mapping));
|
|
scheduleAgentRunRunnerJobCreateAfterAdmission({ fetchImpl, managerUrl, runnerJobInput, timeoutMs, env, traceStore, traceId, backendProfile, run, command, mapping, options, params: dispatchParams });
|
|
finalDispatchParams = dispatchParams;
|
|
break;
|
|
}
|
|
return decorateAgentRunRunningResult({ base: initialAgentRunChatResult({ params: finalDispatchParams, options, traceId }), mapping, traceStore, traceId });
|
|
}
|
|
|
|
function scheduleAgentRunRunnerJobCreateAfterAdmission(args) {
|
|
setTimeout(() => {
|
|
void ensureAgentRunRunnerJobCreateAfterAdmission(args).catch((error) => {
|
|
markAgentRunRunnerJobCreateFailedAfterAdmission(args, error);
|
|
});
|
|
}, 0);
|
|
}
|
|
|
|
async function ensureAgentRunRunnerJobCreateAfterAdmission({ fetchImpl, managerUrl, runnerJobInput, timeoutMs, env = process.env, traceStore = defaultCodeAgentTraceStore, traceId, backendProfile, mapping = {}, options = {}, params = {} }) {
|
|
const runId = requiredString(mapping.runId, "mapping.runId");
|
|
const commandId = requiredString(mapping.commandId, "mapping.commandId");
|
|
const runnerJobStartedAt = Date.now();
|
|
const runnerJob = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/runner-jobs", method: "POST", body: runnerJobInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, commandId, stage: "runner-job-create-async" });
|
|
const nextMapping = agentRunMappingWithRunnerJob(mapping, runnerJob);
|
|
void emitCodeAgentOtelSpan("agentrun_runner_job_create_async", traceId, env, { startTimeMs: runnerJobStartedAt, attributes: { runId, commandId, backendProfile, managerUrl, jobName: nextMapping.jobName ?? null } });
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:runner-job:created",
|
|
message: `AgentRun runner Job ${nextMapping.jobName ?? "unknown"} created in namespace ${nextMapping.namespace ?? DEFAULT_RUNNER_NAMESPACE}.`,
|
|
runId: nextMapping.runId,
|
|
commandId: nextMapping.commandId,
|
|
attemptId: nextMapping.attemptId,
|
|
runnerId: nextMapping.runnerId,
|
|
jobName: nextMapping.jobName,
|
|
namespace: nextMapping.namespace,
|
|
waitingFor: "agentrun-result",
|
|
valuesPrinted: false
|
|
}, nextMapping));
|
|
const current = options.codeAgentChatResults?.get?.(traceId);
|
|
const base = current && typeof current === "object" ? current : initialAgentRunChatResult({ params, options, traceId });
|
|
options.codeAgentChatResults?.set?.(traceId, decorateAgentRunRunningResult({ base: { ...base, agentRun: nextMapping }, mapping: nextMapping, traceStore, traceId }));
|
|
}
|
|
|
|
function markAgentRunRunnerJobCreateFailedAfterAdmission({ traceStore = defaultCodeAgentTraceStore, traceId, backendProfile, mapping = {}, options = {}, params = {} }, error) {
|
|
const failureKind = agentRunDispatchFailureKind(error);
|
|
const retryAttempt = Number(error?.retryAttempt ?? 0);
|
|
const retryMax = Number(error?.retryMax ?? agentRunDispatchRetryPolicy().maxRetries);
|
|
const message = `AgentRun runner Job creation failed after durable run/command admission; Code Agent stopped this turn instead of silently waiting. failureKind=${failureKind}`;
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "error",
|
|
eventType: "error",
|
|
status: "failed",
|
|
label: `agentrun:runner-job:create-async-failed:${failureKind}`,
|
|
errorCode: failureKind,
|
|
failureKind,
|
|
retryAttempt: Number.isFinite(retryAttempt) ? retryAttempt : 0,
|
|
retryMax: Number.isFinite(retryMax) ? retryMax : null,
|
|
retryExhausted: error?.retryExhausted === true,
|
|
runId: mapping.runId ?? null,
|
|
commandId: mapping.commandId ?? null,
|
|
waitingFor: "agentrun-runner-job-failed",
|
|
terminal: true,
|
|
message,
|
|
valuesPrinted: false
|
|
}, mapping || backendProfile));
|
|
const current = options.codeAgentChatResults?.get?.(traceId);
|
|
const base = current && typeof current === "object"
|
|
? current
|
|
: decorateAgentRunRunningResult({ base: initialAgentRunChatResult({ params, options, traceId }), mapping, traceStore, traceId });
|
|
const failed = agentRunResultToCodeAgentPayload({
|
|
base: { ...base, agentRun: { ...mapping, status: "runner-job-failed", commandState: "failed", terminalStatus: "failed", failureKind, valuesPrinted: false } },
|
|
result: { terminalStatus: "failed", failureKind, failureMessage: message, runId: mapping.runId ?? null, commandId: mapping.commandId ?? null, valuesPrinted: false },
|
|
traceStore,
|
|
traceId,
|
|
appendResultEvent: false
|
|
});
|
|
options.codeAgentChatResults?.set?.(traceId, failed);
|
|
}
|
|
|
|
async function agentRunDispatchJson({ fetchImpl, managerUrl, path, method = "GET", body = undefined, timeoutMs, env = process.env, traceStore = defaultCodeAgentTraceStore, traceId, backendProfile, runId = null, commandId = null, stage = "agentrun-dispatch", terminalOnFailure = true } = {}) {
|
|
const policy = agentRunDispatchRetryPolicy(env);
|
|
let retryAttempt = 0;
|
|
for (;;) {
|
|
try {
|
|
return await agentRunJson(fetchImpl, managerUrl, path, { method, body, timeoutMs, env });
|
|
} catch (error) {
|
|
const failureKind = agentRunDispatchFailureKind(error);
|
|
const retryable = agentRunDispatchRetryable(error, failureKind);
|
|
if (!retryable || retryAttempt >= policy.maxRetries) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "error",
|
|
eventType: "error",
|
|
status: terminalOnFailure ? "failed" : "blocked",
|
|
label: retryable ? `agentrun:dispatch-retry-exhausted:${failureKind}` : `agentrun:dispatch-failed:${failureKind}`,
|
|
errorCode: failureKind,
|
|
failureKind,
|
|
retryable,
|
|
willRetry: false,
|
|
retryAttempt,
|
|
retryMax: policy.maxRetries,
|
|
retryExhausted: retryable && retryAttempt >= policy.maxRetries,
|
|
runId,
|
|
commandId,
|
|
waitingFor: terminalOnFailure ? "agentrun-dispatch-failed" : "agentrun-dispatch-recovery",
|
|
message: retryable
|
|
? `AgentRun dispatch transient ${failureKind}; retry ${retryAttempt}/${policy.maxRetries} exhausted at stage ${stage}. ${terminalOnFailure ? "Code Agent stopped this turn." : "HWLAB will continue with the next recovery path for this turn."}`
|
|
: `AgentRun dispatch failed with non-retryable ${failureKind} at stage ${stage}. ${terminalOnFailure ? "Code Agent stopped this turn." : "HWLAB will continue with the next recovery path for this turn."}`,
|
|
terminal: terminalOnFailure,
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
throw Object.assign(error, { retryAttempt, retryMax: policy.maxRetries, failureKind, retryExhausted: retryable && retryAttempt >= policy.maxRetries });
|
|
}
|
|
retryAttempt += 1;
|
|
const retryDelayMs = agentRunDispatchRetryDelayMs(policy, retryAttempt);
|
|
const nextRetryAt = new Date(Date.now() + retryDelayMs).toISOString();
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "provider_retry",
|
|
eventType: "backend",
|
|
status: "retrying",
|
|
label: `agentrun:dispatch-retry:${failureKind}`,
|
|
errorCode: failureKind,
|
|
failureKind,
|
|
retryable: true,
|
|
willRetry: true,
|
|
retryAttempt,
|
|
retryMax: policy.maxRetries,
|
|
retryDelayMs,
|
|
nextRetryAt,
|
|
runId,
|
|
commandId,
|
|
waitingFor: "agentrun-dispatch-retry",
|
|
message: `AgentRun dispatch transient ${failureKind}; retry ${retryAttempt}/${policy.maxRetries} after ${retryDelayMs}ms at stage ${stage}.`,
|
|
terminal: false,
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
await sleepAgentRunDispatchRetry(retryDelayMs);
|
|
}
|
|
}
|
|
}
|
|
|
|
function agentRunDispatchRetryPolicy(env = process.env) {
|
|
return {
|
|
maxRetries: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX, 5),
|
|
baseDelayMs: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_BASE_MS, 1000),
|
|
maxDelayMs: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX_DELAY_MS, 30000)
|
|
};
|
|
}
|
|
|
|
function agentRunDispatchRetryDelayMs(policy, retryAttempt) {
|
|
const base = Math.max(1, Number(policy.baseDelayMs ?? 1000));
|
|
const cap = Math.max(base, Number(policy.maxDelayMs ?? 30000));
|
|
return Math.min(cap, Math.trunc(base * Math.pow(2, Math.max(0, Number(retryAttempt ?? 1) - 1))));
|
|
}
|
|
|
|
const RETRYABLE_AGENTRUN_TRANSPORT_FAILURE_KINDS = new Set([
|
|
"agentrun-connect-failed",
|
|
"agentrun-proxy-exec-failed",
|
|
"agentrun-manager-fetch-failed",
|
|
"agentrun-timeout",
|
|
"timeout",
|
|
"fetch-failed",
|
|
"network-error"
|
|
]);
|
|
|
|
function isRetryableAgentRunTransportFailureKind(failureKind) {
|
|
return RETRYABLE_AGENTRUN_TRANSPORT_FAILURE_KINDS.has(normalizedFailureKind(failureKind));
|
|
}
|
|
|
|
function agentRunDispatchRetryable(error, failureKind) {
|
|
const status = Number(error?.statusCode ?? error?.status ?? error?.agentRunError?.statusCode ?? 0);
|
|
if ([429, 502, 503, 504].includes(status)) return true;
|
|
if (["provider-unavailable", "provider-stream-disconnected"].includes(failureKind)) return true;
|
|
if (isRetryableAgentRunTransportFailureKind(failureKind)) return true;
|
|
return /ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|fetch failed|network|timeout/iu.test(String(error?.message ?? ""));
|
|
}
|
|
|
|
function agentRunDispatchFailureKind(error) {
|
|
return normalizedFailureKind(error?.agentRunError?.failureKind ?? error?.failureKind ?? error?.code ?? error?.name ?? (Number(error?.statusCode) ? `http-${error.statusCode}` : "agentrun-dispatch-error"));
|
|
}
|
|
|
|
function sleepAgentRunDispatchRetry(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0)));
|
|
}
|
|
|
|
export async function syncAgentRunChatResult({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore, appendResultEvent = true, refreshEvents = true, forceResultSync = false }) {
|
|
const initial = currentResult ?? await loadPersistedAgentRunResult(traceId, options);
|
|
if (isAgentRunUserCancelSealed(initial)) {
|
|
const runnerTrace = initial.runnerTrace ?? traceStore.snapshot(traceId);
|
|
return { result: initial, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true };
|
|
}
|
|
const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial;
|
|
if (isAgentRunUserCancelSealed(mapped)) {
|
|
const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId);
|
|
return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true };
|
|
}
|
|
const performanceStore = options.backendPerformanceStore ?? currentBackendPerformanceContext();
|
|
if (!forceResultSync && options.skipAgentRunTerminalRefreshIfComplete === true && agentRunTerminalResultRefreshSatisfied(mapped, traceId)) {
|
|
const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId);
|
|
return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true };
|
|
}
|
|
if (!mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return { result: currentResult, runnerTrace: traceStore.snapshot(traceId), found: Boolean(currentResult), eventsRefreshed: false, resultSynced: false };
|
|
const env = options.env ?? process.env;
|
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
|
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
|
const projectionState = await loadWorkbenchProjectionStateForAgentRun(options.runtimeStore, { traceId, agentRun: mapped.agentRun });
|
|
const fetchPlan = buildAgentRunProjectionEventsFetchPlan({
|
|
projectionState,
|
|
agentRun: mapped.agentRun,
|
|
pageLimit: env.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT
|
|
});
|
|
const previousLastSeq = Number(fetchPlan.afterSeq ?? mapped.agentRun.lastSeq ?? 0);
|
|
const eventsResponse = refreshEvents ? await measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, lastSeq: fetchPlan.afterSeq, afterSeqOverride: fetchPlan.afterSeq, eventsPageLimit: fetchPlan.limit, traceSummary: mapped.traceSummary } }) : null;
|
|
if (eventsResponse) appendAgentRunEventsToTrace(traceStore, traceId, eventsResponse.events, mapped.agentRun);
|
|
const nextLastSeq = eventsResponse ? agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq) : mapped.agentRun.lastSeq;
|
|
recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse, previousLastSeq, nextLastSeq);
|
|
const terminalFromEvents = agentRunTerminalStatusFromEvents(eventsResponse?.events);
|
|
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
|
currentState: projectionState,
|
|
result: mapped,
|
|
agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq },
|
|
eventsResponse,
|
|
resultSyncState: terminalFromEvents ? "pending" : null,
|
|
projectionStatus: terminalFromEvents ? "terminal" : null,
|
|
now: options.now
|
|
});
|
|
const deferResultSync = agentRunResultSyncDeferred({ forceResultSync, options, env });
|
|
if (!forceResultSync && (deferResultSync || !agentRunCommandResultSyncRequired(mapped, eventsResponse))) {
|
|
const nextMapping = withAgentRunRunnerJobCount({
|
|
...mapped.agentRun,
|
|
lastSeq: nextLastSeq,
|
|
status: mapped.agentRun.status ?? "running",
|
|
runStatus: mapped.agentRun.runStatus ?? null,
|
|
commandState: mapped.agentRun.commandState ?? null,
|
|
terminalStatus: null,
|
|
resultSyncState: terminalFromEvents ? "pending" : mapped.agentRun.resultSyncState ?? null,
|
|
updatedAt: nowIso(options.now),
|
|
valuesPrinted: false
|
|
});
|
|
const payload = decorateAgentRunRunningResult({ base: { ...mapped, status: "running", agentRun: nextMapping, updatedAt: nowIso(options.now) }, mapping: nextMapping, traceStore, traceId });
|
|
options.codeAgentChatResults?.set?.(traceId, payload);
|
|
return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false };
|
|
}
|
|
let result;
|
|
try {
|
|
result = await measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped, eventsResponse });
|
|
} catch (error) {
|
|
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
|
currentState: projectionState,
|
|
result: mapped,
|
|
agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq },
|
|
eventsResponse,
|
|
resultSyncState: agentRunMetricStatusForError(error) === "timeout" ? "timed_out" : "failed",
|
|
projectionStatus: terminalFromEvents ? "terminal" : null,
|
|
error,
|
|
now: options.now
|
|
});
|
|
throw error;
|
|
}
|
|
const nextMapping = withAgentRunRunnerJobCount({
|
|
...mapped.agentRun,
|
|
...agentRunResultRefs(result),
|
|
lastSeq: nextLastSeq,
|
|
status: result?.status ?? mapped.agentRun.status ?? "running",
|
|
runStatus: result?.runStatus ?? mapped.agentRun.runStatus ?? null,
|
|
commandState: result?.commandState ?? mapped.agentRun.commandState ?? null,
|
|
terminalStatus: result?.terminalStatus ?? mapped.agentRun.terminalStatus ?? null,
|
|
updatedAt: nowIso(options.now),
|
|
valuesPrinted: false
|
|
});
|
|
const base = { ...mapped, agentRun: nextMapping, updatedAt: nowIso(options.now) };
|
|
const payload = agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent });
|
|
options.codeAgentChatResults?.set?.(traceId, payload);
|
|
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
|
currentState: projectionState,
|
|
result: payload,
|
|
agentRun: nextMapping,
|
|
eventsResponse,
|
|
resultSyncState: "synced",
|
|
projectionStatus: isAgentRunTerminalStatus(payload?.status) ? "terminal" : null,
|
|
now: options.now
|
|
});
|
|
return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: true };
|
|
}
|
|
|
|
async function loadWorkbenchProjectionStateForAgentRun(runtimeStore, { traceId, agentRun = {} } = {}) {
|
|
if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null;
|
|
try {
|
|
const result = await runtimeStore.getWorkbenchProjectionState({ traceId });
|
|
const state = result?.projectionState ?? null;
|
|
if (!state) return null;
|
|
if (text(state.runId ?? state.sourceRunId) !== text(agentRun.runId)) return null;
|
|
if (text(state.commandId ?? state.sourceCommandId) !== text(agentRun.commandId)) return null;
|
|
return state;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function writeWorkbenchProjectionStateForAgentRun(runtimeStore, params = {}) {
|
|
if (!runtimeStore || typeof runtimeStore.writeWorkbenchProjectionState !== "function") return null;
|
|
const projectionState = buildWorkbenchProjectionStateUpdate(params);
|
|
if (!projectionState.traceId || !projectionState.runId || !projectionState.commandId) return null;
|
|
return await runtimeStore.writeWorkbenchProjectionState({ projectionState });
|
|
}
|
|
|
|
function agentRunCommandResultSyncRequired(mapped = {}, eventsResponse = null) {
|
|
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {};
|
|
if (agentRunTerminalResultRefreshSatisfied(mapped, mapped?.traceId)) return true;
|
|
if (isAgentRunTerminalStatus(agentRun.terminalStatus ?? agentRun.commandState)) return true;
|
|
return Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events));
|
|
}
|
|
|
|
export function agentRunTerminalResultRefreshSatisfied(mapped = {}, traceId = null) {
|
|
const safeId = safeTraceId(traceId ?? mapped?.traceId);
|
|
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null;
|
|
const traceSummary = mapped?.traceSummary && typeof mapped.traceSummary === "object" ? mapped.traceSummary : null;
|
|
const finalResponse = mapped?.finalResponse && typeof mapped.finalResponse === "object" ? mapped.finalResponse : null;
|
|
const providerTrace = agentRun?.providerTrace && typeof agentRun.providerTrace === "object"
|
|
? agentRun.providerTrace
|
|
: mapped?.providerTrace && typeof mapped.providerTrace === "object"
|
|
? mapped.providerTrace
|
|
: null;
|
|
const commandId = text(agentRun?.commandId);
|
|
const runId = text(agentRun?.runId);
|
|
if (!safeId || !runId || !commandId) return false;
|
|
if (!isAgentRunTerminalStatus(mapped?.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? agentRun?.status)) return false;
|
|
if (safeTraceId(mapped?.traceId ?? agentRun?.traceId) !== safeId) return false;
|
|
if (safeTraceId(providerTrace?.traceId) !== safeId) return false;
|
|
const finalTraceId = safeTraceId(finalResponse?.traceId ?? safeId);
|
|
if (finalTraceId && finalTraceId !== safeId) return false;
|
|
if (!String(finalResponse?.text ?? "").trim()) return false;
|
|
if (traceSummary?.source !== "agentrun-command-result") return false;
|
|
if (safeTraceId(traceSummary.traceId ?? safeId) !== safeId) return false;
|
|
const summaryAgentRun = traceSummary.agentRun && typeof traceSummary.agentRun === "object" ? traceSummary.agentRun : null;
|
|
if (text(summaryAgentRun?.runId) !== runId) return false;
|
|
if (text(summaryAgentRun?.commandId) !== commandId) return false;
|
|
return true;
|
|
}
|
|
|
|
function agentRunTerminalStatusFromEvents(events = []) {
|
|
if (!Array.isArray(events)) return null;
|
|
for (const event of [...events].reverse()) {
|
|
const payload = event?.payload && typeof event.payload === "object" ? event.payload : {};
|
|
const phase = payload.phase ?? event?.phase;
|
|
const candidates = [
|
|
event?.type === "terminal_status" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null,
|
|
phase === "command-terminal" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null,
|
|
phase === "turn:completed" || phase === "turn/steer:completed" ? "completed" : null,
|
|
event?.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null,
|
|
payload.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null
|
|
];
|
|
for (const value of candidates) {
|
|
const status = normalizeAgentRunStatus(value);
|
|
if (isAgentRunTerminalStatus(status)) return status;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function isAgentRunTerminalStatus(value) {
|
|
const status = normalizeAgentRunStatus(value);
|
|
return status === "timeout" || TERMINAL_RUN_STATUSES.has(status);
|
|
}
|
|
|
|
function normalizeAgentRunStatus(value) {
|
|
const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
|
return status === "cancelled" ? "canceled" : status;
|
|
}
|
|
|
|
function isAgentRunUserCancelSealed(payload = null) {
|
|
if (!payload || typeof payload !== "object") return false;
|
|
const status = normalizeAgentRunStatus(payload.status);
|
|
const cancelStatus = normalizeAgentRunStatus(payload.cancelStatus ?? payload.cancelDisposition?.status);
|
|
const terminalStatus = normalizeAgentRunStatus(payload.agentRun?.terminalStatus ?? payload.agentRun?.commandState ?? payload.agentRun?.status);
|
|
return payload.canceled === true || status === "canceled" || cancelStatus === "canceled" || terminalStatus === "canceled";
|
|
}
|
|
|
|
async function resolveAgentRunTraceCommandMapping({ traceId, mapped, options = {} }) {
|
|
const safeId = safeTraceId(traceId);
|
|
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null;
|
|
if (!safeId || !agentRun?.runId) return mapped;
|
|
const agentRunTraceId = safeTraceId(agentRun.traceId);
|
|
const providerTraceId = safeTraceId(agentRun.providerTrace?.traceId);
|
|
const traceSummaryCommandId = text(mapped?.traceSummary?.agentRun?.commandId);
|
|
const commandId = text(agentRun.commandId);
|
|
const traceFieldsMatch = agentRunTraceId === safeId && providerTraceId === safeId && (!traceSummaryCommandId || traceSummaryCommandId === commandId);
|
|
if (traceFieldsMatch && (mapped.status === "running" || (options.skipAgentRunTerminalRefreshIfComplete === true && agentRunTerminalResultRefreshSatisfied(mapped, safeId)))) return mapped;
|
|
const env = options.env ?? process.env;
|
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
const managerUrl = resolveAgentRunManagerUrl(env, agentRun.managerUrl);
|
|
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
|
const command = await findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId: agentRun.runId, traceId: safeId });
|
|
if (!command?.id) {
|
|
throw Object.assign(new Error(`AgentRun command registry has no command for ${safeId} in run ${agentRun.runId}`), {
|
|
code: "agentrun_trace_command_not_found",
|
|
statusCode: 404,
|
|
traceId: safeId,
|
|
runId: agentRun.runId
|
|
});
|
|
}
|
|
if (command.id === commandId) {
|
|
return {
|
|
...mapped,
|
|
finalResponse: null,
|
|
traceSummary: null,
|
|
agentRun: {
|
|
...agentRun,
|
|
traceId: safeId,
|
|
lastSeq: 0,
|
|
providerTrace: null,
|
|
commandState: command.state ?? agentRun.commandState ?? null,
|
|
updatedAt: nowIso(options.now),
|
|
valuesPrinted: false
|
|
}
|
|
};
|
|
}
|
|
return {
|
|
...mapped,
|
|
finalResponse: null,
|
|
traceSummary: null,
|
|
agentRun: {
|
|
...agentRun,
|
|
commandId: command.id,
|
|
traceId: safeId,
|
|
lastSeq: 0,
|
|
providerTrace: null,
|
|
commandState: command.state ?? agentRun.commandState ?? null,
|
|
updatedAt: nowIso(options.now),
|
|
valuesPrinted: false
|
|
}
|
|
};
|
|
}
|
|
|
|
export async function refreshAgentRunTrace({ traceId, result = null, options = {}, traceStore = defaultCodeAgentTraceStore }) {
|
|
const initial = result ?? await loadPersistedAgentRunResult(traceId, options);
|
|
const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial;
|
|
if (!mapped?.agentRun?.runId) return traceStore.snapshot(traceId);
|
|
const env = options.env ?? process.env;
|
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
|
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
|
const eventsResponse = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, traceSummary: mapped.traceSummary } });
|
|
const events = eventsResponse.events;
|
|
appendAgentRunEventsToTrace(traceStore, traceId, events, mapped.agentRun);
|
|
const lastSeq = agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq);
|
|
if (lastSeq !== Number(mapped.agentRun.lastSeq ?? 0)) {
|
|
options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun: { ...mapped.agentRun, lastSeq, updatedAt: nowIso(options.now) } });
|
|
}
|
|
return traceStore.snapshot(traceId);
|
|
}
|
|
|
|
export async function cancelAgentRunChatTurn({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore }) {
|
|
let mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options);
|
|
if (!mapped?.agentRun?.commandId) return null;
|
|
const env = options.env ?? process.env;
|
|
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
|
const cancelTimeoutMs = Math.min(timeoutMs, parsePositiveInteger(env.HWLAB_CODE_AGENT_CANCEL_TIMEOUT_MS, 5_000));
|
|
const cancelOptions = {
|
|
...options,
|
|
env: {
|
|
...env,
|
|
HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: String(cancelTimeoutMs),
|
|
HWLAB_CODE_AGENT_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs)
|
|
}
|
|
};
|
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
|
try {
|
|
const synced = await syncAgentRunChatResult({ traceId, currentResult: mapped, options: cancelOptions, traceStore, forceResultSync: true });
|
|
mapped = synced?.result ?? synced ?? mapped;
|
|
} catch {
|
|
// Cancel must still expose the command cancel disposition; result refresh failures are reported by the normal trace/result paths.
|
|
}
|
|
const terminalStatus = agentRunCancelTerminalStatus(mapped);
|
|
if (terminalStatus) return agentRunAlreadyTerminalCancelPayload({ traceId, payload: mapped, terminalStatus, options: cancelOptions, traceStore });
|
|
const cancelPath = `/api/v1/commands/${encodeURIComponent(mapped.agentRun.commandId)}/cancel`;
|
|
try {
|
|
await agentRunJson(fetchImpl, managerUrl, cancelPath, {
|
|
method: "POST",
|
|
body: { reason: "hwlab-user-cancel", traceId },
|
|
timeoutMs: cancelTimeoutMs,
|
|
env
|
|
});
|
|
} catch (error) {
|
|
return agentRunCancelBlockedPayload({ traceId, payload: mapped, error, cancelPath, options: cancelOptions, traceStore });
|
|
}
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "cancel",
|
|
status: "canceled",
|
|
label: "agentrun:cancel:canceled",
|
|
message: "HWLAB forwarded cancel to AgentRun command cancel API.",
|
|
runId: mapped.agentRun.runId,
|
|
commandId: mapped.agentRun.commandId,
|
|
terminal: true,
|
|
valuesPrinted: false
|
|
}, mapped.agentRun));
|
|
const now = nowIso(options.now);
|
|
const agentRun = { ...mapped.agentRun, status: "cancelled", commandState: "cancelled", terminalStatus: "cancelled", updatedAt: now };
|
|
const payload = {
|
|
...mapped,
|
|
status: "canceled",
|
|
canceled: true,
|
|
updatedAt: now,
|
|
agentRun,
|
|
runnerTrace: traceStore.snapshot(traceId),
|
|
error: {
|
|
code: "agentrun_canceled",
|
|
layer: "agentrun",
|
|
category: "canceled",
|
|
retryable: true,
|
|
message: "AgentRun command was canceled by user request",
|
|
userMessage: "当前 AgentRun 请求已取消;traceId/runId/commandId 已保留,可重试上一条消息。",
|
|
traceId,
|
|
route: "/v1/agent/chat/cancel",
|
|
toolName: "agentrun.command.cancel"
|
|
},
|
|
userMessage: "当前 AgentRun 请求已取消;traceId/runId/commandId 已保留,可重试上一条消息。"
|
|
};
|
|
options.codeAgentChatResults?.set?.(traceId, payload);
|
|
return payload;
|
|
}
|
|
|
|
export async function steerAgentRunChatTurn({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore }) {
|
|
const mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options);
|
|
if (!safeTraceId(traceId) || !mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return null;
|
|
const message = firstNonEmpty(params.message, params.prompt, params.text);
|
|
if (!message) {
|
|
throw Object.assign(new Error("steer command requires non-empty message text"), { code: "steer_message_missing", statusCode: 400 });
|
|
}
|
|
const env = options.env ?? process.env;
|
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
|
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
|
const steerTraceId = safeTraceId(params.steerTraceId ?? params.steerId) || `trc_steer_${randomUUID().replace(/-/gu, "")}`;
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:steer:accepted",
|
|
message: "HWLAB accepted an in-flight steer request and will create an AgentRun type=steer command on the existing run.",
|
|
runId: mapped.agentRun.runId,
|
|
commandId: mapped.agentRun.commandId,
|
|
steerTraceId,
|
|
waitingFor: "agentrun-steer-command-create",
|
|
valuesPrinted: false
|
|
}, mapped.agentRun));
|
|
const commandInput = buildAgentRunSteerCommandInput({ params: { ...params, message }, traceId, steerTraceId, mapped });
|
|
const command = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(mapped.agentRun.runId)}/commands`, {
|
|
method: "POST",
|
|
body: commandInput,
|
|
timeoutMs,
|
|
env
|
|
});
|
|
const steerCommandId = requiredString(command?.id, "command.id");
|
|
const now = nowIso(options.now);
|
|
const agentRun = {
|
|
...mapped.agentRun,
|
|
lastSteerCommandId: steerCommandId,
|
|
lastSteerTraceId: steerTraceId,
|
|
updatedAt: now,
|
|
valuesPrinted: false
|
|
};
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:steer:command-created",
|
|
message: `AgentRun steer command ${steerCommandId} created on run ${mapped.agentRun.runId}; runner will apply it if the target Codex turn is still active.`,
|
|
runId: mapped.agentRun.runId,
|
|
commandId: mapped.agentRun.commandId,
|
|
steerCommandId,
|
|
targetCommandId: mapped.agentRun.commandId,
|
|
steerTraceId,
|
|
waitingFor: "agentrun-steer-apply",
|
|
valuesPrinted: false
|
|
}, agentRun));
|
|
const payload = {
|
|
ok: true,
|
|
accepted: true,
|
|
status: "running",
|
|
shortConnection: true,
|
|
controlSemantics: "steer-active-turn-and-poll-target-trace",
|
|
route: "/v1/agent/chat/steer",
|
|
traceId,
|
|
targetTraceId: traceId,
|
|
steerTraceId,
|
|
conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null,
|
|
sessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null,
|
|
threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null,
|
|
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
|
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
|
agentRun: {
|
|
...agentRun,
|
|
steerCommandId,
|
|
targetCommandId: mapped.agentRun.commandId,
|
|
steerCommandState: command.state ?? null,
|
|
valuesPrinted: false
|
|
},
|
|
runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})),
|
|
updatedAt: now,
|
|
valuesPrinted: false
|
|
};
|
|
options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun, runnerTrace: payload.runnerTrace, updatedAt: now });
|
|
return payload;
|
|
}
|
|
|
|
export async function loadPersistedAgentRunResult(traceId, options = {}) {
|
|
const safeId = safeTraceId(traceId);
|
|
if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null;
|
|
const session = await loadAgentSessionByTraceId(safeId, options);
|
|
const agentRun = agentRunSeedFromSession(session, safeId);
|
|
if (!agentRun?.runId) return null;
|
|
const backendProfile = firstNonEmpty(agentRun.backendProfile, "deepseek");
|
|
const status = firstNonEmpty(session?.status === "canceled" ? "canceled" : null, agentRun.terminalStatus, agentRun.commandState, agentRun.status, "running");
|
|
const promptEvidence = agentRunPromptMetadataFromSession(session, safeId);
|
|
return {
|
|
accepted: true,
|
|
status,
|
|
shortConnection: true,
|
|
traceId: safeId,
|
|
conversationId: safeConversationId(session?.conversationId) || agentRun.conversationId || null,
|
|
sessionId: safeSessionId(session?.id) || agentRun.sessionId || null,
|
|
threadId: safeOpaqueId(session?.threadId) || agentRun.threadId || null,
|
|
messageId: `msg_${safeId.slice(4)}`,
|
|
...agentRunPromptFields(promptEvidence),
|
|
createdAt: session?.startedAt ?? session?.updatedAt ?? nowIso(options.now),
|
|
updatedAt: session?.updatedAt ?? nowIso(options.now),
|
|
ownerUserId: session?.ownerUserId,
|
|
ownerRole: session?.ownerRole,
|
|
provider: providerForBackendProfile(backendProfile),
|
|
model: modelForBackendProfile(backendProfile, options.env ?? process.env),
|
|
backend: backendForBackendProfile(backendProfile),
|
|
infrastructureBackend: `agentrun-v01/${backendProfile}`,
|
|
capabilityLevel: AGENTRUN_CAPABILITY_LEVEL,
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
...(agentRun?.userBillingReservation ? { userBillingReservation: agentRun.userBillingReservation } : {}),
|
|
agentRun: { ...withoutUserBillingReservation(agentRun), adapter: ADAPTER_ID, traceId: safeId, commandId: text(agentRun.commandId) || null, valuesPrinted: false },
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
async function loadAgentSessionByTraceId(traceId, options = {}) {
|
|
const safeId = safeTraceId(traceId);
|
|
if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null;
|
|
const cache = options.codeAgentTraceSessionCache;
|
|
if (cache instanceof Map) {
|
|
if (cache.has(safeId)) return cache.get(safeId);
|
|
const session = await options.accessController.getAgentSessionByTraceId(safeId);
|
|
cache.set(safeId, session ?? null);
|
|
return session ?? null;
|
|
}
|
|
return await options.accessController.getAgentSessionByTraceId(safeId);
|
|
}
|
|
|
|
function agentRunSeedFromSession(session, traceId) {
|
|
const snapshot = session?.session && typeof session.session === "object" ? session.session : null;
|
|
const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null;
|
|
const traceResult = traceResults?.[traceId] && typeof traceResults[traceId] === "object" ? traceResults[traceId] : null;
|
|
const traceAgentRun = traceResult?.agentRun && typeof traceResult.agentRun === "object" ? traceResult.agentRun : null;
|
|
if (traceAgentRun?.runId) {
|
|
return traceResult?.userBillingReservation
|
|
? { ...traceAgentRun, userBillingReservation: traceResult.userBillingReservation }
|
|
: traceAgentRun;
|
|
}
|
|
const topLevelAgentRun = snapshot?.agentRun && typeof snapshot.agentRun === "object" ? snapshot.agentRun : null;
|
|
return topLevelAgentRun?.runId ? topLevelAgentRun : null;
|
|
}
|
|
|
|
async function findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId, traceId }) {
|
|
if (!runId || !safeTraceId(traceId)) return null;
|
|
const commands = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands?afterSeq=0&limit=100`, { method: "GET", timeoutMs, env });
|
|
return (Array.isArray(commands?.items) ? commands.items : []).find((item) => agentRunCommandMatchesTrace(item, traceId)) ?? null;
|
|
}
|
|
|
|
function agentRunCommandMatchesTrace(command, traceId) {
|
|
const payload = command?.payload && typeof command.payload === "object" ? command.payload : {};
|
|
return command?.idempotencyKey === traceId || payload.traceId === traceId;
|
|
}
|
|
|
|
export function agentRunSessionEvidence(payload = {}) {
|
|
if (!payload?.agentRun) return {};
|
|
return {
|
|
agentRun: {
|
|
adapter: ADAPTER_ID,
|
|
runId: payload.agentRun.runId ?? null,
|
|
commandId: payload.agentRun.commandId ?? null,
|
|
attemptId: payload.agentRun.attemptId ?? null,
|
|
runnerId: payload.agentRun.runnerId ?? null,
|
|
jobName: payload.agentRun.jobName ?? null,
|
|
namespace: payload.agentRun.namespace ?? null,
|
|
backendProfile: payload.agentRun.backendProfile ?? null,
|
|
managerUrl: payload.agentRun.managerUrl ?? DEFAULT_AGENTRUN_MGR_URL,
|
|
repoUrl: payload.agentRun.repoUrl ?? DEFAULT_AGENTRUN_REPO_URL,
|
|
status: payload.agentRun.status ?? null,
|
|
runStatus: payload.agentRun.runStatus ?? null,
|
|
commandState: payload.agentRun.commandState ?? null,
|
|
terminalStatus: payload.agentRun.terminalStatus ?? null,
|
|
traceId: payload.agentRun.traceId ?? payload.traceId ?? payload.providerTrace?.traceId ?? null,
|
|
lastSeq: payload.agentRun.lastSeq ?? 0,
|
|
providerId: payload.agentRun.providerId ?? agentRunProviderIdOrNull(),
|
|
reuseEligible: payload.agentRun.reuseEligible ?? false,
|
|
sessionId: payload.agentRun.sessionId ?? payload.sessionId ?? null,
|
|
conversationId: payload.conversationId ?? payload.agentRun.conversationId ?? null,
|
|
threadId: payload.threadId ?? payload.agentRun.threadId ?? null,
|
|
providerTrace: payload.providerTrace ?? payload.agentRun.providerTrace ?? null,
|
|
valuesPrinted: false
|
|
}
|
|
};
|
|
}
|
|
|
|
function withoutUserBillingReservation(value = {}) {
|
|
if (!value || typeof value !== "object") return value;
|
|
const { userBillingReservation, ...rest } = value;
|
|
return rest;
|
|
}
|
|
|
|
async function ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }) {
|
|
const defaultPolicy = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SESSION_STORAGE, "persistent");
|
|
if (defaultPolicy !== "persistent") return null;
|
|
try {
|
|
const tenantId = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TENANT_ID, DEFAULT_TENANT_ID);
|
|
const projectId = agentRunProjectIdForEnv(env);
|
|
const expiresInDays = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_SESSION_TTL_DAYS, 30);
|
|
const expiresAt = new Date(Date.now() + Math.max(1, expiresInDays) * 24 * 60 * 60 * 1000).toISOString();
|
|
return await agentRunJson(fetchImpl, managerUrl, "/api/v1/sessions", {
|
|
method: "POST",
|
|
timeoutMs: 15_000,
|
|
env,
|
|
body: { sessionId, tenantId, projectId, backendProfile, expiresAt, codexRolloutSubdir: "sessions" },
|
|
});
|
|
} catch (error) {
|
|
const message = error?.message ?? String(error);
|
|
if (/evicted/i.test(message)) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend", status: "running",
|
|
label: "agentrun:session-storage-evicted",
|
|
message: "AgentRun session " + sessionId + " storage was previously evicted; hwlab-cloud-api will create a fresh sessionId for this turn.",
|
|
sessionId, backendProfile, waitingFor: "agentrun-session-reset", valuesPrinted: false,
|
|
}, backendProfile));
|
|
throw error;
|
|
}
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend", status: "running",
|
|
label: "agentrun:session-storage-recover-warning",
|
|
message: "AgentRun session " + sessionId + " pre-flight could not ensure PVC; falling back to legacy metadata-only session (" + message + ").",
|
|
sessionId, backendProfile, valuesPrinted: false,
|
|
}, backendProfile));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function agentRunSessionRecordFromEnsure(value = null) {
|
|
const record = value && typeof value === "object" ? value : null;
|
|
if (!record) return null;
|
|
const session = record.session && typeof record.session === "object" ? record.session : record;
|
|
return session && typeof session === "object" ? session : null;
|
|
}
|
|
|
|
function agentRunSessionThreadIdFromEnsure(value = null, backendProfile = null) {
|
|
const record = agentRunSessionRecordFromEnsure(value);
|
|
if (agentRunSessionThreadInvalidated(record)) return null;
|
|
const existingProfile = firstNonEmpty(record?.backendProfile, record?.metadata?.agentRunSessionProfile);
|
|
if (existingProfile && backendProfile && existingProfile !== backendProfile) return null;
|
|
return safeOpaqueId(record?.threadId) || null;
|
|
}
|
|
|
|
function agentRunSessionThreadInvalidated(record = null) {
|
|
if (!record || typeof record !== "object") return false;
|
|
const failureMessage = firstNonEmpty(
|
|
record.failureMessage,
|
|
record.blocker?.message,
|
|
record.liveness?.terminalClassification?.reason,
|
|
record.supervisor?.terminalClassification?.reason
|
|
);
|
|
if (!agentRunFailureRequiresFreshSession(record.failureKind, failureMessage)) return false;
|
|
const terminalStatus = normalizeAgentRunStatus(record.terminalStatus ?? record.status);
|
|
const executionState = String(record.executionState ?? "").trim().toLowerCase();
|
|
return terminalStatus === "failed" || terminalStatus === "blocked" || executionState === "terminal";
|
|
}
|
|
|
|
function paramsWithAgentRunSessionThread(params = {}, threadId = null) {
|
|
const next = { ...(params ?? {}) };
|
|
const safeThreadId = safeOpaqueId(threadId);
|
|
if (safeThreadId) next.threadId = safeThreadId;
|
|
else delete next.threadId;
|
|
return next;
|
|
}
|
|
|
|
function agentRunProjectIdForEnv(env = process.env) {
|
|
return firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECT_ID, DEFAULT_PROJECT_ID);
|
|
}
|
|
|
|
async function shouldResetSessionAfterEviction(failureKind, failureMessage) {
|
|
return agentRunFailureRequiresFreshSession(failureKind, failureMessage);
|
|
}
|
|
|
|
function agentRunFailureRequiresFreshSession(failureKind, failureMessage) {
|
|
const kind = String(failureKind ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
|
const message = String(failureMessage ?? "");
|
|
return kind === "session-store-evicted" || kind === "thread-resume-failed" || /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|thread\/resume failed/iu.test(message);
|
|
}
|
|
|
|
function newSessionIdAfterEviction(baseSessionId, traceId) {
|
|
const profile = String(traceId).replace(/[^A-Za-z0-9]/gu, "").slice(0, 12) || "fresh";
|
|
return baseSessionId + "-reset-" + profile;
|
|
}
|
|
function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId, toolCapabilities = null }) {
|
|
const commitId = requireAgentRunSourceCommit(env);
|
|
const runtimeAuthority = requireAgentRunRuntimeAuthority(env);
|
|
const threadId = safeOpaqueId(params.threadId);
|
|
const hwlabProjectId = firstNonEmpty(params.projectId);
|
|
rejectRemovedResourceWorkspaceFiles(params.workspaceFiles ?? params.resourceWorkspaceFiles);
|
|
const resourceBundleRef = {
|
|
kind: "gitbundle",
|
|
repoUrl: resolveAgentRunRepoUrl(env),
|
|
commitId,
|
|
submodules: false,
|
|
lfs: false,
|
|
bundles: HWLAB_RESOURCE_GIT_BUNDLES,
|
|
promptRefs: HWLAB_RESOURCE_PROMPT_REFS,
|
|
};
|
|
return {
|
|
tenantId: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TENANT_ID, DEFAULT_TENANT_ID),
|
|
projectId: agentRunProjectIdForEnv(env),
|
|
workspaceRef: {
|
|
kind: "opaque",
|
|
repo: DEFAULT_PROJECT_ID,
|
|
branch: firstNonEmpty(env.HWLAB_BOOT_REF, env.HWLAB_CODE_AGENT_AGENTRUN_BRANCH, defaultAgentRunBranchForRuntimeLane(runtimeAuthority.lane), "v0.2"),
|
|
runtimeNamespace: runtimeAuthority.namespace,
|
|
hwlabTraceId: traceId,
|
|
valuesPrinted: false
|
|
},
|
|
sessionRef: {
|
|
sessionId: sessionId ?? scopedAgentRunSessionIdForParams(params, traceId),
|
|
...(safeConversationId(params.conversationId) ? { conversationId: safeConversationId(params.conversationId) } : {}),
|
|
...(threadId ? { threadId } : {}),
|
|
metadata: {
|
|
adapter: ADAPTER_ID,
|
|
hwlabTraceId: traceId,
|
|
otelTraceId: codeAgentOtelTraceContext(traceId).traceId,
|
|
traceparent: codeAgentOtelTraceContext(traceId).traceparent,
|
|
hwlabApi: "/v1/agent/chat",
|
|
hwlabProjectId,
|
|
hwlabSessionId: safeSessionId(params.sessionId) || null,
|
|
threadContinuityPolicy: THREAD_CONTINUITY_POLICY,
|
|
sessionPolicy: SESSION_POLICY_RUN_LOCAL,
|
|
agentRunSessionProfile: backendProfile,
|
|
agentRunSessionPolicy: "hwlab-session-scoped",
|
|
valuesPrinted: false
|
|
}
|
|
},
|
|
resourceBundleRef,
|
|
providerId: resolveAgentRunProviderId(env),
|
|
backendProfile,
|
|
executionPolicy: {
|
|
sandbox: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SANDBOX, env.HWLAB_CODE_AGENT_CODEX_SANDBOX, "danger-full-access"),
|
|
approval: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_APPROVAL, "never"),
|
|
timeoutMs: parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_TIMEOUT_MS, parsePositiveInteger(env.HWLAB_CODE_AGENT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS)),
|
|
network: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_NETWORK, "enabled"),
|
|
secretScope: {
|
|
allowCredentialEcho: false,
|
|
providerCredentials: [providerCredentialSecretRef(backendProfile, env)],
|
|
toolCredentials: agentRunToolCredentials(env, toolCapabilities)
|
|
}
|
|
},
|
|
traceSink: {
|
|
kind: "hwlab-cloud-api",
|
|
traceId,
|
|
...codeAgentOtelTraceFields(traceId, env),
|
|
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
|
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
|
valuesPrinted: false
|
|
}
|
|
};
|
|
}
|
|
|
|
function rejectRemovedResourceWorkspaceFiles(value) {
|
|
if (value == null || value === false || value === "") return [];
|
|
throw adapterError("legacy_workspace_files_removed", "workspaceFiles/resourceWorkspaceFiles are removed; use AgentRun ResourceBundleRef kind=gitbundle with bundles[]");
|
|
}
|
|
|
|
function adapterError(code, message, details = {}) {
|
|
return Object.assign(new Error(message), {
|
|
code,
|
|
statusCode: 400,
|
|
details
|
|
});
|
|
}
|
|
|
|
function buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId }) {
|
|
const prompt = messageAuthorityTextValue(params.message ?? params.prompt ?? "");
|
|
const promptEvidence = agentRunPromptMetadataFromText(prompt, traceId, "agentrun-command-payload", { fullText: true });
|
|
const threadId = safeOpaqueId(params.threadId);
|
|
return {
|
|
type: "turn",
|
|
payload: {
|
|
prompt,
|
|
message: prompt,
|
|
...agentRunPromptEventFields(promptEvidence),
|
|
...(promptEvidence?.prompt ? { promptMetadata: promptEvidence.prompt } : {}),
|
|
traceId,
|
|
projectId: firstNonEmpty(params.projectId) || null,
|
|
conversationId: safeConversationId(params.conversationId) || null,
|
|
sessionId: sessionId ?? scopedAgentRunSessionIdForParams(params, traceId),
|
|
hwlabSessionId: safeSessionId(params.sessionId) || null,
|
|
threadId: threadId || null,
|
|
threadContinuityPolicy: THREAD_CONTINUITY_POLICY,
|
|
sessionPolicy: SESSION_POLICY_RUN_LOCAL,
|
|
providerProfile: backendProfile,
|
|
source: "hwlab-cloud-api",
|
|
valuesPrinted: false
|
|
},
|
|
idempotencyKey: traceId
|
|
};
|
|
}
|
|
|
|
function buildAgentRunSteerCommandInput({ params, traceId, steerTraceId, mapped }) {
|
|
const prompt = messageAuthorityTextValue(params.message ?? params.prompt ?? params.text ?? "");
|
|
const promptEvidence = agentRunPromptMetadataFromText(prompt, steerTraceId, "agentrun-steer-command-payload", { fullText: true });
|
|
return {
|
|
type: "steer",
|
|
payload: {
|
|
prompt,
|
|
message: prompt,
|
|
text: prompt,
|
|
...agentRunPromptEventFields(promptEvidence),
|
|
...(promptEvidence?.prompt ? { promptMetadata: promptEvidence.prompt } : {}),
|
|
traceId: steerTraceId,
|
|
targetTraceId: traceId,
|
|
targetCommandId: mapped.agentRun.commandId,
|
|
conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null,
|
|
sessionId: mapped.agentRun.sessionId ?? safeSessionId(mapped.sessionId ?? params.sessionId) ?? null,
|
|
hwlabSessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null,
|
|
threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null,
|
|
source: "hwlab-cloud-api",
|
|
valuesPrinted: false
|
|
},
|
|
idempotencyKey: steerTraceId
|
|
};
|
|
}
|
|
|
|
async function resolveOwnerApiKey({ params, options, now }) {
|
|
const ownerUserId = text(params.ownerUserId);
|
|
if (!ownerUserId) return "";
|
|
const accessController = options.accessController;
|
|
if (!accessController?.store?.findActiveDefaultApiKeyForUser) return "";
|
|
const key = await accessController.store.findActiveDefaultApiKeyForUser(ownerUserId) ?? null;
|
|
if (!key) return "";
|
|
return text(key.displaySecret ?? "");
|
|
}
|
|
function buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities = null, backendProfile }) {
|
|
const runtimeAuthority = requireAgentRunRuntimeAuthority(env);
|
|
const baseTransient = buildAgentRunTransientEnv(env, { providerProfile: backendProfile, parentTraceId: traceId });
|
|
assertAgentRunTransientRuntimeAuthority(baseTransient, runtimeAuthority);
|
|
const hwpodAllowed = toolCapabilityAllowed(toolCapabilities, "hwpod");
|
|
const transientEnv = ownerApiKey && hwpodAllowed
|
|
? baseTransient.concat([{ name: "HWLAB_API_KEY", value: ownerApiKey, sensitive: true }])
|
|
: baseTransient;
|
|
return {
|
|
commandId,
|
|
idempotencyKey: traceId,
|
|
namespace: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE, env.AGENTRUN_RUNTIME_NAMESPACE, DEFAULT_RUNNER_NAMESPACE),
|
|
...(firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_IMAGE, env.AGENTRUN_RUNNER_IMAGE)
|
|
? { image: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_IMAGE, env.AGENTRUN_RUNNER_IMAGE) }
|
|
: {}),
|
|
...(firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_SERVICE_ACCOUNT, env.AGENTRUN_RUNNER_SERVICE_ACCOUNT)
|
|
? { serviceAccountName: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_SERVICE_ACCOUNT, env.AGENTRUN_RUNNER_SERVICE_ACCOUNT) }
|
|
: {}),
|
|
...(transientEnv.length > 0 ? { transientEnv } : {})
|
|
};
|
|
}
|
|
|
|
function buildAgentRunTransientEnv(env = process.env, options = {}) {
|
|
const providerProfile = firstNonEmpty(options.providerProfile);
|
|
const parentTraceId = firstNonEmpty(options.parentTraceId);
|
|
const entries = [];
|
|
for (const name of [
|
|
"HWLAB_RUNTIME_API_URL",
|
|
"HWLAB_RUNTIME_WEB_URL",
|
|
"HWLAB_RUNTIME_NAMESPACE",
|
|
"HWLAB_RUNTIME_LANE",
|
|
"HWLAB_RUNTIME_ENDPOINT_LOCKED",
|
|
"HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME",
|
|
"UNIDESK_MAIN_SERVER_IP"
|
|
]) {
|
|
const value = firstNonEmpty(env[name]);
|
|
if (value) entries.push({ name, value, sensitive: false });
|
|
}
|
|
if (providerProfile) entries.push({ name: "HWLAB_CODE_AGENT_PROVIDER_PROFILE", value: providerProfile, sensitive: false });
|
|
if (parentTraceId) entries.push({ name: "HWLAB_CODE_AGENT_PARENT_TRACE_ID", value: parentTraceId, sensitive: false });
|
|
return entries;
|
|
}
|
|
|
|
function requireAgentRunRuntimeAuthority(env = process.env) {
|
|
const explicitLane = normalizeAgentRunRuntimeLane(firstNonEmpty(
|
|
env.HWLAB_RUNTIME_LANE,
|
|
env.HWLAB_GITOPS_LANE,
|
|
env.HWLAB_GITOPS_PROFILE,
|
|
env.HWLAB_ENVIRONMENT,
|
|
env.HWLAB_BOOT_REF
|
|
));
|
|
const namespace = firstNonEmpty(
|
|
env.HWLAB_RUNTIME_NAMESPACE,
|
|
env.POD_NAMESPACE,
|
|
env.HWLAB_NAMESPACE,
|
|
namespaceForAgentRunRuntimeLane(explicitLane)
|
|
);
|
|
if (!namespace) {
|
|
throw agentRunRuntimeAuthorityError("agentrun_runtime_namespace_required", "AgentRun dispatch requires HWLAB_RUNTIME_NAMESPACE/POD_NAMESPACE/HWLAB_NAMESPACE or a supported runtime lane");
|
|
}
|
|
const namespaceLane = agentRunRuntimeLaneForNamespace(namespace);
|
|
if (!namespaceLane) {
|
|
throw agentRunRuntimeAuthorityError("agentrun_runtime_namespace_unsupported", `unsupported AgentRun HWLAB runtime namespace: ${namespace}`);
|
|
}
|
|
if (explicitLane && explicitLane !== namespaceLane) {
|
|
throw agentRunRuntimeAuthorityError("agentrun_runtime_authority_mismatch", `AgentRun HWLAB runtime lane ${explicitLane} does not match namespace ${namespace} (${namespaceLane})`);
|
|
}
|
|
assertAgentRunEndpointNamespace(env.HWLAB_RUNTIME_API_URL, namespace, "HWLAB_RUNTIME_API_URL");
|
|
assertAgentRunEndpointNamespace(env.HWLAB_RUNTIME_WEB_URL, namespace, "HWLAB_RUNTIME_WEB_URL");
|
|
return { namespace, lane: namespaceLane };
|
|
}
|
|
|
|
function assertAgentRunTransientRuntimeAuthority(entries, runtimeAuthority) {
|
|
const values = new Map((Array.isArray(entries) ? entries : []).map((entry) => [entry?.name, firstNonEmpty(entry?.value)]));
|
|
if (values.get("HWLAB_RUNTIME_NAMESPACE") !== runtimeAuthority.namespace) {
|
|
throw agentRunRuntimeAuthorityError("agentrun_transient_namespace_mismatch", `AgentRun transient HWLAB_RUNTIME_NAMESPACE must be ${runtimeAuthority.namespace}`);
|
|
}
|
|
if (normalizeAgentRunRuntimeLane(values.get("HWLAB_RUNTIME_LANE")) !== runtimeAuthority.lane) {
|
|
throw agentRunRuntimeAuthorityError("agentrun_transient_lane_mismatch", `AgentRun transient HWLAB_RUNTIME_LANE must be ${runtimeAuthority.lane}`);
|
|
}
|
|
for (const name of ["HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL"]) {
|
|
const value = values.get(name);
|
|
if (!value) {
|
|
throw agentRunRuntimeAuthorityError("agentrun_transient_endpoint_required", `AgentRun transient ${name} is required`);
|
|
}
|
|
assertAgentRunEndpointNamespace(value, runtimeAuthority.namespace, name);
|
|
}
|
|
}
|
|
|
|
function assertAgentRunEndpointNamespace(value, namespace, fieldName) {
|
|
const endpointNamespace = agentRunRuntimeNamespaceForEndpoint(value);
|
|
if (endpointNamespace && endpointNamespace !== namespace) {
|
|
throw agentRunRuntimeAuthorityError("agentrun_runtime_endpoint_mismatch", `${fieldName} points at ${endpointNamespace}, not ${namespace}`);
|
|
}
|
|
}
|
|
|
|
function agentRunRuntimeNamespaceForEndpoint(value) {
|
|
const match = String(value ?? "").match(/\.((?:hwlab-[a-z0-9-]+))\.svc\.cluster\.local(?::|\/|$)/iu);
|
|
return match?.[1] ?? null;
|
|
}
|
|
|
|
function agentRunRuntimeLaneForNamespace(namespace) {
|
|
const value = String(namespace ?? "").trim().toLowerCase();
|
|
if (value === "hwlab-prod") return "prod";
|
|
if (value === "hwlab-dev") return "dev";
|
|
const versionMatch = value.match(/^hwlab-v(\d+)$/u);
|
|
if (versionMatch) return versionAgentRunRuntimeLane(versionMatch[1]);
|
|
return null;
|
|
}
|
|
|
|
function normalizeAgentRunRuntimeLane(value) {
|
|
const profile = String(value ?? "").trim().toLowerCase();
|
|
if (!profile) return null;
|
|
if (profile === "prod" || profile === "production") return "prod";
|
|
if (profile === "dev" || profile === "development" || profile === "local") return "dev";
|
|
const dotted = profile.match(/^v?0+\.(\d+)$/u);
|
|
if (dotted) return versionAgentRunRuntimeLane(dotted[1]);
|
|
const compact = profile.match(/^v?0*(\d+)$/u);
|
|
if (compact) return versionAgentRunRuntimeLane(compact[1]);
|
|
throw agentRunRuntimeAuthorityError("agentrun_runtime_lane_unsupported", `unsupported AgentRun HWLAB runtime lane: ${String(value).trim()}`);
|
|
}
|
|
|
|
function versionAgentRunRuntimeLane(value) {
|
|
const version = Number.parseInt(value, 10);
|
|
if (!Number.isFinite(version) || version <= 0) return null;
|
|
return `v${String(version).padStart(2, "0")}`;
|
|
}
|
|
|
|
function namespaceForAgentRunRuntimeLane(lane) {
|
|
if (lane === "prod") return "hwlab-prod";
|
|
if (lane === "dev") return "hwlab-dev";
|
|
if (lane) return `hwlab-${lane}`;
|
|
return null;
|
|
}
|
|
|
|
function defaultAgentRunBranchForRuntimeLane(lane) {
|
|
const match = String(lane ?? "").match(/^v(\d+)$/u);
|
|
if (!match) return null;
|
|
return `v0.${Number.parseInt(match[1], 10)}`;
|
|
}
|
|
|
|
function agentRunRuntimeAuthorityError(code, message) {
|
|
return Object.assign(new Error(message), {
|
|
code,
|
|
statusCode: 500,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
|
|
function agentRunToolCredentials(env = process.env, toolCapabilities = null) {
|
|
const namespace = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TOOL_SECRET_NAMESPACE, env.HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE, DEFAULT_RUNNER_NAMESPACE);
|
|
const githubSecretName = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_GITHUB_TOOL_SECRET_NAME, "agentrun-v01-tool-github-pr");
|
|
const githubSecretKey = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_GITHUB_TOOL_SECRET_KEY, "GH_TOKEN");
|
|
const unideskSecretName = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_UNIDESK_SSH_TOOL_SECRET_NAME, "agentrun-v01-tool-unidesk-ssh");
|
|
const credentials = [];
|
|
if (toolCapabilityAllowed(toolCapabilities, "github_pr")) credentials.push({
|
|
tool: "github",
|
|
purpose: "pull-request",
|
|
secretRef: { namespace, name: githubSecretName, keys: [githubSecretKey] },
|
|
projection: { kind: "env", envName: githubSecretKey, secretKey: githubSecretKey }
|
|
});
|
|
if (toolCapabilityAllowed(toolCapabilities, "unidesk_ssh")) credentials.push({
|
|
tool: "unidesk-ssh",
|
|
purpose: "ssh-passthrough",
|
|
secretRef: { namespace, name: unideskSecretName, keys: ["UNIDESK_SSH_CLIENT_TOKEN"] },
|
|
projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN", secretKey: "UNIDESK_SSH_CLIENT_TOKEN" }
|
|
});
|
|
return credentials;
|
|
}
|
|
|
|
async function resolveToolCapabilities({ params = {}, options = {} } = {}) {
|
|
const ownerUserId = text(params.ownerUserId);
|
|
const accessController = options.accessController;
|
|
if (!ownerUserId || typeof accessController?.codeAgentToolCapabilitiesForOwner !== "function") return null;
|
|
return await accessController.codeAgentToolCapabilitiesForOwner(ownerUserId);
|
|
}
|
|
|
|
function toolCapabilityAllowed(toolCapabilities = null, toolId) {
|
|
if (!toolCapabilities || !toolCapabilities.tools) return true;
|
|
return toolCapabilities.tools?.[toolId]?.allowed === true;
|
|
}
|
|
|
|
async function resolveReusableAgentRun({ params = {}, options = {}, env = process.env, managerUrl, fetchImpl, timeoutMs, backendProfile, traceId, traceStore }) {
|
|
const hwlabSessionId = hwlabSessionIdForParams(params, traceId);
|
|
if (!safeSessionId(hwlabSessionId) || hwlabSessionId === agentRunSessionId(traceId) || typeof options.accessController?.getAgentSession !== "function") {
|
|
return null;
|
|
}
|
|
const session = await options.accessController.getAgentSession(hwlabSessionId);
|
|
if (!canReuseAgentRunSessionForOwner(session, params, options)) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:run:reuse-owner-skipped",
|
|
message: "Stored AgentRun session is not visible to the current actor; a new runner Job will be created for this request.",
|
|
sessionId: hwlabSessionId,
|
|
waitingFor: "agentrun-run-create",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
return null;
|
|
}
|
|
const mapping = session?.session?.agentRun;
|
|
if (!isReusableAgentRunMapping(mapping, backendProfile)) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:run:reuse-skipped",
|
|
message: "No reusable AgentRun run was found for this HWLAB session; a new runner Job will be created.",
|
|
sessionId: hwlabSessionId,
|
|
waitingFor: "agentrun-run-create",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
return null;
|
|
}
|
|
let run = null;
|
|
try {
|
|
run = await agentRunDispatchJson({
|
|
fetchImpl,
|
|
managerUrl,
|
|
path: `/api/v1/runs/${encodeURIComponent(mapping.runId)}`,
|
|
method: "GET",
|
|
timeoutMs,
|
|
env,
|
|
traceStore,
|
|
traceId,
|
|
backendProfile,
|
|
runId: mapping.runId,
|
|
stage: "run-reuse-check",
|
|
terminalOnFailure: false
|
|
});
|
|
} catch (error) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:run:reuse-blocked",
|
|
message: `Stored AgentRun run ${mapping.runId} could not be read from AgentRun manager; hwlab-cloud-api will create a fresh run.`,
|
|
errorCode: error?.code ?? error?.agentRunError?.failureKind ?? "agentrun_reuse_check_failed",
|
|
runId: mapping.runId,
|
|
commandId: mapping.commandId ?? null,
|
|
runnerId: mapping.runnerId ?? null,
|
|
jobName: mapping.jobName ?? null,
|
|
namespace: mapping.namespace ?? null,
|
|
sessionId: hwlabSessionId,
|
|
agentRunSessionId: mapping.sessionId ?? null,
|
|
waitingFor: "agentrun-run-create",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
return null;
|
|
}
|
|
const blocker = agentRunReuseBlocker(run, mapping);
|
|
if (blocker) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:run:reuse-blocked",
|
|
message: `Stored AgentRun run ${mapping.runId} cannot be reused (${blocker}); hwlab-cloud-api will create a fresh run while preserving the session-scoped AgentRun PVC.`,
|
|
runId: mapping.runId,
|
|
commandId: mapping.commandId ?? null,
|
|
runnerId: mapping.runnerId ?? null,
|
|
jobName: mapping.jobName ?? null,
|
|
namespace: mapping.namespace ?? null,
|
|
sessionId: hwlabSessionId,
|
|
agentRunSessionId: mapping.sessionId ?? null,
|
|
reuseBlocker: blocker,
|
|
runStatus: run?.terminalStatus ?? run?.status ?? mapping.runStatus ?? null,
|
|
leaseExpiresAt: run?.leaseExpiresAt ?? null,
|
|
waitingFor: "agentrun-run-create",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
return null;
|
|
}
|
|
const agentRunScopedSessionId = safeSessionId(run?.sessionRef?.sessionId) || safeSessionId(mapping.sessionId) || scopedAgentRunSessionIdForParams(params, traceId);
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "backend",
|
|
status: "running",
|
|
label: "agentrun:run:reuse-enabled",
|
|
message: `Stored AgentRun run ${mapping.runId} is still claimed by an active runner; hwlab-cloud-api will append the next command without creating another runner Job.`,
|
|
runId: mapping.runId,
|
|
commandId: mapping.commandId ?? null,
|
|
runnerId: mapping.runnerId ?? null,
|
|
jobName: mapping.jobName ?? null,
|
|
namespace: mapping.namespace ?? null,
|
|
sessionId: hwlabSessionId,
|
|
agentRunSessionId: agentRunScopedSessionId,
|
|
runStatus: run?.status ?? mapping.runStatus ?? null,
|
|
claimedBy: run?.claimedBy ?? null,
|
|
leaseExpiresAt: run?.leaseExpiresAt ?? null,
|
|
waitingFor: "agentrun-command-create",
|
|
valuesPrinted: false
|
|
}, backendProfile));
|
|
return { mapping, run, sessionId: agentRunScopedSessionId };
|
|
}
|
|
|
|
function isReusableAgentRunMapping(mapping, backendProfile) {
|
|
return Boolean(
|
|
mapping &&
|
|
typeof mapping === "object" &&
|
|
mapping.adapter === ADAPTER_ID &&
|
|
mapping.reuseEligible !== false &&
|
|
mapping.runId &&
|
|
mapping.jobName &&
|
|
(!mapping.backendProfile || mapping.backendProfile === backendProfile)
|
|
);
|
|
}
|
|
|
|
function canReuseAgentRunSessionForOwner(session, params = {}, options = {}) {
|
|
if (!session) return false;
|
|
const ownerUserId = options.actor?.id ?? params.ownerUserId ?? null;
|
|
const ownerRole = options.actor?.role ?? params.ownerRole ?? null;
|
|
if (ownerRole === "admin") return true;
|
|
if (!session.ownerUserId) return true;
|
|
return Boolean(ownerUserId && session.ownerUserId === ownerUserId);
|
|
}
|
|
|
|
function agentRunReuseBlocker(run = {}, mapping = {}) {
|
|
const status = String(run?.terminalStatus ?? run?.status ?? mapping.runStatus ?? "").toLowerCase();
|
|
if (TERMINAL_RUN_STATUSES.has(status)) return `run_status_${status}`;
|
|
if (!run?.claimedBy) return "runner_not_claimed";
|
|
if (runLeaseExpired(run.leaseExpiresAt)) return "runner_reuse_window_expired";
|
|
return null;
|
|
}
|
|
|
|
function runLeaseExpired(leaseExpiresAt) {
|
|
const expiresMs = Date.parse(String(leaseExpiresAt ?? ""));
|
|
return !Number.isFinite(expiresMs) || expiresMs <= Date.now();
|
|
}
|
|
|
|
function providerCredentialSecretRef(profile, env) {
|
|
const profileEnvKey = String(profile ?? "").toUpperCase().replace(/[^A-Z0-9]+/gu, "_");
|
|
return {
|
|
profile,
|
|
secretRef: {
|
|
namespace: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE, DEFAULT_RUNNER_NAMESPACE),
|
|
name: firstNonEmpty(env[`HWLAB_CODE_AGENT_AGENTRUN_${profileEnvKey}_SECRET_NAME`], env[`HWLAB_CODE_AGENT_AGENTRUN_${String(profile ?? "").toUpperCase()}_SECRET_NAME`], env.HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_SECRET_NAME, `agentrun-v01-provider-${profile}`),
|
|
keys: ["auth.json", "config.toml"],
|
|
mountPath: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_CODEX_HOME_MOUNT, "~/.codex")
|
|
}
|
|
};
|
|
}
|
|
|
|
function agentRunProviderTrace({ base = {}, result = {}, terminalStatus = null } = {}) {
|
|
return {
|
|
transport: "agentrun-v01",
|
|
protocol: AGENTRUN_PROVIDER_TRACE_PROTOCOL,
|
|
wireApi: AGENTRUN_PROVIDER_TRACE_WIRE_API,
|
|
runnerKind: AGENTRUN_RUNNER_KIND,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
command: "agentrun.v01.command.turn",
|
|
toolName: "agentrun.v01.command.turn",
|
|
source: ADAPTER_ID,
|
|
backendProfile: base.agentRun?.backendProfile ?? null,
|
|
runId: base.agentRun?.runId ?? result?.runId ?? null,
|
|
commandId: base.agentRun?.commandId ?? result?.commandId ?? null,
|
|
runnerId: base.agentRun?.runnerId ?? result?.runnerId ?? null,
|
|
jobName: base.agentRun?.jobName ?? result?.jobName ?? null,
|
|
namespace: base.agentRun?.namespace ?? result?.namespace ?? null,
|
|
traceId: base.traceId ?? base.agentRun?.traceId ?? null,
|
|
threadId: result?.sessionRef?.threadId ?? agentRunEffectiveThreadId(base),
|
|
turnId: result?.turnId ?? null,
|
|
terminalStatus: terminalStatus ?? result?.terminalStatus ?? null,
|
|
failureKind: result?.failureKind ?? null,
|
|
failureMessage: result?.failureMessage ?? result?.blocker?.message ?? null,
|
|
willRetry: result?.willRetry === true ? true : undefined,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunFailureAttribution({ code, message, canceled = false } = {}) {
|
|
if (canceled) {
|
|
return {
|
|
category: "canceled",
|
|
retryable: true,
|
|
userMessage: "AgentRun 请求已取消。",
|
|
summary: message || "AgentRun command was canceled."
|
|
};
|
|
}
|
|
if (code === "provider-invalid-tool-call" || /invalid function arguments json string|invalid_prompt|tool_call_id/iu.test(String(message ?? ""))) {
|
|
return {
|
|
category: "provider_invalid_tool_call",
|
|
retryable: true,
|
|
userMessage: "AgentRun/provider 返回了无效 tool-call arguments JSON;这不是 HWPOD 或 Cloud API 端点失败,请查看 providerTrace.failureKind 和 runnerTrace 定位上游 tool_call_id。",
|
|
summary: message || "AgentRun/provider returned invalid tool-call arguments JSON."
|
|
};
|
|
}
|
|
if (code === "thread-resume-failed") {
|
|
return {
|
|
category: "thread_resume_failed",
|
|
retryable: true,
|
|
userMessage: "AgentRun 复用的 thread 已失效;当前 turn 会按失败终止,Workbench 会清理 stale continuation 指针,下一轮自动从新 session/thread 启动。",
|
|
summary: message || "AgentRun thread/resume failed for an existing thread."
|
|
};
|
|
}
|
|
return {
|
|
category: "agentrun_failed",
|
|
retryable: true,
|
|
userMessage: "AgentRun 请求失败;请查看 runnerTrace、providerTrace 和 agentRun 字段定位 runId/commandId/jobName。",
|
|
summary: message || "AgentRun command failed."
|
|
};
|
|
}
|
|
|
|
function agentRunLongLivedSessionGate(base = {}) {
|
|
const session = agentRunSessionSummary(base, "idle");
|
|
return {
|
|
status: "pass",
|
|
pass: true,
|
|
requiredCapability: AGENTRUN_CAPABILITY_LEVEL,
|
|
currentCapability: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
sessionId: session.sessionId ?? null,
|
|
sessionStatus: session.status ?? null,
|
|
runnerKind: AGENTRUN_RUNNER_KIND,
|
|
provider: providerForBackendProfile(base.agentRun?.backendProfile ?? "deepseek"),
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
adapter: ADAPTER_ID,
|
|
delegatedToAgentRun: true,
|
|
codexStdio: false,
|
|
blockers: [],
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunToolCalls(result = {}, status = "completed", replyTextOverride = null) {
|
|
const replyText = messageAuthorityTextValue(replyTextOverride) || agentRunReplyText(result);
|
|
return [{
|
|
name: "agentrun.v01.command.turn",
|
|
status,
|
|
runId: result?.runId ?? null,
|
|
commandId: result?.commandId ?? null,
|
|
outputSummary: replyText ? `assistantChars=${replyText.length}` : null,
|
|
valuesPrinted: false
|
|
}];
|
|
}
|
|
|
|
function agentRunReplyText(result = {}) {
|
|
for (const value of [result?.reply, result?.assistantMessage, result?.finalResponse, result?.content, result?.text, result?.message]) {
|
|
const text = messageAuthorityTextValue(value);
|
|
if (text) return text;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent = true }) {
|
|
const terminalStatus = String(result?.terminalStatus ?? "");
|
|
const normalizedTerminalStatus = terminalStatus.trim().toLowerCase().replace(/_/gu, "-");
|
|
const retryableInterruption = normalizedTerminalStatus !== "completed" && isRetryableProviderInterruptionPayload(result);
|
|
const terminal = !retryableInterruption && ["completed", "failed", "blocked", "cancelled", "canceled"].includes(normalizedTerminalStatus);
|
|
if (!terminal) {
|
|
if (retryableInterruption && appendResultEvent) {
|
|
traceStore.append(traceId, agentRunProviderRetryEvent({ base, payload: result, status: "retrying", label: "agentrun:provider-retry:provider-stream-disconnected" }), base.agentRun);
|
|
}
|
|
const runningMapping = retryableInterruption ? {
|
|
...base.agentRun,
|
|
status: "retrying",
|
|
commandState: "retrying",
|
|
failureKind: normalizedFailureKind(result?.failureKind),
|
|
willRetry: true
|
|
} : base.agentRun;
|
|
const running = decorateAgentRunRunningResult({ base, mapping: runningMapping, traceStore, traceId });
|
|
if (!retryableInterruption) return running;
|
|
return {
|
|
...running,
|
|
status: "retrying",
|
|
providerTrace: agentRunProviderTrace({ base, result, terminalStatus }),
|
|
transient: { failureKind: normalizedFailureKind(result?.failureKind), willRetry: true, valuesRedacted: true },
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
const now = nowIso();
|
|
const runnerTrace = traceStore.snapshot(traceId, agentRunTraceMeta({}, {}));
|
|
const terminalEventCreatedAt = agentRunResultTraceCreatedAt(runnerTrace, now);
|
|
const providerTrace = agentRunProviderTrace({ base, result, terminalStatus });
|
|
const replyText = agentRunReplyText(result);
|
|
if (normalizedTerminalStatus === "completed") {
|
|
const traceReplyText = agentRunTraceTerminalAssistantText(runnerTrace);
|
|
const completedReplyText = replyText || traceReplyText;
|
|
const finalResponse = completedReplyText ? agentRunCompletedFinalResponse({ base, result, traceId, now, replyText: completedReplyText }) : null;
|
|
const traceSummary = agentRunCompletedTraceSummary({ base, runnerTrace, finalResponse, traceId });
|
|
if (appendResultEvent) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "result",
|
|
status: "completed",
|
|
label: "agentrun:result:completed",
|
|
createdAt: terminalEventCreatedAt,
|
|
message: "AgentRun result is ready for HWLAB short-connection polling.",
|
|
runId: base.agentRun.runId,
|
|
commandId: base.agentRun.commandId,
|
|
terminal: true,
|
|
valuesPrinted: false
|
|
}, base.agentRun));
|
|
}
|
|
return {
|
|
...base,
|
|
status: "completed",
|
|
threadId: agentRunEffectiveThreadId(base),
|
|
updatedAt: now,
|
|
workspace: base.workspace ?? "/home/agentrun/workspace",
|
|
sandbox: base.sandbox ?? "danger-full-access",
|
|
session: agentRunSessionSummary(base, "idle"),
|
|
sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true),
|
|
runner: agentRunRunnerSummary(base.agentRun),
|
|
runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})),
|
|
toolCalls: agentRunToolCalls(result, "completed", completedReplyText),
|
|
skills: { status: "delegated", provider: ADAPTER_ID, count: 0, items: [], valuesPrinted: false },
|
|
longLivedSessionGate: agentRunLongLivedSessionGate(base),
|
|
providerTrace,
|
|
finalResponse: finalResponse ?? null,
|
|
traceSummary,
|
|
reply: finalResponse ? {
|
|
messageId: finalResponse.messageId,
|
|
role: "assistant",
|
|
content: finalResponse.text,
|
|
createdAt: finalResponse.createdAt
|
|
} : null,
|
|
usage: null,
|
|
agentRun: { ...base.agentRun, terminalStatus, completed: true, reuseEligible: true, providerTrace, valuesPrinted: false },
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
const canceled = terminalStatus === "cancelled";
|
|
const code = canceled ? "agentrun_canceled" : result?.failureKind ?? (terminalStatus === "blocked" ? "agentrun_blocked" : "agentrun_failed");
|
|
const message = result?.failureMessage ?? result?.blocker?.message ?? (canceled ? "AgentRun command was canceled" : "AgentRun command failed");
|
|
const attribution = agentRunFailureAttribution({ code, message, canceled });
|
|
const finalResponse = agentRunTerminalFailureFinalResponse({
|
|
base,
|
|
traceId,
|
|
now,
|
|
status: canceled ? "canceled" : "failed",
|
|
text: firstNonEmpty(message, attribution.summary, attribution.userMessage)
|
|
});
|
|
const traceSummary = agentRunTerminalFailureTraceSummary({ base, runnerTrace, finalResponse, traceId, terminalStatus: canceled ? "canceled" : terminalStatus || "failed" });
|
|
if (appendResultEvent) {
|
|
traceStore.append(traceId, agentRunTraceEvent({
|
|
type: "result",
|
|
status: canceled ? "canceled" : "failed",
|
|
label: `agentrun:result:${canceled ? "canceled" : terminalStatus || "failed"}`,
|
|
createdAt: terminalEventCreatedAt,
|
|
errorCode: code,
|
|
message: attribution.summary,
|
|
runId: base.agentRun.runId,
|
|
commandId: base.agentRun.commandId,
|
|
terminal: true,
|
|
valuesPrinted: false
|
|
}, base.agentRun));
|
|
}
|
|
const partialContext = partialAgentRunContext(runnerTrace);
|
|
const freshSessionRequired = !canceled && agentRunFailureRequiresFreshSession(code, message);
|
|
const failedSessionStatus = freshSessionRequired ? "thread-resume-failed" : "failed";
|
|
const resumableAfterFailure = !canceled && !freshSessionRequired && Boolean(agentRunEffectiveThreadId(base));
|
|
return {
|
|
...base,
|
|
status: canceled ? "canceled" : "failed",
|
|
threadId: agentRunEffectiveThreadId(base),
|
|
canceled,
|
|
updatedAt: now,
|
|
session: agentRunSessionSummary(base, canceled ? "canceled" : failedSessionStatus),
|
|
sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true, { status: canceled ? undefined : freshSessionRequired ? "thread-resume-failed" : resumableAfterFailure ? "failed-resumable" : "failed-requires-new-session" }),
|
|
runner: agentRunRunnerSummary(base.agentRun),
|
|
runnerTrace: partialContext ? { ...runnerTrace, partialContext } : runnerTrace,
|
|
toolCalls: agentRunToolCalls(result, canceled ? "canceled" : "failed"),
|
|
skills: { status: "delegated", provider: ADAPTER_ID, count: 0, items: [], valuesPrinted: false },
|
|
providerTrace,
|
|
finalResponse,
|
|
traceSummary,
|
|
error: {
|
|
code,
|
|
layer: "agentrun",
|
|
category: attribution.category,
|
|
retryable: attribution.retryable,
|
|
message,
|
|
userMessage: attribution.userMessage,
|
|
traceId,
|
|
route: "/v1/agent/chat",
|
|
toolName: "agentrun.manual-dispatch"
|
|
},
|
|
blocker: canceled ? null : {
|
|
code,
|
|
layer: "agentrun",
|
|
category: attribution.category,
|
|
retryable: attribution.retryable,
|
|
summary: attribution.summary,
|
|
traceId,
|
|
route: "/v1/agent/chat",
|
|
toolName: "agentrun.manual-dispatch"
|
|
},
|
|
agentRun: { ...base.agentRun, terminalStatus, completed: false, reuseEligible: resumableAfterFailure, providerTrace, valuesPrinted: false },
|
|
reuseEligible: resumableAfterFailure,
|
|
...(partialContext ? { partialContext } : {}),
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunCompletedFinalResponse({ base, result, traceId, now, replyText = null }) {
|
|
const textValue = messageAuthorityTextValue(replyText) || agentRunReplyText(result);
|
|
return {
|
|
text: textValue,
|
|
textChars: textValue.length,
|
|
role: "assistant",
|
|
status: "completed",
|
|
traceId,
|
|
messageId: base.messageId ?? base.reply?.messageId ?? `msg_${traceId.slice(4)}`,
|
|
createdAt: base.reply?.createdAt ?? base.createdAt ?? now,
|
|
updatedAt: now,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunTraceTerminalAssistantText(runnerTrace = {}) {
|
|
const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : [];
|
|
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
const event = events[index];
|
|
if (!event || typeof event !== "object") continue;
|
|
if (!isAgentRunAssistantTraceEvent(event)) continue;
|
|
if (!(event.terminal === true || event.final === true || event.replyAuthority === true)) continue;
|
|
const textValue = messageAuthorityTextValue(event.finalResponse ?? event.text ?? event.content ?? event.message ?? event.summary ?? event.payload?.text ?? event.payload?.content ?? event.payload?.message);
|
|
if (textValue) return textValue;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function isAgentRunAssistantTraceEvent(event = {}) {
|
|
const type = String(event.type ?? event.eventType ?? "").trim().toLowerCase();
|
|
if (type === "assistant" || type === "assistant_message") return true;
|
|
return /assistant:message|assistant_message/u.test(String(event.label ?? "").toLowerCase());
|
|
}
|
|
|
|
function agentRunTerminalFailureFinalResponse({ base, traceId, now, status, text }) {
|
|
const textValue = String(text ?? "").trim();
|
|
return {
|
|
text: textValue || "AgentRun command failed.",
|
|
textChars: (textValue || "AgentRun command failed.").length,
|
|
role: "assistant",
|
|
status,
|
|
traceId,
|
|
messageId: base.messageId ?? base.reply?.messageId ?? `msg_${traceId.slice(4)}`,
|
|
createdAt: base.reply?.createdAt ?? base.createdAt ?? now,
|
|
updatedAt: now,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunTerminalFailureTraceSummary({ base, runnerTrace, finalResponse, traceId, terminalStatus }) {
|
|
const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : [];
|
|
const lastSeq = Math.max(0, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite));
|
|
return {
|
|
traceId,
|
|
source: "agentrun-command-result",
|
|
sourceEventCount: Number(runnerTrace?.eventCount ?? events.length ?? 0),
|
|
terminalStatus,
|
|
finalAssistantRow: {
|
|
role: finalResponse.role,
|
|
status: finalResponse.status,
|
|
textChars: finalResponse.textChars,
|
|
textPreview: finalResponse.text.slice(0, 240),
|
|
messageId: finalResponse.messageId,
|
|
valuesPrinted: false
|
|
},
|
|
agentRun: {
|
|
runId: base.agentRun?.runId ?? null,
|
|
commandId: base.agentRun?.commandId ?? null,
|
|
lastSeq,
|
|
valuesPrinted: false
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunCompletedTraceSummary({ base, runnerTrace, finalResponse, traceId }) {
|
|
const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : [];
|
|
const lastSeq = Math.max(0, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite));
|
|
return {
|
|
traceId,
|
|
source: "agentrun-command-result",
|
|
sourceEventCount: Number(runnerTrace?.eventCount ?? events.length ?? 0),
|
|
terminalStatus: "completed",
|
|
finalAssistantRow: finalResponse ? {
|
|
role: finalResponse.role,
|
|
status: finalResponse.status,
|
|
textChars: finalResponse.textChars,
|
|
textPreview: finalResponse.text.slice(0, 240),
|
|
messageId: finalResponse.messageId,
|
|
valuesPrinted: false
|
|
} : null,
|
|
agentRun: {
|
|
runId: base.agentRun?.runId ?? null,
|
|
commandId: base.agentRun?.commandId ?? null,
|
|
lastSeq,
|
|
valuesPrinted: false
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function partialAgentRunContext(runnerTrace = {}) {
|
|
const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : [];
|
|
const assistantMessages = events
|
|
.filter((event) => event?.type === "assistant" && String(event.text ?? event.message ?? "").trim())
|
|
.map((event) => String(event.text ?? event.message).trim())
|
|
.slice(-4);
|
|
const toolEvidence = events
|
|
.filter((event) => event?.type === "tool_call" && String(event.status ?? "") === "completed")
|
|
.map((event) => [event.toolName ?? event.label ?? "tool", event.command ?? event.outputSummary ?? event.stdoutSummary ?? event.message ?? ""].filter(Boolean).join(": "))
|
|
.filter(Boolean)
|
|
.slice(-6);
|
|
if (assistantMessages.length === 0 && toolEvidence.length === 0) return null;
|
|
return {
|
|
status: "partial-before-terminal",
|
|
summary: "AgentRun terminal 前已有可延续的 assistant/tool 上下文;后续同 conversation/session/thread 轮次必须能继续使用。",
|
|
assistantMessages,
|
|
toolEvidence,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunResultTraceCreatedAt(runnerTrace = {}, fallback) {
|
|
const last = String(runnerTrace?.lastEvent?.createdAt ?? runnerTrace?.updatedAt ?? "");
|
|
return Number.isFinite(Date.parse(last)) ? last : fallback;
|
|
}
|
|
|
|
function decorateAgentRunRunningResult({ base, mapping, traceStore, traceId }) {
|
|
return {
|
|
...base,
|
|
status: "running",
|
|
threadId: agentRunEffectiveThreadId({ agentRun: mapping }),
|
|
accepted: true,
|
|
shortConnection: true,
|
|
updatedAt: nowIso(),
|
|
session: agentRunSessionSummary({ ...base, agentRun: mapping }, "running"),
|
|
sessionReuse: agentRunSessionReuseSummary({ ...base, agentRun: mapping }, mapping.reused === true),
|
|
runner: agentRunRunnerSummary(mapping),
|
|
runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})),
|
|
agentRun: { ...mapping, adapter: ADAPTER_ID, valuesPrinted: false },
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function appendAgentRunEventsToTrace(traceStore, traceId, events, mapping = {}) {
|
|
for (const event of events) {
|
|
if (isForeignAgentRunCommandEvent(event, mapping)) continue;
|
|
const normalized = mapAgentRunEvent(event, mapping);
|
|
if (normalized) traceStore.append(traceId, normalized, agentRunTraceMeta({}, {}));
|
|
}
|
|
}
|
|
|
|
async function fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) {
|
|
const runId = requiredString(mapping.runId, "runId");
|
|
const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : "";
|
|
const replayWindow = agentRunTraceReplayWindow(mapping);
|
|
const afterSeqOverride = Number(mapping.afterSeqOverride);
|
|
const afterSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? Math.floor(afterSeqOverride) : replayWindow.afterSeq;
|
|
const endSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? 0 : replayWindow.endSeq;
|
|
const limit = parsePositiveInteger(mapping.eventsPageLimit ?? env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT, 500);
|
|
const path = `/api/v1/runs/${encodeURIComponent(runId)}/events?afterSeq=${encodeURIComponent(String(afterSeq))}&limit=${encodeURIComponent(String(limit))}`;
|
|
const response = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env });
|
|
const rawEvents = Array.isArray(response?.items) ? response.items : [];
|
|
const events = rawEvents.filter((event) => agentRunEventBelongsToTrace(event, { currentCommandId, afterSeq, endSeq }));
|
|
const maxSeq = Math.max(afterSeq, ...rawEvents.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite));
|
|
const traceLastSeq = Math.max(afterSeq, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite));
|
|
void emitCodeAgentOtelSpan("projection_sync", mapping.traceId ?? mapping.hwlabTraceId ?? mapping.businessTraceId, env, {
|
|
attributes: {
|
|
runId,
|
|
commandId: currentCommandId || null,
|
|
afterSeq,
|
|
endSeq,
|
|
limit,
|
|
rawEventCount: rawEvents.length,
|
|
eventCount: events.length,
|
|
commandFiltered: Boolean(currentCommandId),
|
|
maxSeq,
|
|
traceLastSeq,
|
|
terminalFromRawEvents: agentRunTerminalStatusFromEvents(rawEvents),
|
|
terminalFromEvents: agentRunTerminalStatusFromEvents(events)
|
|
}
|
|
});
|
|
return {
|
|
events,
|
|
afterSeq,
|
|
endSeq,
|
|
rawEventCount: rawEvents.length,
|
|
commandFiltered: Boolean(currentCommandId),
|
|
maxSeq,
|
|
traceLastSeq
|
|
};
|
|
}
|
|
|
|
async function measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) {
|
|
const startedAt = nowMs();
|
|
try {
|
|
const response = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping });
|
|
performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: "ok", durationMs: nowMs() - startedAt });
|
|
return response;
|
|
} catch (error) {
|
|
performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: agentRunMetricStatusForError(error), durationMs: nowMs() - startedAt });
|
|
performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_events_fetch_failed" });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped = {}, eventsResponse = null }) {
|
|
const startedAt = nowMs();
|
|
const path = `/api/v1/runs/${encodeURIComponent(mapped.agentRun.runId)}/commands/${encodeURIComponent(mapped.agentRun.commandId)}/result`;
|
|
try {
|
|
const result = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env });
|
|
const eventCount = agentRunResultEventCount(result, eventsResponse);
|
|
performanceStore?.recordAgentRunResult?.({
|
|
durationMs: nowMs() - startedAt,
|
|
eventCount,
|
|
eventsScanned: eventCount,
|
|
pagesScanned: estimateAgentRunResultPages(eventCount),
|
|
status: result?.terminalStatus ?? result?.status ?? "ok",
|
|
budget: timeoutMs
|
|
});
|
|
return result;
|
|
} catch (error) {
|
|
const status = agentRunMetricStatusForError(error);
|
|
const eventCount = agentRunResultEventCount(null, eventsResponse);
|
|
performanceStore?.recordAgentRunResult?.({
|
|
durationMs: nowMs() - startedAt,
|
|
eventCount,
|
|
eventsScanned: eventCount,
|
|
pagesScanned: estimateAgentRunResultPages(eventCount),
|
|
status,
|
|
budget: timeoutMs,
|
|
timedOut: status === "timeout"
|
|
});
|
|
performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_result_failed" });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse = null, previousLastSeq = 0, nextLastSeq = 0) {
|
|
if (!performanceStore || !eventsResponse) return;
|
|
const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0));
|
|
if (cursorAdvance > 0) {
|
|
performanceStore.recordWorkbenchProjectionCursorAdvance?.({
|
|
status: "advanced",
|
|
reason: eventsResponse.commandFiltered ? "command_filtered_events" : "run_events",
|
|
projectionStatus: "projecting",
|
|
source: "agentrun_v01",
|
|
count: cursorAdvance
|
|
});
|
|
}
|
|
const counts = new Map();
|
|
for (const event of eventsResponse.events ?? []) {
|
|
const kind = String(event?.type ?? event?.payload?.phase ?? "unknown").trim() || "unknown";
|
|
counts.set(kind, (counts.get(kind) ?? 0) + 1);
|
|
}
|
|
if (counts.size === 0 && Number(eventsResponse.rawEventCount ?? 0) > 0) counts.set("filtered", Number(eventsResponse.rawEventCount));
|
|
for (const [eventKind, count] of counts.entries()) {
|
|
performanceStore.recordWorkbenchProjectorEvents?.({ eventKind, status: "ok", count });
|
|
}
|
|
}
|
|
|
|
function agentRunResultEventCount(result = null, eventsResponse = null) {
|
|
const candidates = [
|
|
result?.traceSummary?.sourceEventCount,
|
|
result?.traceSummary?.eventCount,
|
|
result?.sourceEventCount,
|
|
result?.eventCount,
|
|
result?.providerTrace?.eventCount,
|
|
eventsResponse?.maxSeq,
|
|
eventsResponse?.rawEventCount,
|
|
eventsResponse?.events?.length
|
|
];
|
|
for (const value of candidates) {
|
|
const count = Number(value);
|
|
if (Number.isFinite(count) && count >= 0) return Math.floor(count);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
function estimateAgentRunResultPages(eventCount) {
|
|
const count = Number(eventCount);
|
|
if (!Number.isFinite(count) || count <= 0) return 0;
|
|
return Math.max(1, Math.ceil(count / 500));
|
|
}
|
|
|
|
function agentRunMetricStatusForError(error) {
|
|
return error?.code === "agentrun_timeout" || error?.name === "AbortError" || Number(error?.statusCode) === 504 ? "timeout" : "error";
|
|
}
|
|
|
|
function agentRunTraceCursorSeq(eventsResponse = {}, previousLastSeq = 0) {
|
|
const afterSeq = Number(eventsResponse.afterSeq ?? 0);
|
|
const endSeq = Number(eventsResponse.endSeq ?? 0);
|
|
const traceLastSeq = Number(eventsResponse.traceLastSeq ?? 0);
|
|
if (Number.isFinite(traceLastSeq) && traceLastSeq > afterSeq) return Math.floor(traceLastSeq);
|
|
if (Number.isFinite(endSeq) && endSeq > 0) return Math.floor(endSeq);
|
|
if (eventsResponse.commandFiltered === true) return Math.max(Number(previousLastSeq ?? 0), afterSeq);
|
|
return Math.max(Number(previousLastSeq ?? 0), Number(eventsResponse.maxSeq ?? 0));
|
|
}
|
|
|
|
function agentRunTraceReplayWindow(mapping = {}) {
|
|
const eventStartSeq = Number(mapping.eventStartSeq ?? mapping.commandStartSeq ?? mapping.startSeq ?? 0);
|
|
const summary = mapping.traceSummary && typeof mapping.traceSummary === "object" ? mapping.traceSummary : null;
|
|
const summaryAgentRun = summary?.agentRun && typeof summary.agentRun === "object" ? summary.agentRun : null;
|
|
const summaryLastSeq = Number(summaryAgentRun?.lastSeq ?? summary?.lastSeq ?? 0);
|
|
const currentLastSeq = Number(mapping.lastSeq ?? 0);
|
|
const endSeq = Number.isFinite(summaryLastSeq) && summaryLastSeq > 0 ? Math.floor(summaryLastSeq) : 0;
|
|
if (Number.isFinite(eventStartSeq) && eventStartSeq > 0) return { afterSeq: Math.max(0, Math.floor(eventStartSeq) - 1), endSeq };
|
|
if (endSeq > 0) return { afterSeq: Math.max(0, endSeq - 500), endSeq };
|
|
if (Number.isFinite(currentLastSeq) && currentLastSeq > 0) return { afterSeq: Math.floor(currentLastSeq), endSeq: 0 };
|
|
return { afterSeq: 0, endSeq: 0 };
|
|
}
|
|
|
|
function agentRunEventBelongsToTrace(event, { currentCommandId = "", afterSeq = 0, endSeq = 0 } = {}) {
|
|
const eventCommandId = agentRunEventCommandId(event);
|
|
const seq = Number(event?.seq ?? 0);
|
|
if (currentCommandId && eventCommandId && eventCommandId !== currentCommandId) return false;
|
|
if (endSeq > 0 && Number.isFinite(seq)) return seq > afterSeq && seq <= endSeq;
|
|
return true;
|
|
}
|
|
|
|
function agentRunEventCommandId(event) {
|
|
const payload = event?.payload && typeof event.payload === "object" ? event.payload : {};
|
|
if (typeof event?.commandId === "string") return event.commandId;
|
|
return typeof payload.commandId === "string" ? payload.commandId : "";
|
|
}
|
|
|
|
function isForeignAgentRunCommandEvent(event, mapping = {}) {
|
|
const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : "";
|
|
if (!currentCommandId) return false;
|
|
const eventCommandId = agentRunEventCommandId(event);
|
|
return Boolean(eventCommandId && eventCommandId !== currentCommandId);
|
|
}
|
|
|
|
function mapAgentRunEvent(event, mapping = {}) {
|
|
if (!event || typeof event !== "object") return null;
|
|
const payload = event.payload && typeof event.payload === "object" ? event.payload : {};
|
|
const base = {
|
|
createdAt: event.createdAt ?? null,
|
|
source: "agentrun",
|
|
sourceSeq: event.seq ?? null,
|
|
backend: backendForBackendProfile(mapping.backendProfile ?? payload.backendProfile ?? payload.providerProfile ?? "deepseek"),
|
|
runId: event.runId ?? mapping.runId ?? null,
|
|
commandId: payload.commandId ?? mapping.commandId ?? null,
|
|
attemptId: payload.attemptId ?? mapping.attemptId ?? null,
|
|
runnerId: payload.runnerId ?? mapping.runnerId ?? null,
|
|
jobName: payload.jobName ?? mapping.jobName ?? null,
|
|
namespace: payload.namespace ?? mapping.namespace ?? null,
|
|
valuesPrinted: false
|
|
};
|
|
if (event.type === "backend_status") {
|
|
const phase = String(payload.phase ?? "status");
|
|
return {
|
|
...base,
|
|
type: "backend",
|
|
eventType: "backend",
|
|
status: "running",
|
|
label: `agentrun:backend:${phase}`,
|
|
message: textPayload(payload, phase),
|
|
waitingFor: waitingForForPhase(phase),
|
|
details: agentRunBackendStatusDetails(phase, payload)
|
|
};
|
|
}
|
|
if (event.type === "assistant_message") {
|
|
const terminal = payload.replyAuthority === true || payload.final === true;
|
|
return {
|
|
...base,
|
|
type: "assistant",
|
|
eventType: "assistant",
|
|
status: terminal ? "completed" : "running",
|
|
label: "agentrun:assistant:message",
|
|
message: textPayload(payload, "assistant message"),
|
|
text: textPayload(payload, ""),
|
|
itemId: payload.itemId ?? null,
|
|
messageIndex: payload.messageIndex ?? null,
|
|
messageCount: payload.messageCount ?? null,
|
|
replyAuthority: payload.replyAuthority === true,
|
|
final: payload.final === true,
|
|
terminal
|
|
};
|
|
}
|
|
if (event.type === "tool_call") {
|
|
const commandExecution = agentRunCommandExecutionEvent(base, payload);
|
|
if (commandExecution) return commandExecution;
|
|
const preview = agentRunToolCallPreviewSummary(payload);
|
|
const toolName = firstNonEmpty(
|
|
payload.toolName,
|
|
payload.name,
|
|
payload.item?.type,
|
|
payload.type,
|
|
preview.toolName,
|
|
preview.name,
|
|
preview.type,
|
|
isProtocolToolLifecycleMethod(payload.method) ? null : payload.method,
|
|
"tool call"
|
|
);
|
|
const command = firstNonEmpty(payload.command, payload.commandLine, payload.item?.command, payload.item?.commandLine, preview.command, preview.commandLine);
|
|
const output = firstNonEmpty(payload.outputSummary, payload.stdoutSummary, preview.outputSummary, preview.stdoutSummary, payload.summary?.text, payload.itemPreview);
|
|
return {
|
|
...base,
|
|
type: "tool_call",
|
|
eventType: "tool_call",
|
|
status: String(payload.status ?? preview.status ?? (payload.method === "item/completed" ? "completed" : "running")),
|
|
label: `agentrun:tool:${String(toolName)}`,
|
|
itemId: payload.item?.id ?? payload.itemId ?? preview.itemId ?? null,
|
|
toolName,
|
|
command,
|
|
exitCode: Number.isInteger(payload.item?.exitCode) ? payload.item.exitCode : Number.isInteger(payload.exitCode) ? payload.exitCode : undefined,
|
|
durationMs: typeof payload.item?.durationMs === "number" ? payload.item.durationMs : typeof payload.durationMs === "number" ? payload.durationMs : undefined,
|
|
stdoutSummary: output,
|
|
outputSummary: output,
|
|
message: output || command || textPayload(payload, "tool call"),
|
|
outputBytes: typeof payload.outputBytes === "number" ? payload.outputBytes : payload.summary?.outputBytes,
|
|
outputTruncated: payload.outputTruncated === true || payload.summary?.outputTruncated === true
|
|
};
|
|
}
|
|
if (event.type === "command_output") {
|
|
const stream = payload.stream === "stderr" ? "stderr" : "stdout";
|
|
return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true };
|
|
}
|
|
if (event.type === "diff") {
|
|
return { ...base, type: "diff", eventType: "status", status: "running", label: "agentrun:diff", message: textPayload(payload, "diff") };
|
|
}
|
|
if (event.type === "error") {
|
|
const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? "backend") ?? "backend";
|
|
const normalizedPayload = { ...payload, failureKind, errorCode: failureKind };
|
|
if (isRetryableProviderInterruptionPayload(normalizedPayload, { activeError: true })) return agentRunProviderRetryEvent({ base, payload: { ...normalizedPayload, willRetry: true }, status: "retrying", label: `agentrun:provider-retry:${failureKind}` });
|
|
return { ...base, type: "error", eventType: "error", status: "failed", label: `agentrun:error:${failureKind}`, errorCode: failureKind, failureKind, willRetry: payload.willRetry === true ? true : undefined, message: textPayload(payload, "AgentRun error") };
|
|
}
|
|
if (event.type === "terminal_status") {
|
|
const terminalStatus = String(payload.terminalStatus ?? event.terminalStatus ?? payload.status ?? event.status ?? "failed");
|
|
const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? event.failureKind ?? event.errorCode) ?? null;
|
|
const normalizedPayload = failureKind ? { ...payload, failureKind, errorCode: failureKind } : payload;
|
|
if (isRetryableProviderInterruptionPayload(normalizedPayload)) return agentRunProviderRetryEvent({ base, payload: normalizedPayload, status: "retrying", label: `agentrun:provider-retry:${terminalStatus}` });
|
|
return { ...base, type: "result", eventType: "terminal", status: terminalStatus === "cancelled" ? "canceled" : terminalStatus, label: `agentrun:terminal:${terminalStatus}`, errorCode: failureKind, failureKind, willRetry: payload.willRetry === true || event.willRetry === true ? true : undefined, message: textPayload({ ...event, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true };
|
|
}
|
|
return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${String(event.type ?? "unknown")}`, message: textPayload(payload, "AgentRun event") };
|
|
}
|
|
|
|
function agentRunProviderRetryEvent({ base = {}, payload = {}, status = "retrying", label = "agentrun:provider-retry" } = {}) {
|
|
const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? "provider-stream-disconnected");
|
|
const retryMessage = providerRetryMessage(textPayload(payload, "Provider stream disconnected; AgentRun will retry."), payload);
|
|
return {
|
|
...base,
|
|
type: "provider_retry",
|
|
eventType: "backend",
|
|
status,
|
|
label,
|
|
errorCode: failureKind,
|
|
failureKind,
|
|
willRetry: payload.willRetry === true,
|
|
retryable: true,
|
|
transient: true,
|
|
terminal: false,
|
|
retryAttempt: numberOrNull(payload.retryAttempt),
|
|
retryMax: numberOrNull(payload.retryMax),
|
|
retryDelayMs: numberOrNull(payload.retryDelayMs),
|
|
nextRetryAt: firstNonEmpty(payload.nextRetryAt) || null,
|
|
retryOf: payload.retryOf ?? null,
|
|
terminalStatus: payload.terminalStatus ?? null,
|
|
message: retryMessage,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function providerRetryMessage(message, payload = {}) {
|
|
const attempt = numberOrNull(payload.retryAttempt);
|
|
const max = numberOrNull(payload.retryMax);
|
|
const delayMs = numberOrNull(payload.retryDelayMs);
|
|
const nextRetryAt = firstNonEmpty(payload.nextRetryAt) || null;
|
|
const parts = [];
|
|
if (attempt !== null && max !== null) parts.push(`retry ${attempt}/${max}`);
|
|
else if (attempt !== null) parts.push(`retry attempt ${attempt}`);
|
|
else if (max !== null) parts.push(`retry max ${max}`);
|
|
if (delayMs !== null) parts.push(`after ${delayMs}ms`);
|
|
if (nextRetryAt) parts.push(`next=${nextRetryAt}`);
|
|
if (parts.length === 0) return message;
|
|
const text = String(message || "Provider stream disconnected; AgentRun will retry.").trim();
|
|
if (attempt !== null && max !== null && text.includes(`${attempt}/${max}`)) return text;
|
|
return text.replace(/[.。]\s*$/u, "") + `; ${parts.join(" ")}.`;
|
|
}
|
|
|
|
function isRetryableProviderInterruptionPayload(payload = {}, options = {}) {
|
|
const kind = normalizedFailureKind(payload?.failureKind ?? payload?.errorCode ?? payload?.code);
|
|
if (kind === "provider-stream-disconnected") return payload?.willRetry === true;
|
|
if (kind === "provider-unavailable") return payload?.willRetry === true || (options.activeError === true && payload?.terminal !== true && payload?.final !== true);
|
|
if (isRetryableAgentRunTransportFailureKind(kind)) return payload?.willRetry === true;
|
|
return false;
|
|
}
|
|
|
|
function normalizedFailureKind(value) {
|
|
const normalized = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-") || null;
|
|
if (normalized === "agentrun-unreachable") return "agentrun-connect-failed";
|
|
return normalized;
|
|
}
|
|
|
|
function numberOrNull(value) {
|
|
const number = Number(value);
|
|
return Number.isFinite(number) ? number : null;
|
|
}
|
|
|
|
function agentRunTraceEvent(event = {}, backendSource = {}) {
|
|
const source = backendSource && typeof backendSource === "object" ? backendSource : { backendProfile: backendSource };
|
|
const backendProfile = firstNonEmpty(event.backendProfile, source.backendProfile, source.providerProfile, source.agentRun?.backendProfile);
|
|
return {
|
|
...event,
|
|
eventType: event.eventType ?? agentRunManualEventType(event),
|
|
backend: event.backend ?? (backendProfile ? backendForBackendProfile(backendProfile) : null),
|
|
backendProfile: event.backendProfile ?? backendProfile ?? null,
|
|
valuesPrinted: event.valuesPrinted ?? false
|
|
};
|
|
}
|
|
|
|
function agentRunManualEventType(event = {}) {
|
|
const type = String(event.type ?? "").trim().toLowerCase();
|
|
if (type === "request") return "request";
|
|
if (type === "cancel" || type === "result") return "terminal";
|
|
if (type === "assistant") return "assistant";
|
|
if (type === "tool" || type === "tool_call") return "tool_call";
|
|
if (type === "error") return "error";
|
|
if (type === "backend") return "backend";
|
|
const raw = String(`${event.label ?? ""} ${event.status ?? ""}`).toLowerCase();
|
|
if (/request|accepted/u.test(raw)) return "request";
|
|
if (/cancel|terminal|complete|final/u.test(raw)) return "terminal";
|
|
if (/assistant|message|reply/u.test(raw)) return "assistant";
|
|
if (/error|fail|timeout/u.test(raw)) return "error";
|
|
if (/backend|runner|agentrun/u.test(raw)) return "backend";
|
|
return "status";
|
|
}
|
|
|
|
function isProtocolToolLifecycleMethod(value) {
|
|
return /^item\/(started|completed)$/u.test(String(value ?? ""));
|
|
}
|
|
|
|
function agentRunToolCallPreviewSummary(payload = {}) {
|
|
const text = firstNonEmpty(payload.itemPreview, payload.summary?.text);
|
|
return {
|
|
itemId: jsonPreviewStringField(text, "itemId"),
|
|
type: jsonPreviewStringField(text, "type"),
|
|
toolName: jsonPreviewStringField(text, "toolName"),
|
|
name: jsonPreviewStringField(text, "name"),
|
|
status: jsonPreviewStringField(text, "status"),
|
|
command: jsonPreviewStringField(text, "command"),
|
|
commandLine: jsonPreviewStringField(text, "commandLine"),
|
|
outputSummary: jsonPreviewStringField(text, "outputSummary"),
|
|
stdoutSummary: jsonPreviewStringField(text, "stdoutSummary")
|
|
};
|
|
}
|
|
|
|
function jsonPreviewStringField(text, key) {
|
|
const body = String(text ?? "");
|
|
if (!body || !/^[A-Za-z][A-Za-z0-9_]*$/u.test(key)) return null;
|
|
const match = body.match(new RegExp(`"${key}"\\s*:\\s*"((?:\\\\.|[^"\\\\])*)"`, "u"));
|
|
if (!match) return null;
|
|
try {
|
|
const parsed = JSON.parse(`"${match[1]}"`);
|
|
return typeof parsed === "string" && parsed.trim() ? parsed.trim() : null;
|
|
} catch {
|
|
return match[1]?.trim() || null;
|
|
}
|
|
}
|
|
|
|
function agentRunBackendStatusDetails(phase, payload = {}) {
|
|
if (phase === "initial-prompt-assembly") {
|
|
return compactObject({
|
|
initialPromptInjected: payload.initialPromptInjected === true,
|
|
reason: firstNonEmpty(payload.reason),
|
|
initialPrompt: agentRunResourceSummary(payload.initialPrompt),
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
if (phase === "resource-bundle-materialized") {
|
|
return compactObject({
|
|
kind: firstNonEmpty(payload.kind),
|
|
commitId: firstNonEmpty(payload.commitId),
|
|
bundles: agentRunResourceSummary(payload.bundles),
|
|
tools: agentRunResourceSummary(payload.tools),
|
|
promptRefs: agentRunResourceSummary(payload.promptRefs),
|
|
skillDirs: agentRunResourceSummary(payload.skillDirs),
|
|
initialPrompt: agentRunResourceSummary(payload.initialPrompt),
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function agentRunResourceSummary(value) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
return compactObject({
|
|
available: typeof value.available === "boolean" ? value.available : undefined,
|
|
count: Number.isInteger(value.count) ? value.count : undefined,
|
|
bytes: Number.isInteger(value.bytes) ? value.bytes : undefined,
|
|
sha256: firstNonEmpty(value.sha256),
|
|
promptRefCount: Number.isInteger(value.promptRefCount) ? value.promptRefCount : undefined,
|
|
skillCount: Number.isInteger(value.skillCount) ? value.skillCount : undefined,
|
|
toolCount: Number.isInteger(value.toolCount) ? value.toolCount : undefined,
|
|
names: Array.isArray(value.names) ? value.names.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined,
|
|
targetPaths: Array.isArray(value.targetPaths) ? value.targetPaths.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined,
|
|
skillNames: Array.isArray(value.skillNames) ? value.skillNames.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined,
|
|
requiredMissing: Array.isArray(value.requiredMissing) ? value.requiredMissing.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
|
|
function compactObject(value) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
const result = {};
|
|
for (const [key, item] of Object.entries(value)) {
|
|
if (item === undefined || item === null) continue;
|
|
if (Array.isArray(item) && item.length === 0) continue;
|
|
if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue;
|
|
result[key] = item;
|
|
}
|
|
return Object.keys(result).length > 0 ? result : undefined;
|
|
}
|
|
|
|
function agentRunCommandExecutionEvent(base, payload = {}) {
|
|
const item = payload.item && typeof payload.item === "object" ? payload.item : null;
|
|
const payloadToolType = firstNonEmpty(payload.toolName, payload.type, payload.name);
|
|
if (item?.type !== "commandExecution" && payloadToolType !== "commandExecution") return null;
|
|
const method = String(payload.method ?? "");
|
|
const completed = method === "item/completed" || payload.status === "completed" || item?.status === "completed";
|
|
const output = firstNonEmpty(item?.aggregatedOutput, item?.stdout, item?.output, payload.outputSummary, payload.stdoutSummary, payload.summary?.text, payload.itemPreview);
|
|
return {
|
|
...base,
|
|
type: "tool_call",
|
|
status: completed ? "completed" : "started",
|
|
label: completed ? "item/commandExecution:completed" : "item/commandExecution:started",
|
|
toolName: "commandExecution",
|
|
itemId: item?.id ?? payload.itemId ?? null,
|
|
command: firstNonEmpty(item?.command, item?.commandLine, payload.command, payload.commandLine),
|
|
exitCode: Number.isInteger(item?.exitCode) ? item.exitCode : Number.isInteger(payload.exitCode) ? payload.exitCode : undefined,
|
|
durationMs: typeof item?.durationMs === "number" ? item.durationMs : typeof payload.durationMs === "number" ? payload.durationMs : undefined,
|
|
outputBytes: typeof payload.outputBytes === "number" ? payload.outputBytes : payload.summary?.outputBytes,
|
|
stdoutSummary: output,
|
|
outputSummary: output,
|
|
outputTruncated: payload.outputTruncated === true || payload.summary?.outputTruncated === true,
|
|
message: output || textPayload(payload, "commandExecution")
|
|
};
|
|
}
|
|
|
|
function nowMs() {
|
|
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
|
return Date.now();
|
|
}
|
|
|
|
function agentRunMapping({ env, managerUrl, backendProfile, run, command, runnerJob, traceId, startedAt, params = {} }) {
|
|
const threadReused = Boolean(safeOpaqueId(params.threadId));
|
|
const runnerJobCreated = Boolean(runnerJob);
|
|
return {
|
|
adapter: ADAPTER_ID,
|
|
managerUrl,
|
|
backendProfile,
|
|
providerId: resolveAgentRunProviderId(env),
|
|
runId: run.id,
|
|
commandId: command.id,
|
|
attemptId: runnerJob?.attemptId ?? runnerJob?.runner?.attemptId ?? null,
|
|
runnerId: runnerJob?.runnerId ?? runnerJob?.runner?.runnerId ?? null,
|
|
runnerJobId: runnerJob?.id ?? null,
|
|
jobName: runnerJob?.jobName ?? runnerJob?.jobIdentity?.name ?? null,
|
|
namespace: runnerJob?.namespace ?? runnerJob?.jobIdentity?.namespace ?? DEFAULT_RUNNER_NAMESPACE,
|
|
status: runnerJobCreated ? "runner-job-created" : "runner-job-pending",
|
|
runStatus: run.status ?? null,
|
|
commandState: command.state ?? null,
|
|
terminalStatus: null,
|
|
lastSeq: 0,
|
|
traceId,
|
|
...codeAgentOtelTraceFields(traceId, env),
|
|
sessionId: run?.sessionRef?.sessionId ?? null,
|
|
conversationId: run?.sessionRef?.conversationId ?? null,
|
|
threadId: run?.sessionRef?.threadId ?? null,
|
|
runnerReused: false,
|
|
runnerJobCount: runnerJobCreated ? 1 : 0,
|
|
threadReused,
|
|
persistentResume: threadReused,
|
|
reused: false,
|
|
reuseEligible: true,
|
|
createdAt: startedAt,
|
|
updatedAt: nowIso(),
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunMappingWithRunnerJob(mapping = {}, runnerJob = null) {
|
|
return {
|
|
...mapping,
|
|
attemptId: runnerJob?.attemptId ?? runnerJob?.runner?.attemptId ?? mapping.attemptId ?? null,
|
|
runnerId: runnerJob?.runnerId ?? runnerJob?.runner?.runnerId ?? mapping.runnerId ?? null,
|
|
runnerJobId: runnerJob?.id ?? mapping.runnerJobId ?? null,
|
|
jobName: runnerJob?.jobName ?? runnerJob?.jobIdentity?.name ?? mapping.jobName ?? null,
|
|
namespace: runnerJob?.namespace ?? runnerJob?.jobIdentity?.namespace ?? mapping.namespace ?? DEFAULT_RUNNER_NAMESPACE,
|
|
status: "runner-job-created",
|
|
runnerJobCount: Math.max(1, Number(mapping.runnerJobCount ?? 0) + 1),
|
|
updatedAt: nowIso(),
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunReusedMapping({ previous = {}, run = {}, command = {}, runnerJob = null, traceId, startedAt, backendProfile, managerUrl, env }) {
|
|
const threadReused = Boolean(safeOpaqueId(run?.sessionRef?.threadId ?? previous.threadId));
|
|
return {
|
|
...previous,
|
|
adapter: ADAPTER_ID,
|
|
managerUrl,
|
|
backendProfile,
|
|
providerId: previous.providerId ?? resolveAgentRunProviderId(env),
|
|
runId: previous.runId ?? run?.id,
|
|
commandId: command.id,
|
|
attemptId: runnerJob?.attemptId ?? runnerJob?.runner?.attemptId ?? previous.attemptId ?? null,
|
|
runnerId: runnerJob?.runnerId ?? runnerJob?.runner?.runnerId ?? previous.runnerId ?? null,
|
|
runnerJobId: runnerJob?.id ?? previous.runnerJobId ?? null,
|
|
jobName: runnerJob?.jobName ?? runnerJob?.jobIdentity?.name ?? previous.jobName ?? null,
|
|
namespace: runnerJob?.namespace ?? runnerJob?.jobIdentity?.namespace ?? previous.namespace ?? DEFAULT_RUNNER_NAMESPACE,
|
|
status: runnerJob ? "runner-job-ensured" : "runner-job-reused",
|
|
runStatus: run?.status ?? previous.runStatus ?? null,
|
|
commandState: command.state ?? null,
|
|
terminalStatus: null,
|
|
lastSeq: previous.lastSeq ?? 0,
|
|
runnerJobCount: runnerJob ? Math.max(1, Number(previous.runnerJobCount ?? 0) + 1) : Number(previous.runnerJobCount ?? 0),
|
|
traceId,
|
|
sessionId: run?.sessionRef?.sessionId ?? previous.sessionId ?? null,
|
|
conversationId: run?.sessionRef?.conversationId ?? previous.conversationId ?? null,
|
|
threadId: run?.sessionRef?.threadId ?? previous.threadId ?? null,
|
|
runnerReused: true,
|
|
threadReused,
|
|
persistentResume: threadReused,
|
|
reused: true,
|
|
reuseEligible: true,
|
|
createdAt: previous.createdAt ?? startedAt,
|
|
updatedAt: nowIso(),
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function agentRunResultRefs(result = {}) {
|
|
const refs = {};
|
|
for (const key of ["attemptId", "runnerId", "jobName", "namespace", "runnerJobCount"]) {
|
|
if (result?.[key] !== undefined && result?.[key] !== null) refs[key] = result[key];
|
|
}
|
|
if (result?.sessionRef && typeof result.sessionRef === "object") {
|
|
if (result.sessionRef.sessionId) refs.sessionId = result.sessionRef.sessionId;
|
|
if (result.sessionRef.conversationId) refs.conversationId = result.sessionRef.conversationId;
|
|
if (result.sessionRef.threadId) refs.threadId = result.sessionRef.threadId;
|
|
}
|
|
return refs;
|
|
}
|
|
|
|
function withAgentRunRunnerJobCount(mapping = {}) {
|
|
if (!mapping || typeof mapping !== "object") return mapping;
|
|
const count = Number(mapping.runnerJobCount);
|
|
if (Number.isFinite(count) && count >= 0) return mapping;
|
|
const runnerJobEnsured = mapping.reused === true || mapping.status === "runner-job-ensured";
|
|
if (runnerJobEnsured && (mapping.runnerJobId || mapping.jobName || mapping.attemptId || mapping.runnerId)) {
|
|
return { ...mapping, runnerJobCount: 1 };
|
|
}
|
|
return mapping;
|
|
}
|
|
|
|
function hwlabSessionIdForParams(params = {}) {
|
|
return safeSessionId(params.sessionId) || null;
|
|
}
|
|
|
|
function scopedAgentRunSessionIdForParams(params = {}, traceId) {
|
|
const baseSessionId = hwlabSessionIdForParams(params, traceId);
|
|
if (!safeSessionId(baseSessionId)) {
|
|
throw adapterError(
|
|
"agentrun_session_id_required",
|
|
"AgentRun persistent session/PVC requires params.sessionId; refusing to derive sessionRef.sessionId from traceId because that would create a new PVC per turn."
|
|
);
|
|
}
|
|
const base = String(baseSessionId).replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session";
|
|
return `ses_agentrun_${base}`;
|
|
}
|
|
|
|
function agentRunSessionId(traceId) {
|
|
return `ses_agentrun_${String(safeTraceId(traceId) || `trc_${randomUUID()}`).slice(4)}`;
|
|
}
|
|
|
|
function agentRunEffectiveThreadId(base = {}) {
|
|
const agentRun = base?.agentRun && typeof base.agentRun === "object" ? base.agentRun : null;
|
|
if (agentRun && Object.hasOwn(agentRun, "threadId")) return safeOpaqueId(agentRun.threadId) || null;
|
|
return safeOpaqueId(base?.threadId) || null;
|
|
}
|
|
|
|
function agentRunSessionSummary(base, status) {
|
|
return decorateCodeAgentSession({
|
|
sessionId: base.sessionId ?? base.agentRun?.sessionId ?? null,
|
|
conversationId: base.conversationId ?? base.agentRun?.conversationId ?? null,
|
|
threadId: agentRunEffectiveThreadId(base),
|
|
status,
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
runnerKind: AGENTRUN_RUNNER_KIND,
|
|
lastTraceId: base.traceId ?? base.agentRun?.traceId ?? null,
|
|
longLivedSession: true,
|
|
codexStdio: false,
|
|
delegatedToAgentRun: true,
|
|
writeCapable: true,
|
|
durable: true,
|
|
durableSession: true,
|
|
idleTimeoutMs: parsePositiveInteger(base.agentRun?.idleTimeoutMs, 600000),
|
|
secretMaterialStored: false,
|
|
valuesRedacted: true
|
|
});
|
|
}
|
|
|
|
function agentRunThreadReused(base = {}, options = {}) {
|
|
if (options.status === "failed-requires-new-session") return false;
|
|
if (base.agentRun?.threadReused === true || base.agentRun?.persistentResume === true) return true;
|
|
return Boolean(agentRunEffectiveThreadId(base));
|
|
}
|
|
|
|
function agentRunSessionReuseSummary(base, reused, options = {}) {
|
|
const runnerReused = Boolean(reused);
|
|
const threadReused = agentRunThreadReused(base, options);
|
|
const persistentResume = threadReused;
|
|
const sessionReused = runnerReused || persistentResume;
|
|
return {
|
|
sessionId: base.sessionId ?? base.agentRun?.sessionId ?? null,
|
|
conversationId: base.conversationId ?? base.agentRun?.conversationId ?? null,
|
|
threadId: agentRunEffectiveThreadId(base),
|
|
reused: sessionReused,
|
|
status: options.status ?? (runnerReused ? "reused" : persistentResume ? "thread-resumed" : "new"),
|
|
runnerReused,
|
|
threadReused,
|
|
persistentResume,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function agentRunRunnerSummary(mapping = {}) {
|
|
return {
|
|
kind: AGENTRUN_RUNNER_KIND,
|
|
adapter: ADAPTER_ID,
|
|
runId: mapping.runId ?? null,
|
|
commandId: mapping.commandId ?? null,
|
|
attemptId: mapping.attemptId ?? null,
|
|
runnerId: mapping.runnerId ?? null,
|
|
jobName: mapping.jobName ?? null,
|
|
namespace: mapping.namespace ?? null,
|
|
backendProfile: mapping.backendProfile ?? null,
|
|
provider: providerForBackendProfile(mapping.backendProfile ?? "deepseek"),
|
|
sessionMode: AGENTRUN_SESSION_MODE,
|
|
implementationType: AGENTRUN_IMPLEMENTATION_TYPE,
|
|
capabilityLevel: AGENTRUN_CAPABILITY_LEVEL,
|
|
codexStdio: false,
|
|
delegatedToAgentRun: true,
|
|
writeCapable: true,
|
|
durableSession: true,
|
|
longLivedSession: true,
|
|
sandbox: "danger-full-access",
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function hostForUrl(value) {
|
|
if (!value) return null;
|
|
try {
|
|
return new URL(value).hostname.toLowerCase();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function waitingForForPhase(phase) {
|
|
if (phase.includes("runner-job")) return "agentrun-runner";
|
|
if (phase.includes("resource-bundle")) return "agentrun-resource-bundle";
|
|
if (phase.includes("backend")) return "agentrun-backend";
|
|
return "agentrun-result";
|
|
}
|
|
|
|
function textPayload(payload, fallback) {
|
|
for (const key of ["message", "text", "content", "delta", "summary", "phase"]) {
|
|
if (typeof payload?.[key] === "string" && payload[key].length > 0) return payload[key];
|
|
}
|
|
return fallback;
|
|
}
|
|
|
|
function requiredString(value, fieldName) {
|
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
throw Object.assign(new Error(`AgentRun response missing ${fieldName}`), { code: "agentrun_response_invalid", statusCode: 502 });
|
|
}
|
|
|
|
function firstNonEmpty(...values) {
|
|
for (const value of values) {
|
|
const text = String(value ?? "").trim();
|
|
if (text) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function nowIso(now) {
|
|
return typeof now === "function" ? now() : new Date().toISOString();
|
|
}
|
|
|
|
function timestampIsoOrNow(value = null) {
|
|
const parsed = Date.parse(String(value ?? ""));
|
|
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString();
|
|
}
|
|
|
|
function redactUrl(value) {
|
|
try {
|
|
const url = new URL(value);
|
|
if (url.username || url.password) {
|
|
url.username = "***";
|
|
url.password = "";
|
|
}
|
|
return url.toString();
|
|
} catch {
|
|
return "<invalid-url>";
|
|
}
|
|
}
|