root
1815 lines
96 KiB
TypeScript
1815 lines
96 KiB
TypeScript
import type { BackendProfile, BackendTurnResult, CancelRequestRecord, CancelStage, CancelTargetKind, CommandRecord, CreateCommandInput, CreateQueueTaskInput, CreateRunInput, FailureKind, JsonRecord, KafkaEventOutboxRecord, ListGcExpiredSessionsInput, QueueAttemptListResult, QueueAttemptRecord, QueueAttemptRef, QueueCommanderSnapshot, QueueReadCursorRecord, QueueRetryActivation, QueueRetryReservation, QueueStats, QueueTaskListResult, QueueTaskRecord, QueueTaskState, QueueTaskSummary, RunEvent, RunnerDispatchCompletion, RunnerDispatchIntentRecord, RunnerJobRecord, RunnerRecord, RunRecord, SessionEventPage, SessionListResult, SessionListState, SessionReadCursorRecord, SessionRecord, SessionRef, SessionStoragePatch, SessionSummary, TerminalStatus, UpsertSessionInput } from "../common/types.js";
|
|
import { AgentRunError } from "../common/errors.js";
|
|
import { newId, nowIso, stableHash } from "../common/validation.js";
|
|
import { redactJson } from "../common/redaction.js";
|
|
import { backendCapabilities } from "../common/backend-profiles.js";
|
|
import { normalizeRunEventPayload, requireEventType, requireExternallyAppendableEventType, userMessagePayloadForCommand } from "../common/events.js";
|
|
import { buildKafkaEventOutboxRecord } from "./event-outbox.js";
|
|
import { assertRunnerDispatchReplay, attachRunnerDispatchIntent, newRunnerDispatchIntent } from "./runner-dispatch-intent.js";
|
|
|
|
export type MaybePromise<T> = T | Promise<T>;
|
|
|
|
export interface StoreHealth extends JsonRecord {
|
|
adapter: "memory-self-test" | "postgres";
|
|
ready: boolean;
|
|
reachable: boolean;
|
|
migrationReady: boolean;
|
|
migrationId: string | null;
|
|
failureKind: FailureKind | null;
|
|
message: string | null;
|
|
credentialValuesPrinted: false;
|
|
}
|
|
|
|
export interface EventOutboxConfig {
|
|
enabled: boolean;
|
|
topic: string | null;
|
|
source: string | null;
|
|
}
|
|
|
|
export interface DurableQueueClaim {
|
|
id: string;
|
|
leaseOwner: string | null;
|
|
attemptCount: number;
|
|
}
|
|
|
|
export interface RunnerRetentionFenceFacts {
|
|
run: RunRecord;
|
|
command: CommandRecord;
|
|
commands: CommandRecord[];
|
|
events: RunEvent[];
|
|
}
|
|
|
|
export interface RunnerRetentionFenceInput {
|
|
runId: string;
|
|
commandId: string;
|
|
terminalizeStalePending: boolean;
|
|
releaseRunnerClaim: boolean;
|
|
reason: string;
|
|
}
|
|
|
|
export interface SessionTurnAdmissionInput {
|
|
sessionId: string;
|
|
run: CreateRunInput;
|
|
command: CreateCommandInput;
|
|
}
|
|
|
|
export interface SessionTurnAdmission extends JsonRecord {
|
|
run: RunRecord;
|
|
command: CommandRecord;
|
|
disposition: "created" | "replayed" | "recovered";
|
|
recoveredPriorPartialWrite: boolean;
|
|
partialWrite: false;
|
|
}
|
|
|
|
export interface AgentRunStore {
|
|
health(): MaybePromise<StoreHealth>;
|
|
createRun(input: CreateRunInput, identity?: CreateRunIdentity): MaybePromise<RunRecord>;
|
|
getRun(runId: string): MaybePromise<RunRecord>;
|
|
listEvents(runId: string, afterSeq: number, limit: number): MaybePromise<RunEvent[]>;
|
|
listEventsForCommand(runId: string, commandId: string, limit: number): MaybePromise<RunEvent[]>;
|
|
createCommand(runId: string, input: CreateCommandInput): MaybePromise<CommandRecord>;
|
|
admitSessionTurn(input: SessionTurnAdmissionInput): MaybePromise<SessionTurnAdmission>;
|
|
getCommand(commandId: string): MaybePromise<CommandRecord>;
|
|
listCommands(runId: string, afterSeq: number, limit: number): MaybePromise<CommandRecord[]>;
|
|
getRunnerDispatchIntent(commandId: string): MaybePromise<RunnerDispatchIntentRecord | null>;
|
|
claimRunnerDispatchIntents(input: { owner: string; leaseMs: number; limit: number }): MaybePromise<RunnerDispatchIntentRecord[]>;
|
|
completeRunnerDispatchIntent(claim: DurableQueueClaim, completion: RunnerDispatchCompletion, eventPayload: JsonRecord): MaybePromise<RunnerDispatchIntentRecord>;
|
|
retryRunnerDispatchIntent(claim: DurableQueueClaim, error: JsonRecord, nextAttemptAt: string, eventPayload: JsonRecord): MaybePromise<RunnerDispatchIntentRecord>;
|
|
terminalizeRunnerDispatchIntent(claim: DurableQueueClaim, error: JsonRecord): MaybePromise<JsonRecord>;
|
|
runnerDispatchStatus(): MaybePromise<JsonRecord>;
|
|
claimKafkaEventOutbox(input: { owner: string; leaseMs: number; limit: number }): MaybePromise<KafkaEventOutboxRecord[]>;
|
|
completeKafkaEventOutbox(claim: DurableQueueClaim): MaybePromise<KafkaEventOutboxRecord>;
|
|
retryKafkaEventOutbox(claim: DurableQueueClaim, error: JsonRecord, nextAttemptAt: string): MaybePromise<KafkaEventOutboxRecord>;
|
|
kafkaEventOutboxStatus(): MaybePromise<JsonRecord>;
|
|
registerRunner(input: Partial<RunnerRecord>): MaybePromise<RunnerRecord>;
|
|
listRunnerJobs(runId: string, commandId?: string): MaybePromise<RunnerJobRecord[]>;
|
|
listRunnerJobsForReconciliation(limit: number): MaybePromise<RunnerJobRecord[]>;
|
|
getRunnerJobByIdempotencyKey(runId: string, idempotencyKey: string, payloadHash: string): MaybePromise<RunnerJobRecord | null>;
|
|
saveRunnerJob(input: SaveRunnerJobInput): MaybePromise<RunnerJobRecord>;
|
|
updateRunnerJobResult(runnerJobId: string, patch: JsonRecord): MaybePromise<RunnerJobRecord>;
|
|
withRunnerCreateFence<T>(fenceKey: string, operation: () => Promise<T>): MaybePromise<T>;
|
|
withRunnerRetentionFence<T>(input: RunnerRetentionFenceInput, operation: (facts: RunnerRetentionFenceFacts) => Promise<T>): MaybePromise<T>;
|
|
claimRun(runId: string, runnerId: string, leaseMs: number): MaybePromise<RunRecord>;
|
|
heartbeat(runId: string, runnerId: string, leaseMs: number): MaybePromise<RunRecord>;
|
|
ackCommand(commandId: string): MaybePromise<CommandRecord>;
|
|
finishCommand(commandId: string, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">): MaybePromise<CommandRecord>;
|
|
appendEvent(runId: string, type: RunEvent["type"], payload: JsonRecord): MaybePromise<RunEvent>;
|
|
finishRun(runId: string, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">): MaybePromise<RunRecord>;
|
|
cancelRun(runId: string, reason?: string): MaybePromise<RunRecord>;
|
|
cancelCommand(commandId: string, reason?: string): MaybePromise<CommandRecord>;
|
|
getSession(sessionId: string): MaybePromise<SessionRecord | null>;
|
|
getSessionSummary(sessionId: string, readerId?: string | null): MaybePromise<SessionSummary>;
|
|
listSessions(input: ListSessionsInput): MaybePromise<SessionListResult>;
|
|
listSessionTrace(sessionId: string, input: SessionEventPageInput): MaybePromise<SessionEventPage>;
|
|
listSessionOutput(sessionId: string, input: SessionEventPageInput): MaybePromise<SessionEventPage>;
|
|
markSessionRead(sessionId: string, readerId: string): MaybePromise<SessionReadCursorRecord>;
|
|
upsertSession(input: UpsertSessionInput): MaybePromise<SessionRecord>;
|
|
refreshSessionStorage(input: SessionStoragePatch): MaybePromise<SessionRecord>;
|
|
markSessionStorageEvicted(input: { sessionId: string; pvcName: string }): MaybePromise<SessionRecord>;
|
|
listGcExpiredSessions(input: ListGcExpiredSessionsInput): MaybePromise<SessionRecord[]>;
|
|
createQueueTask(input: CreateQueueTaskInput): MaybePromise<QueueTaskRecord>;
|
|
listQueueTasks(input: ListQueueTasksInput): MaybePromise<QueueTaskListResult>;
|
|
getQueueTask(taskId: string): MaybePromise<QueueTaskRecord>;
|
|
listQueueAttempts(taskId: string, input: ListQueueAttemptsInput): MaybePromise<QueueAttemptListResult>;
|
|
reserveQueueRetryAttempt(taskId: string, input: ReserveQueueRetryAttemptInput): MaybePromise<QueueRetryReservation>;
|
|
recordQueueRetryAttemptFailure(taskId: string, attemptId: string, input: RecordQueueRetryAttemptFailureInput): MaybePromise<QueueAttemptRecord>;
|
|
activateQueueRetryAttempt(taskId: string, attemptId: string, input: ActivateQueueRetryAttemptInput): MaybePromise<QueueRetryActivation>;
|
|
updateQueueTaskAttempt(taskId: string, input: UpdateQueueTaskAttemptInput): MaybePromise<QueueTaskRecord>;
|
|
cancelQueueTask(taskId: string, reason?: string): MaybePromise<QueueTaskRecord>;
|
|
markQueueTaskRead(taskId: string, readerId: string): MaybePromise<QueueReadCursorRecord>;
|
|
queueStats(queue?: string): MaybePromise<QueueStats>;
|
|
queueCommander(queue?: string, readerId?: string | null): MaybePromise<QueueCommanderSnapshot>;
|
|
backends(): MaybePromise<JsonRecord[]>;
|
|
close?(): MaybePromise<void>;
|
|
}
|
|
|
|
export interface CreateRunIdentity {
|
|
id: string;
|
|
}
|
|
|
|
export interface ListQueueTasksInput {
|
|
queue?: string;
|
|
state?: QueueTaskState;
|
|
cursor?: string;
|
|
limit: number;
|
|
updatedAfter?: number;
|
|
}
|
|
|
|
export interface ListQueueAttemptsInput {
|
|
cursor?: number;
|
|
limit: number;
|
|
}
|
|
|
|
export interface ReserveQueueRetryAttemptInput {
|
|
idempotencyKey: string;
|
|
reason: string;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
export interface ActivateQueueRetryAttemptInput {
|
|
commandId: string;
|
|
sessionId: string | null;
|
|
sessionPath: string | null;
|
|
state: Exclude<QueueTaskState, "pending">;
|
|
failureKind: FailureKind | null;
|
|
failureMessage: string | null;
|
|
}
|
|
|
|
export interface RecordQueueRetryAttemptFailureInput {
|
|
failureKind: FailureKind;
|
|
failureMessage: string;
|
|
}
|
|
|
|
export interface ListSessionsInput {
|
|
state?: SessionListState;
|
|
backendProfile?: BackendProfile;
|
|
readerId?: string | null;
|
|
cursor?: string;
|
|
limit: number;
|
|
}
|
|
|
|
export interface SessionEventPageInput {
|
|
runId?: string | null;
|
|
afterSeq: number;
|
|
limit: number;
|
|
}
|
|
|
|
export interface UpdateQueueTaskAttemptInput {
|
|
state: QueueTaskState;
|
|
latestAttempt: QueueAttemptRef;
|
|
sessionPath: string | null;
|
|
}
|
|
|
|
export interface SaveRunnerJobInput {
|
|
id?: string;
|
|
runId: string;
|
|
commandId: string;
|
|
idempotencyKey?: string | null;
|
|
payloadHash: string;
|
|
attemptId: string;
|
|
runnerId: string;
|
|
namespace: string;
|
|
jobName: string;
|
|
managerUrl: string;
|
|
image: string;
|
|
sourceCommit: string;
|
|
serviceAccountName?: string | null;
|
|
result: JsonRecord;
|
|
}
|
|
|
|
export async function openAgentRunStoreFromEnv(env: NodeJS.ProcessEnv = process.env): Promise<AgentRunStore> {
|
|
const eventOutbox = eventOutboxConfigFromEnv(env);
|
|
const databaseUrl = env.DATABASE_URL?.trim();
|
|
if (databaseUrl) {
|
|
const { createPostgresAgentRunStore } = await import("./postgres-store.js");
|
|
const poolMax = optionalPositiveIntegerEnv(env, "AGENTRUN_POSTGRES_POOL_MAX");
|
|
return createPostgresAgentRunStore({ connectionString: databaseUrl, ...(poolMax !== undefined ? { poolMax } : {}), eventOutbox });
|
|
}
|
|
const storeMode = env.AGENTRUN_STORE ?? env.AGENTRUN_MGR_STORE;
|
|
if (storeMode === "memory") return new MemoryAgentRunStore({ eventOutbox });
|
|
throw new AgentRunError("infra-failed", "DATABASE_URL is required for agentrun-mgr live runtime; set AGENTRUN_STORE=memory only for explicit self-test/dev mode", { httpStatus: 503, details: { adapter: "postgres", databaseUrl: "missing", memoryFallback: "disabled" } });
|
|
}
|
|
|
|
function eventOutboxConfigFromEnv(env: NodeJS.ProcessEnv): EventOutboxConfig {
|
|
const enabled = ["1", "true", "yes", "on"].includes(String(env.AGENTRUN_KAFKA_ENABLED ?? "").trim().toLowerCase());
|
|
if (!enabled) return { enabled: false, topic: null, source: null };
|
|
const topic = env.AGENTRUN_KAFKA_EVENT_TOPIC?.trim();
|
|
const source = env.AGENTRUN_KAFKA_CLIENT_ID?.trim();
|
|
const missing = [...(!topic ? ["AGENTRUN_KAFKA_EVENT_TOPIC"] : []), ...(!source ? ["AGENTRUN_KAFKA_CLIENT_ID"] : [])];
|
|
if (missing.length > 0) throw new AgentRunError("infra-failed", "Kafka event outbox configuration is incomplete", { httpStatus: 503, details: { missing, valuesPrinted: false } });
|
|
return { enabled: true, topic: topic as string, source: source as string };
|
|
}
|
|
|
|
function optionalPositiveIntegerEnv(env: NodeJS.ProcessEnv, name: string): number | undefined {
|
|
const raw = env[name]?.trim();
|
|
if (!raw) return undefined;
|
|
const value = Number(raw);
|
|
if (!Number.isSafeInteger(value) || value < 1) {
|
|
throw new AgentRunError("infra-failed", `${name} must be a positive integer`, { httpStatus: 503, details: { env: name, valuesPrinted: false } });
|
|
}
|
|
return value;
|
|
}
|
|
|
|
export class MemoryAgentRunStore implements AgentRunStore {
|
|
private readonly eventOutboxConfig: EventOutboxConfig;
|
|
private readonly runs = new Map<string, RunRecord>();
|
|
private readonly commands = new Map<string, CommandRecord>();
|
|
private readonly eventsByRun = new Map<string, RunEvent[]>();
|
|
private readonly runners = new Map<string, RunnerRecord>();
|
|
private readonly sessions = new Map<string, SessionRecord>();
|
|
private readonly cancelRequests = new Map<string, CancelRequestRecord>();
|
|
private readonly sessionReadCursors = new Map<string, SessionReadCursorRecord>();
|
|
private readonly runnerJobs = new Map<string, RunnerJobRecord>();
|
|
private readonly runnerDispatchIntents = new Map<string, RunnerDispatchIntentRecord>();
|
|
private readonly kafkaEventOutbox = new Map<string, KafkaEventOutboxRecord>();
|
|
private readonly queueTasks = new Map<string, QueueTaskRecord>();
|
|
private readonly queueAttempts = new Map<string, QueueAttemptRecord>();
|
|
private readonly queueReadCursors = new Map<string, QueueReadCursorRecord>();
|
|
private readonly runnerCreateFenceTails = new Map<string, Promise<void>>();
|
|
private readonly runnerRetentionFences = new Set<string>();
|
|
private queueVersion = 0;
|
|
private sessionVersion = 0;
|
|
private kafkaOutboxSeq = 0;
|
|
|
|
constructor(options: { eventOutbox?: EventOutboxConfig } = {}) {
|
|
this.eventOutboxConfig = options.eventOutbox ?? { enabled: false, topic: null, source: null };
|
|
}
|
|
|
|
health(): StoreHealth {
|
|
return { adapter: "memory-self-test", ready: true, reachable: true, migrationReady: true, migrationId: "memory-self-test", failureKind: null, message: null, credentialValuesPrinted: false };
|
|
}
|
|
|
|
createRun(input: CreateRunInput, identity?: CreateRunIdentity): RunRecord {
|
|
if (identity) {
|
|
const existing = this.runs.get(identity.id);
|
|
if (existing) {
|
|
assertRunCreateReplay(existing, input);
|
|
return existing;
|
|
}
|
|
}
|
|
const at = nowIso();
|
|
const sessionRef = this.resolveSessionForRun(input, at);
|
|
const run: RunRecord = { ...input, sessionRef, resourceBundleRef: input.resourceBundleRef ?? null, id: identity?.id ?? newId("run"), status: "pending", terminalStatus: null, failureKind: null, failureMessage: null, cancelEpoch: 0, cancelRequestId: null, cancelRequestedAt: null, cancelReason: null, createdAt: at, updatedAt: at, claimedBy: null, leaseExpiresAt: null };
|
|
this.runs.set(run.id, run);
|
|
this.eventsByRun.set(run.id, []);
|
|
this.touchSessionForRun(run, { lastRunId: run.id, lastActivityAt: at }, { bumpVersion: false, at });
|
|
return run;
|
|
}
|
|
|
|
getRun(runId: string): RunRecord {
|
|
const run = this.runs.get(runId);
|
|
if (!run) throw new AgentRunError("schema-invalid", `run ${runId} was not found`, { httpStatus: 404 });
|
|
return run;
|
|
}
|
|
|
|
listEvents(runId: string, afterSeq: number, limit: number): RunEvent[] {
|
|
this.getRun(runId);
|
|
return (this.eventsByRun.get(runId) ?? []).filter((event) => event.seq > afterSeq).slice(0, Math.max(1, Math.min(limit, 500)));
|
|
}
|
|
|
|
listEventsForCommand(runId: string, commandId: string, limit: number): RunEvent[] {
|
|
this.getRun(runId);
|
|
const clamped = Math.max(1, Math.min(limit, 2_000));
|
|
return (this.eventsByRun.get(runId) ?? [])
|
|
.filter((event) => eventMatchesCommand(event, commandId))
|
|
.slice(0, clamped);
|
|
}
|
|
|
|
createCommand(runId: string, input: CreateCommandInput): CommandRecord {
|
|
const run = this.getRun(runId);
|
|
const payloadHash = stableHash(input.payload);
|
|
if (input.idempotencyKey) {
|
|
const existing = Array.from(this.commands.values()).find((command) => command.runId === runId && command.idempotencyKey === input.idempotencyKey);
|
|
if (existing) {
|
|
if (existing.payloadHash !== payloadHash) throw new AgentRunError("schema-invalid", "idempotency key reused with different payload", { httpStatus: 409 });
|
|
const intent = this.getRunnerDispatchIntent(existing.id);
|
|
assertRunnerDispatchReplay(intent, existing, input.dispatch);
|
|
return attachRunnerDispatchIntent(existing, intent);
|
|
}
|
|
}
|
|
if (isTerminalRunStatus(run.status)) throw new AgentRunError(run.failureKind ?? (run.status === "cancelled" ? "cancelled" : "schema-invalid"), `run ${runId} is already terminal: ${run.status}`, { httpStatus: 409 });
|
|
const at = nowIso();
|
|
const seq = Array.from(this.commands.values()).filter((command) => command.runId === runId).length + 1;
|
|
const { dispatch: _dispatch, ...commandInput } = input;
|
|
const command: CommandRecord = { ...commandInput, id: newId("cmd"), runId, seq, state: "pending", payloadHash, cancelEpoch: 0, cancelRequestId: null, cancelRequestedAt: null, cancelReason: null, createdAt: at, updatedAt: at, acknowledgedAt: null };
|
|
const userMessage = userMessagePayloadForCommand(run, command);
|
|
const needsRunCreatedEvent = !(this.eventsByRun.get(runId) ?? []).some(isRunCreatedEvent);
|
|
const eventSpecs: Array<{ type: RunEvent["type"]; payload: JsonRecord; command: CommandRecord | null }> = [
|
|
...(userMessage ? [{ type: "user_message" as const, payload: userMessage, command }] : []),
|
|
...(needsRunCreatedEvent ? [{ type: "backend_status" as const, payload: runCreatedEventPayload(run), command: null }] : []),
|
|
{ type: "backend_status", payload: { phase: "command-created", commandId: command.id, commandType: command.type }, command },
|
|
];
|
|
const eventSeqBase = (this.eventsByRun.get(runId) ?? []).length;
|
|
const outboxSeqBase = this.kafkaOutboxSeq;
|
|
const preparedEvents = eventSpecs.map((spec, index) => this.prepareEvent(runId, spec.type, spec.payload, {
|
|
command: spec.command,
|
|
eventSeq: eventSeqBase + index + 1,
|
|
outboxSeq: outboxSeqBase + index + 1,
|
|
}));
|
|
this.commands.set(command.id, command);
|
|
const intent = input.dispatch ? newRunnerDispatchIntent(command, input.dispatch, at) : null;
|
|
if (intent) this.runnerDispatchIntents.set(intent.id, intent);
|
|
if (command.type === "turn") this.touchSessionForRun(run, { executionState: "running", lastRunId: run.id, lastCommandId: command.id, activeRunId: run.id, activeCommandId: command.id, terminalStatus: null, failureKind: null, title: sessionTitleFromCommand(command), lastActivityAt: at }, { bumpVersion: true, at });
|
|
else if (command.type === "steer") this.touchSessionForRun(run, { executionState: "running", lastRunId: run.id, lastCommandId: command.id, activeRunId: run.id, lastActivityAt: at }, { bumpVersion: true, at });
|
|
for (const prepared of preparedEvents) this.commitPreparedEvent(prepared);
|
|
return attachRunnerDispatchIntent(command, intent);
|
|
}
|
|
|
|
admitSessionTurn(input: SessionTurnAdmissionInput): SessionTurnAdmission {
|
|
assertSessionTurnAdmissionContract(input);
|
|
const session = this.sessions.get(input.sessionId) ?? null;
|
|
const replay = input.command.idempotencyKey
|
|
? Array.from(this.commands.values())
|
|
.filter((command) => command.idempotencyKey === input.command.idempotencyKey)
|
|
.map((command) => ({ command, run: this.runs.get(command.runId) ?? null }))
|
|
.find((item) => item.run?.sessionRef?.sessionId === input.sessionId) ?? null
|
|
: null;
|
|
if (replay?.run) return this.replayOrRecoverSessionTurn(replay.run, replay.command, input, "replayed");
|
|
|
|
if (session?.activeRunId || session?.activeCommandId) {
|
|
if (!session.activeRunId || !session.activeCommandId) throw invalidSessionAdmissionProjection(input.sessionId);
|
|
const run = this.runs.get(session.activeRunId);
|
|
const command = this.commands.get(session.activeCommandId);
|
|
if (!run || !command || command.runId !== run.id || run.sessionRef?.sessionId !== input.sessionId) throw invalidSessionAdmissionProjection(input.sessionId);
|
|
if (!isTerminalRunStatus(run.status) && !isTerminalCommandState(command.state)) {
|
|
try {
|
|
return this.replayOrRecoverSessionTurn(run, command, input, "replayed");
|
|
} catch (error) {
|
|
if (error instanceof AgentRunError && error.failureKind === "schema-invalid") throw activeSessionAdmissionConflict(input.sessionId, run, command);
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
const sessionBefore = session ? { ...session } : null;
|
|
const sessionVersionBefore = this.sessionVersion;
|
|
const kafkaOutboxSeqBefore = this.kafkaOutboxSeq;
|
|
let run: RunRecord | null = null;
|
|
try {
|
|
run = this.createRun(input.run);
|
|
const command = this.createCommand(run.id, input.command);
|
|
return { run, command, disposition: "created", recoveredPriorPartialWrite: false, partialWrite: false };
|
|
} catch (error) {
|
|
if (run) {
|
|
this.runs.delete(run.id);
|
|
this.eventsByRun.delete(run.id);
|
|
for (const [commandId, command] of this.commands) if (command.runId === run.id) this.commands.delete(commandId);
|
|
for (const [intentId, intent] of this.runnerDispatchIntents) if (intent.runId === run.id) this.runnerDispatchIntents.delete(intentId);
|
|
for (const [outboxId, item] of this.kafkaEventOutbox) if (item.runId === run.id) this.kafkaEventOutbox.delete(outboxId);
|
|
}
|
|
if (sessionBefore) this.sessions.set(input.sessionId, sessionBefore);
|
|
else this.sessions.delete(input.sessionId);
|
|
this.sessionVersion = sessionVersionBefore;
|
|
this.kafkaOutboxSeq = kafkaOutboxSeqBefore;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private replayOrRecoverSessionTurn(run: RunRecord, command: CommandRecord, input: SessionTurnAdmissionInput, disposition: "replayed" | "recovered"): SessionTurnAdmission {
|
|
assertRunCreateReplay(run, input.run);
|
|
assertSessionCommandReplay(command, input.command);
|
|
const intent = this.getRunnerDispatchIntent(command.id);
|
|
if (intent) {
|
|
assertRunnerDispatchReplay(intent, command, input.command.dispatch);
|
|
return { run, command: attachRunnerDispatchIntent(command, intent), disposition: "replayed", recoveredPriorPartialWrite: false, partialWrite: false };
|
|
}
|
|
if (!input.command.dispatch) {
|
|
return { run, command: attachRunnerDispatchIntent(command, null), disposition, recoveredPriorPartialWrite: false, partialWrite: false };
|
|
}
|
|
if (run.status !== "pending" || run.claimedBy || command.state !== "pending") throw activeSessionAdmissionConflict(input.sessionId, run, command);
|
|
const at = nowIso();
|
|
const recovered = newRunnerDispatchIntent(command, input.command.dispatch, at);
|
|
const event = this.prepareEvent(run.id, "backend_status", {
|
|
phase: "runner-dispatch-recovered",
|
|
reason: "session-runner-admission-partial-write-recovered",
|
|
commandId: command.id,
|
|
dispatchIntentId: recovered.id,
|
|
plannedRunnerJobId: recovered.runnerJobId,
|
|
valuesPrinted: false,
|
|
});
|
|
this.runnerDispatchIntents.set(recovered.id, recovered);
|
|
this.commitPreparedEvent(event);
|
|
return { run, command: attachRunnerDispatchIntent(command, recovered), disposition: "recovered", recoveredPriorPartialWrite: true, partialWrite: false };
|
|
}
|
|
|
|
getRunnerDispatchIntent(commandId: string): RunnerDispatchIntentRecord | null {
|
|
return Array.from(this.runnerDispatchIntents.values()).find((intent) => intent.commandId === commandId) ?? null;
|
|
}
|
|
|
|
claimRunnerDispatchIntents(input: { owner: string; leaseMs: number; limit: number }): RunnerDispatchIntentRecord[] {
|
|
const now = Date.now();
|
|
for (const intent of this.runnerDispatchIntents.values()) {
|
|
if (!isActiveRunnerDispatchIntent(intent)) continue;
|
|
const run = this.getRun(intent.runId);
|
|
const command = this.getCommand(intent.commandId);
|
|
if (isTerminalRunStatus(run.status) || isTerminalCommandState(command.state)) this.cancelRunnerDispatchIntent(intent, "run or command is terminal");
|
|
}
|
|
const claimed = Array.from(this.runnerDispatchIntents.values())
|
|
.filter((intent) => (intent.state === "pending" || intent.state === "retry" || (intent.state === "dispatching" && Date.parse(intent.leaseExpiresAt ?? "") <= now)) && Date.parse(intent.nextAttemptAt) <= now)
|
|
.sort((a, b) => a.nextAttemptAt.localeCompare(b.nextAttemptAt) || a.createdAt.localeCompare(b.createdAt))
|
|
.slice(0, Math.max(1, Math.min(input.limit, 500)));
|
|
return claimed.map((intent) => {
|
|
const updated: RunnerDispatchIntentRecord = { ...intent, state: "dispatching", attemptCount: intent.attemptCount + 1, leaseOwner: input.owner, leaseExpiresAt: new Date(now + input.leaseMs).toISOString(), updatedAt: nowIso() };
|
|
this.runnerDispatchIntents.set(updated.id, updated);
|
|
return updated;
|
|
});
|
|
}
|
|
|
|
completeRunnerDispatchIntent(claim: DurableQueueClaim, completion: RunnerDispatchCompletion, eventPayload: JsonRecord): RunnerDispatchIntentRecord {
|
|
const intent = this.requireRunnerDispatchIntent(claim.id);
|
|
this.fenceRunnerDispatchClaim(intent, claim);
|
|
const event = this.prepareEvent(intent.runId, "backend_status", eventPayload);
|
|
const at = nowIso();
|
|
const updated: RunnerDispatchIntentRecord = { ...intent, state: "dispatched", ...completion, leaseOwner: null, leaseExpiresAt: null, lastError: null, dispatchedAt: at, updatedAt: at };
|
|
this.runnerDispatchIntents.set(intent.id, updated);
|
|
this.commitPreparedEvent(event);
|
|
return updated;
|
|
}
|
|
|
|
retryRunnerDispatchIntent(claim: DurableQueueClaim, error: JsonRecord, nextAttemptAt: string, eventPayload: JsonRecord): RunnerDispatchIntentRecord {
|
|
const intent = this.requireRunnerDispatchIntent(claim.id);
|
|
this.fenceRunnerDispatchClaim(intent, claim);
|
|
const event = this.prepareEvent(intent.runId, "backend_status", eventPayload);
|
|
const updated: RunnerDispatchIntentRecord = { ...intent, state: "retry", leaseOwner: null, leaseExpiresAt: null, nextAttemptAt, lastError: redactJson(error), updatedAt: nowIso() };
|
|
this.runnerDispatchIntents.set(intent.id, updated);
|
|
this.commitPreparedEvent(event);
|
|
return updated;
|
|
}
|
|
|
|
terminalizeRunnerDispatchIntent(claim: DurableQueueClaim, error: JsonRecord): JsonRecord {
|
|
const intent = this.requireRunnerDispatchIntent(claim.id);
|
|
this.fenceRunnerDispatchClaim(intent, claim);
|
|
const at = nowIso();
|
|
const failure = redactJson(error);
|
|
const message = typeof failure.message === "string" ? failure.message : "runner dispatch failed";
|
|
const reason = typeof failure.reason === "string" ? failure.reason : "runner-dispatch-attempt-failed";
|
|
const updatedIntent: RunnerDispatchIntentRecord = { ...intent, state: "failed", leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: at, lastError: failure, updatedAt: at };
|
|
this.runnerDispatchIntents.set(intent.id, updatedIntent);
|
|
const command = this.getCommand(intent.commandId);
|
|
const updatedCommand = isTerminalCommandState(command.state) ? command : { ...command, state: "failed" as const, updatedAt: at };
|
|
this.commands.set(command.id, updatedCommand);
|
|
const run = this.getRun(intent.runId);
|
|
const updatedRun = isTerminalRunStatus(run.status) ? run : this.updateRun(run.id, { status: "failed", terminalStatus: "failed", failureKind: "infra-failed", failureMessage: message });
|
|
if (!isTerminalRunStatus(run.status)) {
|
|
this.cancelRunnerDispatchIntentsForRun(run.id, "run terminalized by runner dispatch failure", at);
|
|
this.appendEvent(run.id, "backend_status", { phase: "runner-dispatch-failed", reason, commandId: command.id, dispatchIntentId: intent.id, attemptCount: intent.attemptCount, failureKind: "infra-failed", message, valuesPrinted: false });
|
|
this.appendEvent(run.id, "backend_status", { phase: "command-terminal", commandId: command.id, state: "failed", terminalStatus: "failed", failureKind: "infra-failed", message, threadId: null, turnId: null });
|
|
this.appendEvent(run.id, "terminal_status", { terminalStatus: "failed", failureKind: "infra-failed", message });
|
|
this.touchSessionForRun(this.getRun(run.id), { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: run.id, lastCommandId: command.id, terminalStatus: "failed", failureKind: "infra-failed", lastActivityAt: at }, { bumpVersion: true, at });
|
|
}
|
|
return { intent: updatedIntent, command: updatedCommand, run: updatedRun, valuesPrinted: false };
|
|
}
|
|
|
|
runnerDispatchStatus(): JsonRecord {
|
|
return workQueueStatus(Array.from(this.runnerDispatchIntents.values()), "runner-dispatch");
|
|
}
|
|
|
|
claimKafkaEventOutbox(input: { owner: string; leaseMs: number; limit: number }): KafkaEventOutboxRecord[] {
|
|
const now = Date.now();
|
|
const all = Array.from(this.kafkaEventOutbox.values());
|
|
const claimed = all
|
|
.filter((item) => !item.deliveredAt && Date.parse(item.nextAttemptAt) <= now && (!item.leaseExpiresAt || Date.parse(item.leaseExpiresAt) <= now))
|
|
.filter((item) => !all.some((earlier) => !earlier.deliveredAt && earlier.topic === item.topic && earlier.partitionKey === item.partitionKey && earlier.outboxSeq < item.outboxSeq))
|
|
.sort((a, b) => a.outboxSeq - b.outboxSeq)
|
|
.slice(0, Math.max(1, Math.min(input.limit, 500)));
|
|
return claimed.map((item) => {
|
|
const updated: KafkaEventOutboxRecord = { ...item, attemptCount: item.attemptCount + 1, leaseOwner: input.owner, leaseExpiresAt: new Date(now + input.leaseMs).toISOString(), updatedAt: nowIso() };
|
|
this.kafkaEventOutbox.set(item.id, updated);
|
|
return updated;
|
|
});
|
|
}
|
|
|
|
completeKafkaEventOutbox(claim: DurableQueueClaim): KafkaEventOutboxRecord {
|
|
const item = this.requireKafkaEventOutbox(claim.id);
|
|
assertMemoryClaim(item, claim, "Kafka event outbox", !item.deliveredAt);
|
|
const at = nowIso();
|
|
const updated: KafkaEventOutboxRecord = { ...item, deliveredAt: at, leaseOwner: null, leaseExpiresAt: null, lastError: null, updatedAt: at };
|
|
this.kafkaEventOutbox.set(item.id, updated);
|
|
return updated;
|
|
}
|
|
|
|
retryKafkaEventOutbox(claim: DurableQueueClaim, error: JsonRecord, nextAttemptAt: string): KafkaEventOutboxRecord {
|
|
const item = this.requireKafkaEventOutbox(claim.id);
|
|
assertMemoryClaim(item, claim, "Kafka event outbox", !item.deliveredAt);
|
|
const updated: KafkaEventOutboxRecord = { ...item, leaseOwner: null, leaseExpiresAt: null, nextAttemptAt, lastError: redactJson(error), updatedAt: nowIso() };
|
|
this.kafkaEventOutbox.set(item.id, updated);
|
|
return updated;
|
|
}
|
|
|
|
kafkaEventOutboxStatus(): JsonRecord {
|
|
return { ...workQueueStatus(Array.from(this.kafkaEventOutbox.values()), "kafka-event-outbox", (item) => Boolean((item as KafkaEventOutboxRecord).deliveredAt)), enabled: this.eventOutboxConfig.enabled };
|
|
}
|
|
|
|
getCommand(commandId: string): CommandRecord {
|
|
const command = this.commands.get(commandId);
|
|
if (!command) throw new AgentRunError("schema-invalid", `command ${commandId} was not found`, { httpStatus: 404 });
|
|
return command;
|
|
}
|
|
|
|
listCommands(runId: string, afterSeq: number, limit: number): CommandRecord[] {
|
|
this.getRun(runId);
|
|
return Array.from(this.commands.values()).filter((command) => command.runId === runId && command.seq > afterSeq).slice(0, Math.max(1, Math.min(limit, 100)));
|
|
}
|
|
|
|
registerRunner(input: Partial<RunnerRecord>): RunnerRecord {
|
|
const at = nowIso();
|
|
const runner: RunnerRecord = { id: input.id ?? newId("runner"), registeredAt: at, heartbeatAt: at, ...input };
|
|
this.runners.set(runner.id, runner);
|
|
return runner;
|
|
}
|
|
|
|
listRunnerJobs(runId: string, commandId?: string): RunnerJobRecord[] {
|
|
this.getRun(runId);
|
|
return Array.from(this.runnerJobs.values())
|
|
.filter((job) => job.runId === runId && (!commandId || job.commandId === commandId))
|
|
.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
}
|
|
|
|
listRunnerJobsForReconciliation(limit: number): RunnerJobRecord[] {
|
|
const clamped = Math.max(1, Math.min(limit, 500));
|
|
return Array.from(this.runnerJobs.values())
|
|
.filter((job) => {
|
|
const run = this.runs.get(job.runId);
|
|
const command = this.commands.get(job.commandId);
|
|
return !run || !command || !isTerminalRunStatus(run.status) || !isTerminalCommandState(command.state);
|
|
})
|
|
.sort((a, b) => a.updatedAt.localeCompare(b.updatedAt) || a.createdAt.localeCompare(b.createdAt))
|
|
.slice(0, clamped);
|
|
}
|
|
|
|
getRunnerJobByIdempotencyKey(runId: string, idempotencyKey: string, payloadHash: string): RunnerJobRecord | null {
|
|
const existing = Array.from(this.runnerJobs.values()).find((job) => job.runId === runId && job.idempotencyKey === idempotencyKey);
|
|
if (!existing) return null;
|
|
if (existing.payloadHash !== payloadHash) throw new AgentRunError("schema-invalid", "runner job idempotency key reused with different payload", { httpStatus: 409 });
|
|
return existing;
|
|
}
|
|
|
|
saveRunnerJob(input: SaveRunnerJobInput): RunnerJobRecord {
|
|
if (input.idempotencyKey) {
|
|
const existing = this.getRunnerJobByIdempotencyKey(input.runId, input.idempotencyKey, input.payloadHash);
|
|
if (existing) return existing;
|
|
}
|
|
const at = nowIso();
|
|
const record: RunnerJobRecord = { ...input, id: input.id ?? newId("rjob"), idempotencyKey: input.idempotencyKey ?? null, serviceAccountName: input.serviceAccountName ?? null, createdAt: at, updatedAt: at };
|
|
this.runnerJobs.set(record.id, record);
|
|
return record;
|
|
}
|
|
|
|
updateRunnerJobResult(runnerJobId: string, patch: JsonRecord): RunnerJobRecord {
|
|
const existing = this.runnerJobs.get(runnerJobId);
|
|
if (!existing) throw new AgentRunError("schema-invalid", `runner job ${runnerJobId} was not found`, { httpStatus: 404 });
|
|
const next: RunnerJobRecord = { ...existing, result: { ...existing.result, ...patch }, updatedAt: nowIso() };
|
|
this.runnerJobs.set(runnerJobId, next);
|
|
return next;
|
|
}
|
|
|
|
async withRunnerCreateFence<T>(fenceKey: string, operation: () => Promise<T>): Promise<T> {
|
|
const predecessor = this.runnerCreateFenceTails.get(fenceKey) ?? Promise.resolve();
|
|
let release = (): void => {};
|
|
const current = new Promise<void>((resolve) => { release = resolve; });
|
|
const tail = predecessor.then(async () => current);
|
|
this.runnerCreateFenceTails.set(fenceKey, tail);
|
|
await predecessor;
|
|
try {
|
|
return await operation();
|
|
} finally {
|
|
release();
|
|
if (this.runnerCreateFenceTails.get(fenceKey) === tail) this.runnerCreateFenceTails.delete(fenceKey);
|
|
}
|
|
}
|
|
|
|
async withRunnerRetentionFence<T>(input: RunnerRetentionFenceInput, operation: (facts: RunnerRetentionFenceFacts) => Promise<T>): Promise<T> {
|
|
if (this.runnerRetentionFences.has(input.runId)) throw runnerRetentionFenceConflict(input.runId);
|
|
const run = this.getRun(input.runId);
|
|
const command = this.getCommand(input.commandId);
|
|
if (command.runId !== run.id) throw new AgentRunError("schema-invalid", `command ${input.commandId} does not belong to run ${input.runId}`, { httpStatus: 409 });
|
|
this.runnerRetentionFences.add(input.runId);
|
|
try {
|
|
const commands = this.listCommands(run.id, 0, 100);
|
|
const events = this.listEvents(run.id, 0, 500);
|
|
const result = await operation({ run, command, commands, events });
|
|
if (input.terminalizeStalePending && !isTerminalRunStatus(run.status)) this.terminalizeRunnerRetention(run, commands, input.reason);
|
|
else if (input.releaseRunnerClaim && (isTerminalRunStatus(run.status) || isTerminalCommandState(command.state))) this.releaseRunnerClaim(run);
|
|
return result;
|
|
} finally {
|
|
this.runnerRetentionFences.delete(input.runId);
|
|
}
|
|
}
|
|
|
|
claimRun(runId: string, runnerId: string, leaseMs: number): RunRecord {
|
|
if (this.runnerRetentionFences.has(runId)) throw runnerRetentionFenceConflict(runId);
|
|
const run = this.getRun(runId);
|
|
if (isTerminalRunStatus(run.status)) throw new AgentRunError(run.failureKind ?? (run.status === "cancelled" ? "cancelled" : "schema-invalid"), `run ${runId} is already terminal: ${run.status}`, { httpStatus: 409 });
|
|
if (run.claimedBy && run.claimedBy !== runnerId && !isLeaseExpired(run.leaseExpiresAt)) throw new AgentRunError("runner-lease-conflict", `run ${runId} is already claimed`, { httpStatus: 409 });
|
|
const next = this.updateRun(runId, { status: "claimed", claimedBy: runnerId, leaseExpiresAt: new Date(Date.now() + leaseMs).toISOString() });
|
|
this.touchSessionForRun(next, { executionState: "running", activeRunId: runId, lastRunId: runId, lastActivityAt: next.updatedAt }, { bumpVersion: false, at: next.updatedAt });
|
|
this.appendEvent(runId, "backend_status", { phase: "run-claimed", runnerId });
|
|
return next;
|
|
}
|
|
|
|
heartbeat(runId: string, runnerId: string, leaseMs: number): RunRecord {
|
|
if (this.runnerRetentionFences.has(runId)) throw runnerRetentionFenceConflict(runId);
|
|
const run = this.getRun(runId);
|
|
if (isTerminalRunStatus(run.status)) return run;
|
|
if (run.claimedBy !== runnerId) throw new AgentRunError("runner-lease-conflict", `run ${runId} is not claimed by ${runnerId}`, { httpStatus: 409 });
|
|
return this.updateRun(runId, { leaseExpiresAt: new Date(Date.now() + leaseMs).toISOString() });
|
|
}
|
|
|
|
ackCommand(commandId: string): CommandRecord {
|
|
const command = this.getCommand(commandId);
|
|
if (isTerminalCommandState(command.state) || command.state === "acknowledged") return command;
|
|
const next = { ...command, state: "acknowledged" as const, acknowledgedAt: nowIso(), updatedAt: nowIso() };
|
|
this.commands.set(commandId, next);
|
|
return next;
|
|
}
|
|
|
|
finishCommand(commandId: string, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">): CommandRecord {
|
|
const command = this.getCommand(commandId);
|
|
let run = this.getRun(command.runId);
|
|
if (isTerminalRunStatus(run.status)) run = this.releaseTerminalRunClaim(run);
|
|
if (isTerminalCommandState(command.state)) {
|
|
this.cancelRunnerDispatchIntentForCommand(commandId, "command is terminal");
|
|
if (command.state === "cancelled" && result.terminalStatus !== "cancelled") this.appendLateWriteRejected(run.id, command, result, "command-status");
|
|
return command;
|
|
}
|
|
const next = { ...command, state: commandStateFromTerminal(result.terminalStatus), updatedAt: nowIso() };
|
|
this.commands.set(commandId, next);
|
|
this.cancelRunnerDispatchIntentForCommand(commandId, `command terminalized as ${next.state}`, next.updatedAt);
|
|
if (result.threadId && run.sessionRef?.sessionId) this.upsertSessionThread(run, result.threadId, result.turnId ?? null);
|
|
if (command.type === "turn") this.touchSessionForRun(this.getRun(command.runId), { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: command.runId, lastCommandId: command.id, terminalStatus: result.terminalStatus, failureKind: result.failureKind, lastActivityAt: next.updatedAt }, { bumpVersion: true, at: next.updatedAt });
|
|
this.appendEvent(command.runId, "backend_status", { phase: "command-terminal", commandId, state: next.state, terminalStatus: result.terminalStatus, failureKind: result.failureKind, message: result.failureMessage ?? null, threadId: result.threadId ?? null, turnId: result.turnId ?? null });
|
|
return next;
|
|
}
|
|
|
|
appendEvent(runId: string, type: RunEvent["type"], payload: JsonRecord): RunEvent {
|
|
const prepared = this.prepareEvent(runId, requireExternallyAppendableEventType(type), payload);
|
|
this.commitPreparedEvent(prepared);
|
|
return prepared.event;
|
|
}
|
|
|
|
protected beforeEventCommit(_event: RunEvent): void {}
|
|
|
|
private prepareEvent(runId: string, type: RunEvent["type"], payload: JsonRecord, options: { command?: CommandRecord | null; eventSeq?: number; outboxSeq?: number } = {}): { run: RunRecord; event: RunEvent; outbox: KafkaEventOutboxRecord | null } {
|
|
const run = this.getRun(runId);
|
|
const eventType = requireEventType(type);
|
|
const fenced = fenceLateEventForCancelledRun(run, eventType, payload);
|
|
const eventPayload = normalizeRunEventPayload(fenced.type, fenced.payload);
|
|
const events = this.eventsByRun.get(runId) ?? [];
|
|
const event: RunEvent = { id: newId("evt"), runId, seq: options.eventSeq ?? events.length + 1, type: fenced.type, payload: redactJson(eventPayload), createdAt: nowIso() };
|
|
let outbox: KafkaEventOutboxRecord | null = null;
|
|
if (this.eventOutboxConfig.enabled) {
|
|
const command = options.command === undefined ? commandForEvent(this.commands, event) : options.command;
|
|
outbox = buildKafkaEventOutboxRecord({ source: this.eventOutboxConfig.source as string, topic: this.eventOutboxConfig.topic as string, outboxSeq: options.outboxSeq ?? this.kafkaOutboxSeq + 1, run, command, event });
|
|
}
|
|
this.beforeEventCommit(event);
|
|
return { run, event, outbox };
|
|
}
|
|
|
|
private commitPreparedEvent(prepared: { run: RunRecord; event: RunEvent; outbox: KafkaEventOutboxRecord | null }): void {
|
|
const { run, event, outbox } = prepared;
|
|
const events = this.eventsByRun.get(run.id) ?? [];
|
|
events.push(event);
|
|
this.eventsByRun.set(run.id, events);
|
|
if (outbox) {
|
|
this.kafkaOutboxSeq = outbox.outboxSeq;
|
|
this.kafkaEventOutbox.set(outbox.id, outbox);
|
|
}
|
|
this.touchSessionForRun(this.getRun(run.id), { lastEventSeq: event.seq, lastActivityAt: event.createdAt }, { bumpVersion: false, at: event.createdAt });
|
|
}
|
|
|
|
finishRun(runId: string, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">): RunRecord {
|
|
const existing = this.getRun(runId);
|
|
if (isTerminalRunStatus(existing.status)) {
|
|
this.cancelRunnerDispatchIntentsForRun(runId, "run is terminal");
|
|
const released = this.releaseTerminalRunClaim(existing);
|
|
if (released.status === "cancelled" && result.terminalStatus !== "cancelled") this.appendLateWriteRejected(runId, null, result, "run-status");
|
|
return released;
|
|
}
|
|
const status = statusFromTerminal(result.terminalStatus);
|
|
const next = this.updateRun(runId, { status, terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: result.failureMessage, claimedBy: null, leaseExpiresAt: null });
|
|
this.cancelRunnerDispatchIntentsForRun(runId, `run terminalized as ${status}`, next.updatedAt);
|
|
if (result.threadId && next.sessionRef?.sessionId) this.upsertSessionThread(next, result.threadId, result.turnId ?? null);
|
|
this.appendEvent(runId, "terminal_status", { terminalStatus: result.terminalStatus, failureKind: result.failureKind, message: result.failureMessage });
|
|
this.touchSessionForRun(this.getRun(runId), { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: runId, terminalStatus: result.terminalStatus, failureKind: result.failureKind, lastActivityAt: next.updatedAt }, { bumpVersion: true, at: next.updatedAt });
|
|
return next;
|
|
}
|
|
|
|
cancelRun(runId: string, reason = "cancel requested"): RunRecord {
|
|
const run = this.getRun(runId);
|
|
if (isTerminalRunStatus(run.status)) {
|
|
this.cancelRunnerDispatchIntentsForRun(runId, reason);
|
|
return this.releaseTerminalRunClaim(run);
|
|
}
|
|
const at = nowIso();
|
|
const cancel = this.createCancelRequest({ targetKind: "run", targetId: runId, run, command: null, reason, at, stage: "accepted" });
|
|
this.appendCancelStage(runId, "accepted", cancel);
|
|
const persistedRun = this.updateRun(runId, { cancelEpoch: cancel.epoch, cancelRequestId: cancel.id, cancelRequestedAt: at, cancelReason: reason });
|
|
this.appendCancelStage(runId, "persisted", cancel);
|
|
const leaseExpired = Boolean(persistedRun.claimedBy && isLeaseExpired(persistedRun.leaseExpiresAt));
|
|
if (leaseExpired) this.appendCancelStage(runId, "fenced", cancel, { claimedBy: persistedRun.claimedBy, leaseExpiresAt: persistedRun.leaseExpiresAt });
|
|
else if (persistedRun.claimedBy) {
|
|
this.appendCancelStage(runId, "delivered", cancel, { claimedBy: persistedRun.claimedBy });
|
|
this.appendCancelStage(runId, "aborting", cancel, { claimedBy: persistedRun.claimedBy });
|
|
}
|
|
for (const command of Array.from(this.commands.values()).filter((item) => item.runId === runId && !isTerminalCommandState(item.state))) {
|
|
const cancelled = { ...command, state: "cancelled" as const, cancelEpoch: cancel.epoch, cancelRequestId: cancel.id, cancelRequestedAt: at, cancelReason: reason, updatedAt: at };
|
|
this.commands.set(command.id, cancelled);
|
|
this.appendEvent(runId, "backend_status", { phase: "command-terminal", commandId: command.id, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
|
}
|
|
this.cancelRunnerDispatchIntentsForRun(runId, reason, at);
|
|
const next = this.updateRun(runId, { status: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", failureMessage: reason, claimedBy: null, leaseExpiresAt: null });
|
|
this.appendEvent(runId, "terminal_status", { terminalStatus: "cancelled", failureKind: "cancelled", message: reason, cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
|
this.appendCancelStage(runId, "terminalized", cancel);
|
|
this.touchSessionForRun(next, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: runId, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: next.updatedAt }, { bumpVersion: true, at: next.updatedAt });
|
|
return next;
|
|
}
|
|
|
|
cancelCommand(commandId: string, reason = "cancel requested"): CommandRecord {
|
|
const command = this.getCommand(commandId);
|
|
const run = this.getRun(command.runId);
|
|
if (isTerminalRunStatus(run.status)) this.releaseTerminalRunClaim(run);
|
|
if (isTerminalCommandState(command.state)) {
|
|
this.cancelRunnerDispatchIntentForCommand(commandId, reason);
|
|
return command;
|
|
}
|
|
const at = nowIso();
|
|
const cancel = this.createCancelRequest({ targetKind: "command", targetId: commandId, run, command, reason, at, stage: "accepted" });
|
|
this.appendCancelStage(command.runId, "accepted", cancel);
|
|
const next = { ...command, state: "cancelled" as const, cancelEpoch: cancel.epoch, cancelRequestId: cancel.id, cancelRequestedAt: at, cancelReason: reason, updatedAt: at };
|
|
this.commands.set(commandId, next);
|
|
this.cancelRunnerDispatchIntentForCommand(commandId, reason, at);
|
|
this.appendCancelStage(command.runId, "persisted", cancel);
|
|
const leaseExpired = Boolean(run.claimedBy && isLeaseExpired(run.leaseExpiresAt));
|
|
if (leaseExpired) this.appendCancelStage(command.runId, "fenced", cancel, { claimedBy: run.claimedBy, leaseExpiresAt: run.leaseExpiresAt });
|
|
else if (run.claimedBy || command.state === "acknowledged") {
|
|
this.appendCancelStage(command.runId, "delivered", cancel, { claimedBy: run.claimedBy, commandState: command.state });
|
|
this.appendCancelStage(command.runId, "aborting", cancel, { claimedBy: run.claimedBy, commandState: command.state });
|
|
}
|
|
this.appendEvent(command.runId, "backend_status", { phase: "command-terminal", commandId, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", message: reason, cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
|
this.appendCancelStage(command.runId, "terminalized", cancel);
|
|
if (command.type === "turn") this.touchSessionForRun(run, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: command.runId, lastCommandId: command.id, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: next.updatedAt }, { bumpVersion: true, at: next.updatedAt });
|
|
return next;
|
|
}
|
|
|
|
getSession(sessionId: string): SessionRecord | null {
|
|
return this.sessions.get(sessionId) ?? null;
|
|
}
|
|
|
|
getSessionSummary(sessionId: string, readerId: string | null = null): SessionSummary {
|
|
const session = this.getSession(sessionId);
|
|
if (!session) throw new AgentRunError("schema-invalid", `session ${sessionId} was not found`, { httpStatus: 404 });
|
|
return buildSessionSummary(session, readerId, readerId ? this.sessionReadCursors.get(sessionReadKey(sessionId, readerId)) ?? null : null);
|
|
}
|
|
|
|
listSessions(input: ListSessionsInput): SessionListResult {
|
|
const startVersion = parseSessionCursor(input.cursor) ?? 0;
|
|
const state = input.state ?? "default";
|
|
const items = Array.from(this.sessions.values())
|
|
.map((session) => buildSessionSummary(session, input.readerId ?? null, input.readerId ? this.sessionReadCursors.get(sessionReadKey(session.sessionId, input.readerId)) ?? null : null))
|
|
.filter((session) => session.version > startVersion)
|
|
.filter((session) => !input.backendProfile || session.backendProfile === input.backendProfile)
|
|
.filter((session) => sessionMatchesListState(session, state))
|
|
.sort(sessionSort)
|
|
.slice(0, clampSessionLimit(input.limit));
|
|
return { items, count: items.length, cursor: items.length > 0 ? String(items[items.length - 1]?.version ?? startVersion) : null, filters: sessionListFilters(input) };
|
|
}
|
|
|
|
listSessionTrace(sessionId: string, input: SessionEventPageInput): SessionEventPage {
|
|
const runId = this.resolveSessionRunId(sessionId, input.runId ?? null);
|
|
if (!runId) return { sessionId, runId: null, items: [], count: 0, cursor: null };
|
|
const items = this.listEvents(runId, input.afterSeq, input.limit);
|
|
return { sessionId, runId, items, count: items.length, cursor: items.length > 0 ? String(items[items.length - 1]?.seq ?? input.afterSeq) : null };
|
|
}
|
|
|
|
listSessionOutput(sessionId: string, input: SessionEventPageInput): SessionEventPage {
|
|
const page = this.listSessionTrace(sessionId, input);
|
|
const items = page.items.filter(isSessionOutputEvent);
|
|
return { ...page, items, count: items.length, cursor: items.length > 0 ? String(items[items.length - 1]?.seq ?? input.afterSeq) : null };
|
|
}
|
|
|
|
markSessionRead(sessionId: string, readerId: string): SessionReadCursorRecord {
|
|
const session = this.getSession(sessionId);
|
|
if (!session) throw new AgentRunError("schema-invalid", `session ${sessionId} was not found`, { httpStatus: 404 });
|
|
const record: SessionReadCursorRecord = { sessionId, readerId, sessionVersion: session.version, readAt: nowIso() };
|
|
this.sessionReadCursors.set(sessionReadKey(sessionId, readerId), record);
|
|
return record;
|
|
}
|
|
|
|
upsertSession(input: UpsertSessionInput): SessionRecord {
|
|
const at = nowIso();
|
|
const existing = this.sessions.get(input.sessionId);
|
|
if (existing) {
|
|
const next: SessionRecord = {
|
|
...existing,
|
|
tenantId: input.tenantId,
|
|
projectId: input.projectId,
|
|
backendProfile: input.backendProfile,
|
|
conversationId: input.conversationId,
|
|
threadId: input.threadId,
|
|
metadata: input.metadata,
|
|
expiresAt: input.expiresAt,
|
|
codexRolloutSubdir: input.codexRolloutSubdir,
|
|
version: existing.version + 1,
|
|
updatedAt: at,
|
|
};
|
|
this.sessions.set(input.sessionId, next);
|
|
this.sessionVersion = Math.max(this.sessionVersion, next.version) + 1;
|
|
return next;
|
|
}
|
|
const next: SessionRecord = {
|
|
sessionId: input.sessionId,
|
|
tenantId: input.tenantId,
|
|
projectId: input.projectId,
|
|
backendProfile: input.backendProfile,
|
|
conversationId: input.conversationId,
|
|
threadId: input.threadId,
|
|
metadata: input.metadata,
|
|
version: 1,
|
|
executionState: "idle",
|
|
lastRunId: null,
|
|
lastCommandId: null,
|
|
activeRunId: null,
|
|
activeCommandId: null,
|
|
lastEventSeq: 0,
|
|
terminalStatus: null,
|
|
failureKind: null,
|
|
title: null,
|
|
summary: {},
|
|
lastActivityAt: null,
|
|
createdAt: at,
|
|
updatedAt: at,
|
|
expiresAt: input.expiresAt,
|
|
storageKind: "none",
|
|
codexRolloutSubdir: input.codexRolloutSubdir,
|
|
};
|
|
this.sessions.set(input.sessionId, next);
|
|
this.sessionVersion = Math.max(this.sessionVersion, next.version) + 1;
|
|
return next;
|
|
}
|
|
|
|
refreshSessionStorage(input: SessionStoragePatch): SessionRecord {
|
|
const session = this.sessions.get(input.sessionId);
|
|
if (!session) throw new AgentRunError("schema-invalid", `session ${input.sessionId} was not found`, { httpStatus: 404 });
|
|
const at = nowIso();
|
|
const next: SessionRecord = {
|
|
...session,
|
|
storageKind: input.storageKind,
|
|
storagePvcName: input.pvcName ?? null,
|
|
storageNamespace: input.storageNamespace ?? null,
|
|
storagePvcPhase: input.pvcPhase ?? null,
|
|
storageSizeBytes: input.storageSizeBytes ?? null,
|
|
storageFilesCount: input.storageFilesCount ?? null,
|
|
storageSha256: input.storageSha256 ?? null,
|
|
codexRolloutSubdir: input.codexRolloutSubdir ?? session.codexRolloutSubdir ?? "sessions",
|
|
storageUpdatedAt: at,
|
|
version: session.version + 1,
|
|
updatedAt: at,
|
|
};
|
|
this.sessions.set(input.sessionId, next);
|
|
this.sessionVersion = Math.max(this.sessionVersion, next.version) + 1;
|
|
return next;
|
|
}
|
|
|
|
markSessionStorageEvicted(input: { sessionId: string; pvcName: string }): SessionRecord {
|
|
const session = this.sessions.get(input.sessionId);
|
|
if (!session) throw new AgentRunError("schema-invalid", `session ${input.sessionId} was not found`, { httpStatus: 404 });
|
|
const at = nowIso();
|
|
const next: SessionRecord = {
|
|
...session,
|
|
storageKind: "evicted",
|
|
storagePvcName: input.pvcName,
|
|
storageEvictedAt: at,
|
|
storageUpdatedAt: at,
|
|
version: session.version + 1,
|
|
updatedAt: at,
|
|
};
|
|
this.sessions.set(input.sessionId, next);
|
|
this.sessionVersion = Math.max(this.sessionVersion, next.version) + 1;
|
|
return next;
|
|
}
|
|
|
|
listGcExpiredSessions(input: ListGcExpiredSessionsInput): SessionRecord[] {
|
|
const now = input.now;
|
|
return Array.from(this.sessions.values())
|
|
.filter((session) => session.storageKind === "pvc" && Boolean(session.storagePvcName))
|
|
.filter((session) => session.expiresAt !== null && Date.parse(session.expiresAt ?? "") <= now)
|
|
.sort((a, b) => a.updatedAt.localeCompare(b.updatedAt))
|
|
.slice(0, Math.max(1, input.limit));
|
|
}
|
|
|
|
createQueueTask(input: CreateQueueTaskInput): QueueTaskRecord {
|
|
const payloadHash = queueTaskPayloadHash(input);
|
|
if (input.idempotencyKey) {
|
|
const existing = Array.from(this.queueTasks.values()).find((task) => task.tenantId === input.tenantId && task.projectId === input.projectId && task.idempotencyKey === input.idempotencyKey);
|
|
if (existing) {
|
|
if (existing.payloadHash !== payloadHash) throw new AgentRunError("schema-invalid", "queue task idempotency key reused with different payload", { httpStatus: 409 });
|
|
return existing;
|
|
}
|
|
}
|
|
const at = nowIso();
|
|
const sessionId = input.sessionRef?.sessionId ?? null;
|
|
const sessionPath = sessionId ? `/api/v1/sessions/${encodeURIComponent(sessionId)}` : null;
|
|
const task: QueueTaskRecord = { ...input, id: newId("qt"), state: "pending", version: this.nextQueueVersion(), payloadHash, latestAttempt: null, sessionPath, createdAt: at, updatedAt: at, cancelledAt: null, cancelReason: null };
|
|
this.queueTasks.set(task.id, task);
|
|
return task;
|
|
}
|
|
|
|
listQueueTasks(input: ListQueueTasksInput): QueueTaskListResult {
|
|
const startVersion = parseQueueCursor(input.cursor) ?? input.updatedAfter ?? 0;
|
|
const items = Array.from(this.queueTasks.values())
|
|
.filter((task) => task.version > startVersion)
|
|
.filter((task) => !input.queue || task.queue === input.queue)
|
|
.filter((task) => !input.state || task.state === input.state)
|
|
.sort(queueTaskSort)
|
|
.slice(0, clampQueueLimit(input.limit));
|
|
return { items, count: items.length, cursor: items.length > 0 ? String(items[items.length - 1]?.version ?? startVersion) : null };
|
|
}
|
|
|
|
getQueueTask(taskId: string): QueueTaskRecord {
|
|
const task = this.queueTasks.get(taskId);
|
|
if (!task) throw new AgentRunError("schema-invalid", `queue task ${taskId} was not found`, { httpStatus: 404 });
|
|
return task;
|
|
}
|
|
|
|
listQueueAttempts(taskId: string, input: ListQueueAttemptsInput): QueueAttemptListResult {
|
|
this.getQueueTask(taskId);
|
|
const cursor = input.cursor ?? Number.POSITIVE_INFINITY;
|
|
const items = Array.from(this.queueAttempts.values())
|
|
.filter((attempt) => attempt.taskId === taskId && attempt.retryIndex < cursor)
|
|
.sort((a, b) => b.retryIndex - a.retryIndex || b.createdAt.localeCompare(a.createdAt))
|
|
.slice(0, clampQueueLimit(input.limit));
|
|
return { taskId, items, count: items.length, cursor: items.length === clampQueueLimit(input.limit) ? String(items.at(-1)?.retryIndex ?? "") : null };
|
|
}
|
|
|
|
reserveQueueRetryAttempt(taskId: string, input: ReserveQueueRetryAttemptInput): QueueRetryReservation {
|
|
const task = this.getQueueTask(taskId);
|
|
assertQueueTaskPayloadHash(task);
|
|
const attempts = this.queueAttemptsForTask(taskId);
|
|
const existing = attempts.find((attempt) => attempt.idempotencyKey === input.idempotencyKey) ?? null;
|
|
if (existing) {
|
|
if (existing.reason !== input.reason) throw new AgentRunError("schema-invalid", "queue retry idempotency key reused with different reason", { httpStatus: 409 });
|
|
return queueRetryReservation(task, existing, attempts, false, true);
|
|
}
|
|
assertQueueRetryable(task, attempts);
|
|
const retryIndex = nextQueueRetryIndex(attempts);
|
|
if (input.dryRun) return queueRetryReservation(task, null, attempts, false, false, retryIndex);
|
|
const at = nowIso();
|
|
const attempt: QueueAttemptRecord = {
|
|
attemptId: newId("qa"),
|
|
taskId,
|
|
retryIndex,
|
|
state: "reserved",
|
|
idempotencyKey: input.idempotencyKey,
|
|
reason: input.reason,
|
|
payloadHash: task.payloadHash,
|
|
previousAttemptId: latestQueueAttempt(attempts)?.attemptId ?? task.latestAttempt?.attemptId ?? null,
|
|
runId: newId("run"),
|
|
commandId: null,
|
|
runnerJobId: newId("rjob"),
|
|
sessionId: task.sessionRef?.sessionId ?? null,
|
|
sessionPath: task.sessionPath,
|
|
failureKind: null,
|
|
failureMessage: null,
|
|
createdAt: at,
|
|
updatedAt: at,
|
|
activatedAt: null,
|
|
terminalAt: null,
|
|
};
|
|
this.queueAttempts.set(queueAttemptKey(taskId, attempt.attemptId), attempt);
|
|
return queueRetryReservation(task, attempt, attempts, true, false);
|
|
}
|
|
|
|
recordQueueRetryAttemptFailure(taskId: string, attemptId: string, input: RecordQueueRetryAttemptFailureInput): QueueAttemptRecord {
|
|
this.getQueueTask(taskId);
|
|
const attemptKey = queueAttemptKey(taskId, attemptId);
|
|
const attempt = this.queueAttempts.get(attemptKey);
|
|
if (!attempt) throw new AgentRunError("schema-invalid", `queue attempt ${attemptId} was not found for task ${taskId}`, { httpStatus: 404 });
|
|
if (attempt.state !== "reserved") return attempt;
|
|
const next: QueueAttemptRecord = { ...attempt, failureKind: input.failureKind, failureMessage: input.failureMessage, updatedAt: nowIso() };
|
|
this.queueAttempts.set(attemptKey, next);
|
|
return next;
|
|
}
|
|
|
|
activateQueueRetryAttempt(taskId: string, attemptId: string, input: ActivateQueueRetryAttemptInput): QueueRetryActivation {
|
|
const task = this.getQueueTask(taskId);
|
|
const attempt = this.queueAttempts.get(queueAttemptKey(taskId, attemptId));
|
|
if (!attempt) throw new AgentRunError("schema-invalid", `queue attempt ${attemptId} was not found for task ${taskId}`, { httpStatus: 404 });
|
|
if (attempt.state !== "reserved") {
|
|
assertQueueAttemptActivationReplay(attempt, input);
|
|
return { task, attempt, activated: false };
|
|
}
|
|
if (task.state !== "failed" && task.state !== "blocked") throw queueRetryStateError(task);
|
|
const at = nowIso();
|
|
const nextAttempt: QueueAttemptRecord = {
|
|
...attempt,
|
|
state: input.state,
|
|
commandId: input.commandId,
|
|
sessionId: input.sessionId,
|
|
sessionPath: input.sessionPath,
|
|
failureKind: input.failureKind,
|
|
failureMessage: input.failureMessage,
|
|
updatedAt: at,
|
|
activatedAt: at,
|
|
terminalAt: isTerminalQueueTaskState(input.state) ? at : null,
|
|
};
|
|
const latestAttempt = queueAttemptRef(nextAttempt);
|
|
const nextTask: QueueTaskRecord = { ...task, state: input.state, latestAttempt, sessionPath: input.sessionPath, version: this.nextQueueVersion(), updatedAt: at };
|
|
this.queueAttempts.set(queueAttemptKey(taskId, attemptId), nextAttempt);
|
|
this.queueTasks.set(taskId, nextTask);
|
|
return { task: nextTask, attempt: nextAttempt, activated: true };
|
|
}
|
|
|
|
updateQueueTaskAttempt(taskId: string, input: UpdateQueueTaskAttemptInput): QueueTaskRecord {
|
|
const task = this.getQueueTask(taskId);
|
|
if (isTerminalQueueTaskState(task.state)) throw new AgentRunError(task.state === "cancelled" ? "cancelled" : "schema-invalid", `queue task ${taskId} is already terminal: ${task.state}`, { httpStatus: 409 });
|
|
const next: QueueTaskRecord = { ...task, state: input.state, latestAttempt: input.latestAttempt, sessionPath: input.sessionPath, version: this.nextQueueVersion(), updatedAt: nowIso() };
|
|
this.upsertQueueAttemptProjection(task, input.latestAttempt, next.updatedAt);
|
|
this.queueTasks.set(taskId, next);
|
|
return next;
|
|
}
|
|
|
|
cancelQueueTask(taskId: string, reason = "cancel requested"): QueueTaskRecord {
|
|
const task = this.getQueueTask(taskId);
|
|
if (isTerminalQueueTaskState(task.state)) return task;
|
|
if (task.latestAttempt?.runId) this.cancelRun(task.latestAttempt.runId, reason);
|
|
else if (task.latestAttempt?.commandId) this.cancelCommand(task.latestAttempt.commandId, reason);
|
|
const at = nowIso();
|
|
const latestAttempt = task.latestAttempt ? { ...task.latestAttempt, state: "cancelled" as const, failureKind: "cancelled" as const, failureMessage: reason } : null;
|
|
const next: QueueTaskRecord = { ...task, state: "cancelled", latestAttempt, version: this.nextQueueVersion(), updatedAt: at, cancelledAt: at, cancelReason: reason };
|
|
if (latestAttempt) this.upsertQueueAttemptProjection(task, latestAttempt, at);
|
|
this.queueTasks.set(taskId, next);
|
|
return next;
|
|
}
|
|
|
|
markQueueTaskRead(taskId: string, readerId: string): QueueReadCursorRecord {
|
|
const task = this.getQueueTask(taskId);
|
|
const record: QueueReadCursorRecord = { taskId, readerId, taskVersion: task.version, readAt: nowIso() };
|
|
this.queueReadCursors.set(queueReadKey(taskId, readerId), record);
|
|
return record;
|
|
}
|
|
|
|
queueStats(queue?: string): QueueStats {
|
|
return buildQueueStats(Array.from(this.queueTasks.values()).filter((task) => !queue || task.queue === queue), queue ?? null);
|
|
}
|
|
|
|
queueCommander(queue?: string, readerId: string | null = null): QueueCommanderSnapshot {
|
|
const tasks = Array.from(this.queueTasks.values()).filter((task) => !queue || task.queue === queue);
|
|
const generatedAt = nowIso();
|
|
const items = tasks
|
|
.map((task) => buildQueueTaskSummary(task, readerId, readerId ? this.queueReadCursors.get(queueReadKey(task.id, readerId)) ?? null : null))
|
|
.filter((task) => queueTaskMatchesCommander(task, readerId))
|
|
.sort(queueTaskSort)
|
|
.slice(0, 20);
|
|
return { queue: queue ?? null, readerId, stats: buildQueueStats(tasks, queue ?? null, generatedAt), items, generatedAt };
|
|
}
|
|
|
|
private queueAttemptsForTask(taskId: string): QueueAttemptRecord[] {
|
|
return Array.from(this.queueAttempts.values()).filter((attempt) => attempt.taskId === taskId).sort((a, b) => a.retryIndex - b.retryIndex || a.createdAt.localeCompare(b.createdAt));
|
|
}
|
|
|
|
private upsertQueueAttemptProjection(task: QueueTaskRecord, ref: QueueAttemptRef, at: string): void {
|
|
const attemptKey = queueAttemptKey(task.id, ref.attemptId);
|
|
const existing = this.queueAttempts.get(attemptKey);
|
|
if (existing) {
|
|
if (isTerminalQueueAttemptState(existing.state)) {
|
|
if (stableHash(queueAttemptRef(existing)) !== stableHash(ref)) throw new AgentRunError("schema-invalid", `queue attempt ${ref.attemptId} is terminal and immutable`, { httpStatus: 409 });
|
|
return;
|
|
}
|
|
const next: QueueAttemptRecord = {
|
|
...existing,
|
|
...ref,
|
|
state: ref.state,
|
|
failureKind: ref.failureKind ?? null,
|
|
failureMessage: ref.failureMessage ?? null,
|
|
updatedAt: at,
|
|
activatedAt: existing.activatedAt ?? at,
|
|
terminalAt: isTerminalQueueTaskState(ref.state) ? at : null,
|
|
};
|
|
this.queueAttempts.set(attemptKey, next);
|
|
return;
|
|
}
|
|
const attempts = this.queueAttemptsForTask(task.id);
|
|
const record: QueueAttemptRecord = {
|
|
attemptId: ref.attemptId,
|
|
taskId: task.id,
|
|
retryIndex: nextQueueRetryIndex(attempts),
|
|
state: ref.state,
|
|
idempotencyKey: null,
|
|
reason: null,
|
|
payloadHash: task.payloadHash,
|
|
previousAttemptId: latestQueueAttempt(attempts)?.attemptId ?? null,
|
|
runId: ref.runId,
|
|
commandId: ref.commandId,
|
|
runnerJobId: ref.runnerJobId,
|
|
sessionId: ref.sessionId,
|
|
sessionPath: ref.sessionPath,
|
|
failureKind: ref.failureKind ?? null,
|
|
failureMessage: ref.failureMessage ?? null,
|
|
createdAt: at,
|
|
updatedAt: at,
|
|
activatedAt: at,
|
|
terminalAt: isTerminalQueueTaskState(ref.state) ? at : null,
|
|
};
|
|
this.queueAttempts.set(attemptKey, record);
|
|
}
|
|
|
|
backends(): JsonRecord[] {
|
|
return backendCapabilities();
|
|
}
|
|
|
|
private updateRun(runId: string, patch: Partial<RunRecord>): RunRecord {
|
|
const run = this.getRun(runId);
|
|
const next = { ...run, ...patch, updatedAt: nowIso() };
|
|
this.runs.set(runId, next);
|
|
return next;
|
|
}
|
|
|
|
private releaseTerminalRunClaim(run: RunRecord): RunRecord {
|
|
if (!isTerminalRunStatus(run.status)) return run;
|
|
return this.releaseRunnerClaim(run);
|
|
}
|
|
|
|
private releaseRunnerClaim(run: RunRecord): RunRecord {
|
|
if (run.claimedBy === null && run.leaseExpiresAt === null) return run;
|
|
return this.updateRun(run.id, { claimedBy: null, leaseExpiresAt: null });
|
|
}
|
|
|
|
private terminalizeRunnerRetention(run: RunRecord, commands: CommandRecord[], reason: string): void {
|
|
const at = nowIso();
|
|
for (const command of commands.filter((item) => !isTerminalCommandState(item.state))) {
|
|
this.commands.set(command.id, { ...command, state: "failed", updatedAt: at });
|
|
this.cancelRunnerDispatchIntentForCommand(command.id, reason, at);
|
|
const prepared = this.prepareEvent(run.id, "backend_status", { phase: "command-terminal", commandId: command.id, state: "failed", terminalStatus: "failed", failureKind: "infra-failed", message: reason, retentionFence: true });
|
|
this.commitPreparedEvent(prepared);
|
|
}
|
|
const next = this.updateRun(run.id, { status: "failed", terminalStatus: "failed", failureKind: "infra-failed", failureMessage: reason, claimedBy: null, leaseExpiresAt: null });
|
|
this.cancelRunnerDispatchIntentsForRun(run.id, reason, at);
|
|
this.commitPreparedEvent(this.prepareEvent(run.id, "terminal_status", { terminalStatus: "failed", failureKind: "infra-failed", message: reason, retentionFence: true }));
|
|
this.touchSessionForRun(next, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: run.id, terminalStatus: "failed", failureKind: "infra-failed", lastActivityAt: at }, { bumpVersion: true, at });
|
|
}
|
|
|
|
private requireRunnerDispatchIntent(intentId: string): RunnerDispatchIntentRecord {
|
|
const intent = this.runnerDispatchIntents.get(intentId);
|
|
if (!intent) throw new AgentRunError("schema-invalid", `runner dispatch intent ${intentId} was not found`, { httpStatus: 404 });
|
|
return intent;
|
|
}
|
|
|
|
private fenceRunnerDispatchClaim(intent: RunnerDispatchIntentRecord, claim: DurableQueueClaim): void {
|
|
assertMemoryClaim(intent, claim, "runner dispatch", intent.state === "dispatching");
|
|
const run = this.getRun(intent.runId);
|
|
const command = this.getCommand(intent.commandId);
|
|
if (!isTerminalRunStatus(run.status) && !isTerminalCommandState(command.state)) return;
|
|
this.cancelRunnerDispatchIntent(intent, "run or command became terminal during dispatch");
|
|
throw staleClaimError("runner dispatch", claim);
|
|
}
|
|
|
|
private cancelRunnerDispatchIntentForCommand(commandId: string, reason: string, at = nowIso()): void {
|
|
const intent = this.getRunnerDispatchIntent(commandId);
|
|
if (intent) this.cancelRunnerDispatchIntent(intent, reason, at);
|
|
}
|
|
|
|
private cancelRunnerDispatchIntentsForRun(runId: string, reason: string, at = nowIso()): void {
|
|
for (const intent of this.runnerDispatchIntents.values()) if (intent.runId === runId) this.cancelRunnerDispatchIntent(intent, reason, at);
|
|
}
|
|
|
|
private cancelRunnerDispatchIntent(intent: RunnerDispatchIntentRecord, reason: string, at = nowIso()): void {
|
|
if (!isActiveRunnerDispatchIntent(intent)) return;
|
|
this.runnerDispatchIntents.set(intent.id, { ...intent, state: "cancelled", leaseOwner: null, leaseExpiresAt: null, lastError: { code: "cancelled", message: reason, valuesPrinted: false }, updatedAt: at });
|
|
}
|
|
|
|
private requireKafkaEventOutbox(outboxId: string): KafkaEventOutboxRecord {
|
|
const item = this.kafkaEventOutbox.get(outboxId);
|
|
if (!item) throw new AgentRunError("schema-invalid", `Kafka event outbox ${outboxId} was not found`, { httpStatus: 404 });
|
|
return item;
|
|
}
|
|
|
|
private createCancelRequest(input: { targetKind: CancelTargetKind; targetId: string; run: RunRecord; command: CommandRecord | null; reason: string; at: string; stage: CancelStage }): CancelRequestRecord {
|
|
const sessionId = input.run.sessionRef?.sessionId ?? null;
|
|
const epoch = input.command ? input.command.cancelEpoch + 1 : input.run.cancelEpoch + 1;
|
|
const record: CancelRequestRecord = {
|
|
id: newId("cancel"),
|
|
targetKind: input.targetKind,
|
|
targetId: input.targetId,
|
|
runId: input.run.id,
|
|
commandId: input.command?.id ?? null,
|
|
sessionId,
|
|
taskId: null,
|
|
reason: input.reason,
|
|
requestedBy: null,
|
|
epoch,
|
|
stage: input.stage,
|
|
metadata: {},
|
|
createdAt: input.at,
|
|
updatedAt: input.at,
|
|
};
|
|
this.cancelRequests.set(record.id, record);
|
|
return record;
|
|
}
|
|
|
|
private appendCancelStage(runId: string, stage: CancelStage, cancel: CancelRequestRecord, extra: JsonRecord = {}): void {
|
|
const next: CancelRequestRecord = { ...cancel, stage, updatedAt: nowIso() };
|
|
this.cancelRequests.set(cancel.id, next);
|
|
this.appendEvent(runId, "backend_status", cancelStagePayload(next, stage, extra));
|
|
}
|
|
|
|
private appendLateWriteRejected(runId: string, command: CommandRecord | null, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">, source: string): void {
|
|
this.appendEvent(runId, "backend_status", lateWriteRejectedPayload(this.getRun(runId), command, { source, terminalStatus: result.terminalStatus, failureKind: result.failureKind, message: result.failureMessage ?? null, threadId: result.threadId ?? null, turnId: result.turnId ?? null }));
|
|
}
|
|
|
|
private nextQueueVersion(): number {
|
|
this.queueVersion += 1;
|
|
return this.queueVersion;
|
|
}
|
|
|
|
private resolveSessionForRun(input: CreateRunInput, at: string): SessionRef | null {
|
|
if (!input.sessionRef) return null;
|
|
const existing = this.sessions.get(input.sessionRef.sessionId);
|
|
if (existing) {
|
|
assertSessionBoundary(existing, input);
|
|
return sessionRefFromRecord(existing, input.sessionRef);
|
|
}
|
|
const record: SessionRecord = {
|
|
sessionId: input.sessionRef.sessionId,
|
|
tenantId: input.tenantId,
|
|
projectId: input.projectId,
|
|
backendProfile: input.backendProfile,
|
|
conversationId: input.sessionRef.conversationId ?? null,
|
|
threadId: input.sessionRef.threadId ?? null,
|
|
metadata: input.sessionRef.metadata ?? {},
|
|
version: this.nextSessionVersion(),
|
|
executionState: "idle",
|
|
lastRunId: null,
|
|
lastCommandId: null,
|
|
activeRunId: null,
|
|
activeCommandId: null,
|
|
lastEventSeq: 0,
|
|
terminalStatus: null,
|
|
failureKind: null,
|
|
title: titleFromMetadata(input.sessionRef.metadata ?? {}),
|
|
summary: {},
|
|
lastActivityAt: at,
|
|
createdAt: at,
|
|
updatedAt: at,
|
|
expiresAt: input.sessionRef.expiresAt ?? null,
|
|
};
|
|
this.sessions.set(record.sessionId, record);
|
|
return sessionRefFromRecord(record, input.sessionRef);
|
|
}
|
|
|
|
private upsertSessionThread(run: RunRecord, threadId: string, turnId: string | null): void {
|
|
if (!run.sessionRef?.sessionId) return;
|
|
const at = nowIso();
|
|
const existing = this.sessions.get(run.sessionRef.sessionId);
|
|
const record: SessionRecord = {
|
|
sessionId: run.sessionRef.sessionId,
|
|
tenantId: run.tenantId,
|
|
projectId: run.projectId,
|
|
backendProfile: run.backendProfile,
|
|
conversationId: run.sessionRef.conversationId ?? existing?.conversationId ?? null,
|
|
threadId,
|
|
metadata: { ...(existing?.metadata ?? {}), ...(run.sessionRef.metadata ?? {}), ...(turnId ? { lastTurnId: turnId } : {}) },
|
|
version: this.nextSessionVersion(),
|
|
executionState: existing?.executionState ?? "idle",
|
|
lastRunId: existing?.lastRunId ?? run.id,
|
|
lastCommandId: existing?.lastCommandId ?? null,
|
|
activeRunId: existing?.activeRunId ?? null,
|
|
activeCommandId: existing?.activeCommandId ?? null,
|
|
lastEventSeq: existing?.lastEventSeq ?? 0,
|
|
terminalStatus: existing?.terminalStatus ?? null,
|
|
failureKind: existing?.failureKind ?? null,
|
|
title: existing?.title ?? titleFromMetadata(run.sessionRef.metadata ?? {}),
|
|
summary: existing?.summary ?? {},
|
|
lastActivityAt: at,
|
|
createdAt: existing?.createdAt ?? at,
|
|
updatedAt: at,
|
|
expiresAt: run.sessionRef.expiresAt ?? existing?.expiresAt ?? null,
|
|
};
|
|
this.sessions.set(record.sessionId, record);
|
|
const nextSessionRef = sessionRefFromRecord(record, run.sessionRef);
|
|
this.updateRun(run.id, { sessionRef: nextSessionRef });
|
|
this.appendEvent(run.id, "backend_status", { phase: "session-updated", sessionRef: summarizeSessionRef(nextSessionRef), turnId });
|
|
}
|
|
|
|
private touchSessionForRun(run: RunRecord, patch: Partial<SessionRecord>, options: { bumpVersion: boolean; at?: string }): void {
|
|
const sessionId = run.sessionRef?.sessionId;
|
|
if (!sessionId) return;
|
|
const existing = this.sessions.get(sessionId);
|
|
if (!existing) return;
|
|
const at = options.at ?? nowIso();
|
|
const next: SessionRecord = { ...existing, ...patch, version: options.bumpVersion ? this.nextSessionVersion() : existing.version, updatedAt: at, lastActivityAt: patch.lastActivityAt ?? at };
|
|
this.sessions.set(sessionId, next);
|
|
}
|
|
|
|
private resolveSessionRunId(sessionId: string, requestedRunId: string | null): string | null {
|
|
const session = this.getSession(sessionId);
|
|
if (!session) throw new AgentRunError("schema-invalid", `session ${sessionId} was not found`, { httpStatus: 404 });
|
|
if (requestedRunId) {
|
|
const run = this.getRun(requestedRunId);
|
|
if (run.sessionRef?.sessionId !== sessionId) throw new AgentRunError("schema-invalid", `run ${requestedRunId} does not belong to session ${sessionId}`, { httpStatus: 404 });
|
|
return requestedRunId;
|
|
}
|
|
return session.activeRunId ?? session.lastRunId;
|
|
}
|
|
|
|
private nextSessionVersion(): number {
|
|
this.sessionVersion += 1;
|
|
return this.sessionVersion;
|
|
}
|
|
}
|
|
|
|
export function assertSessionBoundary(existing: SessionRecord, input: CreateRunInput): void {
|
|
if (existing.tenantId !== input.tenantId || existing.projectId !== input.projectId) {
|
|
throw new AgentRunError("tenant-policy-denied", "sessionRef cannot be reused across tenant or project boundary", { httpStatus: 403, details: { sessionId: existing.sessionId, valuesPrinted: false } });
|
|
}
|
|
// backendProfile is run-scoped, not session/PVC-scoped. A HWLAB session must be
|
|
// able to switch providers without losing its session storage or workspace.
|
|
}
|
|
|
|
export function statusFromTerminal(terminalStatus: TerminalStatus): RunRecord["status"] {
|
|
if (terminalStatus === "completed") return "completed";
|
|
if (terminalStatus === "cancelled") return "cancelled";
|
|
if (terminalStatus === "blocked") return "blocked";
|
|
return "failed";
|
|
}
|
|
|
|
export function commandStateFromTerminal(terminalStatus: TerminalStatus): CommandRecord["state"] {
|
|
if (terminalStatus === "completed") return "completed";
|
|
if (terminalStatus === "cancelled") return "cancelled";
|
|
return "failed";
|
|
}
|
|
|
|
export function isTerminalRunStatus(status: RunRecord["status"]): boolean {
|
|
return status === "completed" || status === "failed" || status === "blocked" || status === "cancelled";
|
|
}
|
|
|
|
export function isTerminalCommandState(state: CommandRecord["state"]): boolean {
|
|
return state === "completed" || state === "failed" || state === "cancelled";
|
|
}
|
|
|
|
export function cancelStagePayload(cancel: CancelRequestRecord, stage: CancelStage, extra: JsonRecord = {}): JsonRecord {
|
|
return {
|
|
phase: cancelPhase(stage),
|
|
cancelStage: stage,
|
|
cancelRequestId: cancel.id,
|
|
targetKind: cancel.targetKind,
|
|
targetId: cancel.targetId,
|
|
runId: cancel.runId,
|
|
commandId: cancel.commandId,
|
|
sessionId: cancel.sessionId,
|
|
taskId: cancel.taskId,
|
|
reason: cancel.reason,
|
|
requestedBy: cancel.requestedBy,
|
|
cancelEpoch: cancel.epoch,
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
export function fenceLateEventForCancelledRun(run: RunRecord, type: RunEvent["type"], payload: JsonRecord): { type: RunEvent["type"]; payload: JsonRecord } {
|
|
if (run.status !== "cancelled" || isAllowedAfterCancel(type, payload)) return { type, payload };
|
|
return { type: "backend_status", payload: lateWriteRejectedPayload(run, null, { eventType: type, payload }) };
|
|
}
|
|
|
|
export function lateWriteRejectedPayload(run: RunRecord, command: CommandRecord | null, details: JsonRecord): JsonRecord {
|
|
return {
|
|
phase: "late-write-rejected",
|
|
cancelStage: "late-write-rejected",
|
|
cancelRequestId: command?.cancelRequestId ?? run.cancelRequestId,
|
|
cancelEpoch: command?.cancelEpoch ?? run.cancelEpoch,
|
|
runId: run.id,
|
|
commandId: command?.id ?? null,
|
|
reason: command?.cancelReason ?? run.cancelReason,
|
|
rejected: redactJson(details),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function cancelPhase(stage: CancelStage): string {
|
|
return stage === "late-write-rejected" ? "late-write-rejected" : `cancel-${stage}`;
|
|
}
|
|
|
|
function isAllowedAfterCancel(type: RunEvent["type"], payload: JsonRecord): boolean {
|
|
if (type === "terminal_status" && payload.terminalStatus === "cancelled") return true;
|
|
const phase = typeof payload.phase === "string" ? payload.phase : "";
|
|
if (phase.startsWith("cancel-") || phase === "late-write-rejected" || phase === "turn-cancelled") return true;
|
|
if (phase === "command-terminal" && payload.failureKind === "cancelled") return true;
|
|
return false;
|
|
}
|
|
|
|
export function isTerminalQueueTaskState(state: QueueTaskState): boolean {
|
|
return state === "completed" || state === "failed" || state === "blocked" || state === "cancelled";
|
|
}
|
|
|
|
export function isLeaseExpired(leaseExpiresAt: string | null): boolean {
|
|
if (!leaseExpiresAt) return true;
|
|
return new Date(leaseExpiresAt).getTime() <= Date.now();
|
|
}
|
|
|
|
export function sessionRefFromRecord(record: SessionRecord, fallback: SessionRef): SessionRef {
|
|
return {
|
|
sessionId: record.sessionId,
|
|
...(record.conversationId ? { conversationId: record.conversationId } : fallback.conversationId ? { conversationId: fallback.conversationId } : {}),
|
|
...(record.threadId ? { threadId: record.threadId } : fallback.threadId ? { threadId: fallback.threadId } : {}),
|
|
...(record.expiresAt ? { expiresAt: record.expiresAt } : fallback.expiresAt ? { expiresAt: fallback.expiresAt } : {}),
|
|
...(Object.keys(record.metadata).length > 0 ? { metadata: record.metadata } : fallback.metadata ? { metadata: fallback.metadata } : {}),
|
|
};
|
|
}
|
|
|
|
export function buildSessionSummary(record: SessionRecord, readerId: string | null, readCursor: SessionReadCursorRecord | null): SessionSummary {
|
|
const active = record.executionState === "running" || record.activeRunId !== null || record.activeCommandId !== null;
|
|
const unread = !active && (!readCursor || readCursor.sessionVersion < record.version);
|
|
const attentionState = active ? "active" : unread ? "unread" : "read";
|
|
return { ...record, sessionPath: `${record.tenantId}/${record.projectId}/${record.sessionId}`, readerId, readCursor, attentionState, unread, active };
|
|
}
|
|
|
|
export function buildQueueTaskSummary(record: QueueTaskRecord, readerId: string | null, readCursor: QueueReadCursorRecord | null): QueueTaskSummary {
|
|
const active = !isTerminalQueueTaskState(record.state);
|
|
const unread = !active && (!readCursor || readCursor.taskVersion < record.version);
|
|
const attentionState = active ? "active" : unread ? "unread" : "read";
|
|
return { ...record, readerId, readCursor, attentionState, unread, active };
|
|
}
|
|
|
|
export function queueTaskMatchesCommander(task: QueueTaskSummary, readerId: string | null): boolean {
|
|
if (!readerId) return true;
|
|
return task.active || task.unread;
|
|
}
|
|
|
|
export function sessionMatchesListState(session: SessionSummary, state: SessionListState): boolean {
|
|
if (state === "all") return true;
|
|
if (state === "default") return session.active || session.unread;
|
|
if (state === "running") return session.active;
|
|
if (state === "unread") return session.unread;
|
|
if (state === "terminal") return session.executionState === "terminal";
|
|
if (state === "idle") return session.executionState === "idle";
|
|
return false;
|
|
}
|
|
|
|
export function sessionSort(a: SessionSummary, b: SessionSummary): number {
|
|
if (a.active !== b.active) return a.active ? -1 : 1;
|
|
if (a.unread !== b.unread) return a.unread ? -1 : 1;
|
|
return (b.lastActivityAt ?? b.updatedAt).localeCompare(a.lastActivityAt ?? a.updatedAt) || b.updatedAt.localeCompare(a.updatedAt) || a.sessionId.localeCompare(b.sessionId);
|
|
}
|
|
|
|
export function clampSessionLimit(limit: number): number {
|
|
return Math.max(1, Math.min(Number.isFinite(limit) ? Math.trunc(limit) : 50, 100));
|
|
}
|
|
|
|
export function parseSessionCursor(cursor: string | undefined): number | null {
|
|
if (!cursor) return null;
|
|
const value = Number(cursor);
|
|
return Number.isInteger(value) && value >= 0 ? value : null;
|
|
}
|
|
|
|
export function sessionListFilters(input: ListSessionsInput): JsonRecord {
|
|
return { state: input.state ?? "default", backendProfile: input.backendProfile ?? null, readerId: input.readerId ?? null, cursor: input.cursor ?? null, limit: clampSessionLimit(input.limit) };
|
|
}
|
|
|
|
export function summarizeSessionRef(sessionRef: SessionRef | null): JsonRecord | null {
|
|
if (!sessionRef) return null;
|
|
return {
|
|
sessionId: sessionRef.sessionId,
|
|
conversationId: sessionRef.conversationId ?? null,
|
|
threadId: sessionRef.threadId ?? null,
|
|
expiresAt: sessionRef.expiresAt ?? null,
|
|
metadataKeys: Object.keys(sessionRef.metadata ?? {}).sort(),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function runCreatedEventPayload(run: RunRecord): JsonRecord {
|
|
return {
|
|
phase: "run-created",
|
|
backendProfile: run.backendProfile,
|
|
sessionRef: summarizeSessionRef(run.sessionRef ?? null),
|
|
resourceBundleRef: summarizeResourceBundleRef(run.resourceBundleRef ?? null),
|
|
};
|
|
}
|
|
|
|
export function summarizeResourceBundleRef(resourceBundleRef: RunRecord["resourceBundleRef"] | null | undefined): JsonRecord | null {
|
|
if (!resourceBundleRef) return null;
|
|
return {
|
|
kind: resourceBundleRef.kind,
|
|
repoUrl: resourceBundleRef.repoUrl,
|
|
commitId: resourceBundleRef.commitId ?? null,
|
|
ref: resourceBundleRef.ref ?? null,
|
|
bundles: {
|
|
count: resourceBundleRef.bundles.length,
|
|
items: resourceBundleRef.bundles.map((item) => ({ name: item.name ?? null, repoUrl: item.repoUrl ?? resourceBundleRef.repoUrl, commitId: item.commitId ?? resourceBundleRef.commitId ?? null, ref: item.ref ?? resourceBundleRef.ref ?? null, subpath: item.subpath, targetPath: item.targetPath, valuesPrinted: false })),
|
|
valuesPrinted: false,
|
|
},
|
|
promptRefs: resourceBundleRef.promptRefs ? { count: resourceBundleRef.promptRefs.length, names: resourceBundleRef.promptRefs.map((item) => item.name), required: resourceBundleRef.promptRefs.filter((item) => item.required === true).map((item) => item.name), valuesPrinted: false } : { count: 0, names: [], required: [], valuesPrinted: false },
|
|
requiredSkills: resourceBundleRef.requiredSkills ? { count: resourceBundleRef.requiredSkills.length, names: resourceBundleRef.requiredSkills.map((item) => item.name), valuesPrinted: false } : { count: 0, names: [], valuesPrinted: false },
|
|
submodules: resourceBundleRef.submodules ?? false,
|
|
lfs: resourceBundleRef.lfs ?? false,
|
|
credentialRef: resourceBundleRef.credentialRef ? { name: resourceBundleRef.credentialRef.name, namespace: resourceBundleRef.credentialRef.namespace ?? null, keys: resourceBundleRef.credentialRef.keys ?? [], valuesPrinted: false } : null,
|
|
};
|
|
}
|
|
|
|
export function queueTaskSort(a: QueueTaskRecord, b: QueueTaskRecord): number {
|
|
if (a.priority !== b.priority) return b.priority - a.priority;
|
|
return a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id);
|
|
}
|
|
|
|
export function queueTaskPayloadHash(input: CreateQueueTaskInput): string {
|
|
return stableHash({
|
|
tenantId: input.tenantId,
|
|
projectId: input.projectId,
|
|
queue: input.queue,
|
|
lane: input.lane,
|
|
title: input.title,
|
|
priority: input.priority,
|
|
backendProfile: input.backendProfile,
|
|
providerId: input.providerId,
|
|
workspaceRef: input.workspaceRef,
|
|
sessionRef: input.sessionRef,
|
|
executionPolicy: input.executionPolicy,
|
|
resourceBundleRef: input.resourceBundleRef,
|
|
payload: input.payload,
|
|
references: input.references,
|
|
metadata: input.metadata,
|
|
});
|
|
}
|
|
|
|
export function assertRunCreateReplay(existing: RunRecord, input: CreateRunInput): void {
|
|
const existingContract = {
|
|
tenantId: existing.tenantId,
|
|
projectId: existing.projectId,
|
|
workspaceRef: existing.workspaceRef,
|
|
sessionId: existing.sessionRef?.sessionId ?? null,
|
|
resourceBundleRef: existing.resourceBundleRef,
|
|
providerId: existing.providerId,
|
|
backendProfile: existing.backendProfile,
|
|
executionPolicy: existing.executionPolicy,
|
|
traceSink: existing.traceSink,
|
|
};
|
|
const inputContract = {
|
|
tenantId: input.tenantId,
|
|
projectId: input.projectId,
|
|
workspaceRef: input.workspaceRef,
|
|
sessionId: input.sessionRef?.sessionId ?? null,
|
|
resourceBundleRef: input.resourceBundleRef ?? null,
|
|
providerId: input.providerId,
|
|
backendProfile: input.backendProfile,
|
|
executionPolicy: input.executionPolicy,
|
|
traceSink: input.traceSink,
|
|
};
|
|
if (stableHash(existingContract) !== stableHash(inputContract)) {
|
|
throw new AgentRunError("schema-invalid", `run identity ${existing.id} reused with a different creation contract`, { httpStatus: 409 });
|
|
}
|
|
}
|
|
|
|
export function assertSessionTurnAdmissionContract(input: SessionTurnAdmissionInput): void {
|
|
if (!input.sessionId.trim()) throw new AgentRunError("schema-invalid", "session turn admission requires a non-empty sessionId", { httpStatus: 400 });
|
|
if (input.run.sessionRef?.sessionId !== input.sessionId) {
|
|
throw new AgentRunError("schema-invalid", "session turn admission run must reference the target session", {
|
|
httpStatus: 400,
|
|
details: { sessionId: input.sessionId, runSessionId: input.run.sessionRef?.sessionId ?? null, valuesPrinted: false },
|
|
});
|
|
}
|
|
if (input.command.type !== "turn") throw new AgentRunError("schema-invalid", "session turn admission only accepts turn commands", { httpStatus: 400 });
|
|
}
|
|
|
|
export function assertSessionCommandReplay(existing: CommandRecord, input: CreateCommandInput): void {
|
|
if (existing.type !== input.type || existing.payloadHash !== stableHash(input.payload) || (existing.idempotencyKey ?? null) !== (input.idempotencyKey ?? null)) {
|
|
throw new AgentRunError("schema-invalid", "session turn admission replay does not match the existing command", {
|
|
httpStatus: 409,
|
|
details: { runId: existing.runId, commandId: existing.id, reason: "session-turn-admission-replay-conflict", valuesPrinted: false },
|
|
});
|
|
}
|
|
}
|
|
|
|
export function activeSessionAdmissionConflict(sessionId: string, run: RunRecord, command: CommandRecord): AgentRunError {
|
|
return new AgentRunError("runner-lease-conflict", `session ${sessionId} already has an active turn admission`, {
|
|
httpStatus: 409,
|
|
details: {
|
|
reason: "session-turn-admission-active",
|
|
sessionId,
|
|
runId: run.id,
|
|
commandId: command.id,
|
|
runStatus: run.status,
|
|
commandState: command.state,
|
|
valuesPrinted: false,
|
|
},
|
|
});
|
|
}
|
|
|
|
export function invalidSessionAdmissionProjection(sessionId: string): AgentRunError {
|
|
return new AgentRunError("infra-failed", `session ${sessionId} active admission projection is inconsistent`, {
|
|
httpStatus: 503,
|
|
details: { reason: "session-turn-admission-projection-inconsistent", sessionId, valuesPrinted: false },
|
|
});
|
|
}
|
|
|
|
export function assertQueueTaskPayloadHash(task: QueueTaskRecord): void {
|
|
if (queueTaskPayloadHash(task) !== task.payloadHash) throw new AgentRunError("schema-invalid", `queue task ${task.id} payloadHash does not match its immutable contract`, { httpStatus: 409 });
|
|
}
|
|
|
|
export function assertQueueRetryable(task: QueueTaskRecord, attempts: QueueAttemptRecord[]): void {
|
|
if (task.state !== "failed" && task.state !== "blocked") throw queueRetryStateError(task);
|
|
if (!task.workspaceRef) throw new AgentRunError("schema-invalid", "queue retry requires the original workspaceRef", { httpStatus: 400 });
|
|
if (!task.providerId) throw new AgentRunError("schema-invalid", "queue retry requires the original providerId", { httpStatus: 400 });
|
|
if (!task.executionPolicy) throw new AgentRunError("schema-invalid", "queue retry requires the original executionPolicy", { httpStatus: 400 });
|
|
const active = attempts.find((attempt) => attempt.state === "reserved" || attempt.state === "running");
|
|
if (active) throw new AgentRunError("schema-invalid", `queue task ${task.id} already has an active retry attempt: ${active.attemptId}`, { httpStatus: 409, details: { attemptId: active.attemptId, state: active.state, valuesPrinted: false } });
|
|
}
|
|
|
|
export function queueRetryStateError(task: QueueTaskRecord): AgentRunError {
|
|
const failureKind = task.state === "cancelled" ? "cancelled" : "schema-invalid";
|
|
return new AgentRunError(failureKind, `queue task ${task.id} cannot be retried from state ${task.state}`, { httpStatus: 409, details: { allowedStates: ["failed", "blocked"], state: task.state, valuesPrinted: false } });
|
|
}
|
|
|
|
export function queueRetryReservation(task: QueueTaskRecord, attempt: QueueAttemptRecord | null, attempts: QueueAttemptRecord[], mutation: boolean, idempotentReplay: boolean, plannedRetryIndex?: number): QueueRetryReservation {
|
|
const retryIndex = attempt?.retryIndex ?? plannedRetryIndex ?? nextQueueRetryIndex(attempts);
|
|
const previousAttempt = attempt?.previousAttemptId
|
|
? attempts.find((item) => item.attemptId === attempt.previousAttemptId) ?? null
|
|
: latestQueueAttempt(attempts);
|
|
return {
|
|
action: "queue-retry-reservation",
|
|
mutation,
|
|
idempotentReplay,
|
|
task,
|
|
attempt,
|
|
previousAttempt,
|
|
retryIndex,
|
|
allowedTransition: {
|
|
from: task.state,
|
|
to: attempt && attempt.state !== "reserved" ? attempt.state : "running",
|
|
retryIndex,
|
|
previousAttemptId: attempt?.previousAttemptId ?? previousAttempt?.attemptId ?? task.latestAttempt?.attemptId ?? null,
|
|
payloadHash: task.payloadHash,
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function latestQueueAttempt(attempts: QueueAttemptRecord[]): QueueAttemptRecord | null {
|
|
return attempts.reduce<QueueAttemptRecord | null>((latest, attempt) => !latest || attempt.retryIndex > latest.retryIndex ? attempt : latest, null);
|
|
}
|
|
|
|
export function nextQueueRetryIndex(attempts: QueueAttemptRecord[]): number {
|
|
return (latestQueueAttempt(attempts)?.retryIndex ?? -1) + 1;
|
|
}
|
|
|
|
export function queueAttemptRef(attempt: QueueAttemptRecord): QueueAttemptRef {
|
|
if (attempt.state === "reserved" || attempt.state === "pending") throw new AgentRunError("schema-invalid", `queue attempt ${attempt.attemptId} is not active`, { httpStatus: 409 });
|
|
return {
|
|
attemptId: attempt.attemptId,
|
|
state: attempt.state,
|
|
runId: attempt.runId,
|
|
commandId: attempt.commandId,
|
|
runnerJobId: attempt.runnerJobId,
|
|
sessionId: attempt.sessionId,
|
|
sessionPath: attempt.sessionPath,
|
|
failureKind: attempt.failureKind,
|
|
failureMessage: attempt.failureMessage,
|
|
};
|
|
}
|
|
|
|
export function assertQueueAttemptActivationReplay(attempt: QueueAttemptRecord, input: ActivateQueueRetryAttemptInput): void {
|
|
const expected = {
|
|
state: input.state,
|
|
commandId: input.commandId,
|
|
sessionId: input.sessionId,
|
|
sessionPath: input.sessionPath,
|
|
failureKind: input.failureKind,
|
|
failureMessage: input.failureMessage,
|
|
};
|
|
const actual = {
|
|
state: attempt.state,
|
|
commandId: attempt.commandId,
|
|
sessionId: attempt.sessionId,
|
|
sessionPath: attempt.sessionPath,
|
|
failureKind: attempt.failureKind,
|
|
failureMessage: attempt.failureMessage,
|
|
};
|
|
if (stableHash(actual) !== stableHash(expected)) throw new AgentRunError("schema-invalid", `queue attempt ${attempt.attemptId} activation replay does not match its immutable execution identity`, { httpStatus: 409 });
|
|
}
|
|
|
|
export function isTerminalQueueAttemptState(state: QueueAttemptRecord["state"]): boolean {
|
|
return state === "completed" || state === "failed" || state === "blocked" || state === "cancelled";
|
|
}
|
|
|
|
export function buildQueueStats(tasks: QueueTaskRecord[], queue: string | null, generatedAt = nowIso()): QueueStats {
|
|
const byState: Record<string, number> = {};
|
|
const byLane: Record<string, number> = {};
|
|
let maxVersion = 0;
|
|
for (const task of tasks) {
|
|
byState[task.state] = (byState[task.state] ?? 0) + 1;
|
|
byLane[task.lane] = (byLane[task.lane] ?? 0) + 1;
|
|
maxVersion = Math.max(maxVersion, task.version);
|
|
}
|
|
return { queue, total: tasks.length, byState, byLane, maxVersion, generatedAt };
|
|
}
|
|
|
|
export function clampQueueLimit(limit: number): number {
|
|
return Math.max(1, Math.min(Number.isFinite(limit) ? Math.trunc(limit) : 50, 100));
|
|
}
|
|
|
|
export function parseQueueCursor(cursor: string | undefined): number | null {
|
|
if (!cursor) return null;
|
|
const value = Number(cursor);
|
|
return Number.isInteger(value) && value >= 0 ? value : null;
|
|
}
|
|
|
|
function queueReadKey(taskId: string, readerId: string): string {
|
|
return `${taskId}:${readerId}`;
|
|
}
|
|
|
|
function queueAttemptKey(taskId: string, attemptId: string): string {
|
|
return `${taskId}:${attemptId}`;
|
|
}
|
|
|
|
export function sessionReadKey(sessionId: string, readerId: string): string {
|
|
return `${sessionId}:${readerId}`;
|
|
}
|
|
|
|
export function titleFromMetadata(metadata: JsonRecord): string | null {
|
|
const title = metadata.title;
|
|
return typeof title === "string" && title.trim().length > 0 ? title.trim().slice(0, 200) : null;
|
|
}
|
|
|
|
export function sessionTitleFromCommand(command: CommandRecord): string | null {
|
|
const value = command.payload.prompt;
|
|
if (typeof value !== "string") return null;
|
|
return value.trim().replace(/\s+/gu, " ").slice(0, 120) || null;
|
|
}
|
|
|
|
function isRunCreatedEvent(event: RunEvent): boolean {
|
|
return event.type === "backend_status" && event.payload.phase === "run-created";
|
|
}
|
|
|
|
function eventMatchesCommand(event: RunEvent, commandId: string): boolean {
|
|
const payload = event.payload;
|
|
return payload.commandId === commandId || payload.targetCommandId === commandId || (event.type === "terminal_status" && payload.commandId === undefined);
|
|
}
|
|
|
|
function commandForEvent(commands: Map<string, CommandRecord>, event: RunEvent): CommandRecord | null {
|
|
for (const candidate of [event.payload.commandId, event.payload.targetCommandId]) {
|
|
if (typeof candidate === "string") {
|
|
const command = commands.get(candidate);
|
|
if (command) return command;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function assertMemoryClaim(record: DurableQueueClaim & { leaseExpiresAt?: string | null }, claim: DurableQueueClaim, component: string, stateMatches: boolean): void {
|
|
const leaseExpiresAt = Date.parse(record.leaseExpiresAt ?? "");
|
|
if (!stateMatches || !claim.leaseOwner || record.leaseOwner !== claim.leaseOwner || record.attemptCount !== claim.attemptCount || !Number.isFinite(leaseExpiresAt) || leaseExpiresAt <= Date.now()) {
|
|
throw staleClaimError(component, claim);
|
|
}
|
|
}
|
|
|
|
function staleClaimError(component: string, claim: DurableQueueClaim): AgentRunError {
|
|
return new AgentRunError("runner-lease-conflict", `${component} claim is stale`, { httpStatus: 409, details: { id: claim.id, attemptCount: claim.attemptCount, valuesPrinted: false } });
|
|
}
|
|
|
|
function runnerRetentionFenceConflict(runId: string): AgentRunError {
|
|
return new AgentRunError("runner-lease-conflict", `run ${runId} is fenced by runner retention`, {
|
|
httpStatus: 409,
|
|
details: { reason: "runner-retention-db-fenced", runId, valuesPrinted: false },
|
|
});
|
|
}
|
|
|
|
function isActiveRunnerDispatchIntent(intent: RunnerDispatchIntentRecord): boolean {
|
|
return intent.state === "pending" || intent.state === "dispatching" || intent.state === "retry";
|
|
}
|
|
|
|
function workQueueStatus<T extends JsonRecord>(items: T[], component: string, completed: (item: T) => boolean = (item) => ["dispatched", "failed", "cancelled"].includes(String(item.state ?? ""))): JsonRecord {
|
|
const pending = items.filter((item) => !completed(item));
|
|
const retrying = pending.filter((item) => Number(item.attemptCount ?? 0) > 0);
|
|
const oldest = pending.slice().sort((a, b) => String(a.createdAt ?? "").localeCompare(String(b.createdAt ?? "")))[0];
|
|
return {
|
|
component,
|
|
totalCount: items.length,
|
|
backlogCount: pending.length,
|
|
retryingCount: retrying.length,
|
|
oldestPendingAt: typeof oldest?.createdAt === "string" ? oldest.createdAt : null,
|
|
maxAttemptCount: items.reduce((max, item) => Math.max(max, Number(item.attemptCount ?? 0)), 0),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function isSessionOutputEvent(event: RunEvent): boolean {
|
|
return event.type === "assistant_progress" || event.type === "assistant_message" || event.type === "command_output" || event.type === "diff" || event.type === "error" || event.type === "terminal_status";
|
|
}
|