diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index e50f0188..37e70282 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -67,6 +67,7 @@ import { } from "./code-agent-agentrun-prompt.ts"; import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import * as agentRunResult from "./code-agent-agentrun-result.ts"; const ADAPTER_ID = "agentrun-v01"; const ADAPTER_IDS = new Set([ADAPTER_ID, "agentrun-v02"]); @@ -91,6 +92,68 @@ const HWLAB_RESOURCE_GIT_BUNDLES = Object.freeze([ const HWLAB_RESOURCE_PROMPT_REFS = Object.freeze([ Object.freeze({ name: "hwlab-v02-runtime", path: "internal/agent/prompts/hwlab-v02-runtime.md", inject: "thread-start", required: true }) ]); +const { + isRetryableAgentRunTransportFailureKind, + agentRunSessionEvidence, + agentRunFailureRequiresFreshSession, + agentRunFreshSessionFailure, + providerCredentialSecretRef, + agentRunProviderTrace, + agentRunFailureAttribution, + agentRunLongLivedSessionGate, + agentRunToolCalls, + agentRunReplyText, + agentRunResultToCodeAgentPayload, + agentRunCompletedFinalResponse, + agentRunTraceTerminalAssistantText, + isAgentRunAssistantTraceEvent, + agentRunTerminalFailureFinalResponse, + agentRunTerminalFailureTraceSummary, + agentRunCompletedTraceSummary, + partialAgentRunContext, + agentRunResultTraceCreatedAt, + decorateAgentRunRunningResult, + appendAgentRunEventsToTrace, + isForeignAgentRunCommandEvent, + mapAgentRunEvent, + agentRunProviderRetryEvent, + providerRetryMessage, + isRetryableProviderInterruptionPayload, + normalizedFailureKind, + numberOrNull, + agentRunTraceEvent, + agentRunManualEventType, + isProtocolToolLifecycleMethod, + agentRunToolCallPreviewSummary, + jsonPreviewStringField, + agentRunBackendStatusDetails, + agentRunResourceSummary, + compactObject, + agentRunCommandExecutionEvent, + nowMs, + agentRunMapping, + agentRunReusedMapping, + agentRunResultRefs, + withAgentRunRunnerJobCount, + hwlabSessionIdForParams, + scopedAgentRunSessionIdForParams, + agentRunSessionId, + agentRunEffectiveThreadId, + agentRunSessionSummary, + agentRunThreadReused, + agentRunSessionReuseSummary, + agentRunRunnerSummary, + hostForUrl, + waitingForForPhase, + textPayload, + requiredString, + firstNonEmpty, + nowIso, + timestampIsoOrNow, + redactUrl +} = agentRunResult; + +export { agentRunSessionEvidence } from "./code-agent-agentrun-result.ts"; export function codeAgentAgentRunAdapterEnabled(env = process.env) { const value = String(env.HWLAB_CODE_AGENT_ADAPTER ?? env.HWLAB_CODE_AGENT_PROVIDER ?? "").trim().toLowerCase(); @@ -332,7 +395,6 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI const fetchImpl = options.fetchImpl ?? globalThis.fetch; const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); const startedAt = nowIso(options.now); - const ownerApiKey = await resolveOwnerApiKey({ params, options, now: () => nowIso(options.now) }); const toolCapabilities = await resolveToolCapabilities({ params, options }); const promptEvidence = agentRunPromptMetadataFromParams(params, traceId); @@ -341,7 +403,7 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI type: "request", status: "accepted", label: "agentrun:request:accepted", - message: "HWLAB Code Agent request accepted by the AgentRun v0.1 adapter; hwlab-cloud-api will reuse a warm session run when its runner lease is valid, otherwise create a fresh run and runner Job outside the browser admission path.", + message: "HWLAB Code Agent request accepted by the AgentRun adapter; a warm runner is reused when valid, otherwise AgentRun durably admits command dispatch before this request returns.", waitingFor: "agentrun-run-reuse-check", adapter: ADAPTER_ID, managerHost: new URL(managerUrl).hostname, @@ -388,7 +450,7 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI type: "backend", status: "running", label: "agentrun:run:reuse-command-failed", - message: `AgentRun reused run ${reusable.mapping.runId} rejected the new command; hwlab-cloud-api will create a fresh run/runner Job for this turn.`, + message: `AgentRun reused run ${reusable.mapping.runId} rejected the new command; hwlab-cloud-api will create a fresh run and ask AgentRun manager to admit its durable runner dispatch.`, errorCode: error?.code ?? "agentrun_reuse_command_failed", runId: reusable.mapping.runId, commandId: reusable.mapping.commandId ?? null, @@ -410,15 +472,12 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI waitingFor: "agentrun-result", valuesPrinted: false }, backendProfile)); - const runnerJob = null; - const mapping = agentRunReusedMapping({ previous: reusable.mapping, run: reusable.run, command, runnerJob, traceId, startedAt, backendProfile, managerUrl, env }); + const mapping = agentRunReusedMapping({ previous: reusable.mapping, run: reusable.run, command, traceId, startedAt, backendProfile, managerUrl, env }); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", - label: runnerJob ? "agentrun:runner-job:ensured" : "agentrun:runner-job:reused", - message: runnerJob - ? `AgentRun runner Job ${mapping.jobName ?? "unknown"} ensured for reused run ${mapping.runId}; this keeps the persistent session resumable after pod replacement.` - : `AgentRun runner Job ${mapping.jobName ?? "unknown"} remains active for this HWLAB session turn; no replacement Job was created.`, + label: "agentrun:runner-job:reused", + message: `AgentRun runner Job ${mapping.jobName ?? "unknown"} remains active for this HWLAB session turn; no replacement Job was created.`, runId: mapping.runId, commandId: mapping.commandId, attemptId: mapping.attemptId, @@ -501,107 +560,41 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI message: "AgentRun run " + runId + " created through internal k3s Service DNS.", runId, backendProfile, waitingFor: "agentrun-command-create", valuesPrinted: false, }, backendProfile)); - const commandInput = buildAgentRunCommandInput({ params: dispatchParams, traceId, backendProfile, sessionId }); + const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }); + const commandInput = buildAgentRunCommandInput({ params: dispatchParams, traceId, backendProfile, sessionId, dispatchInput }); const commandCreateStartedAt = Date.now(); command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create" }); const commandId = requiredString(command?.id, "command.id"); + const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent); void emitCodeAgentOtelSpan("agentrun_command_create", traceId, env, { startTimeMs: commandCreateStartedAt, attributes: { runId, commandId, backendProfile, managerUrl } }); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:command:created", - message: "AgentRun command " + commandId + " created; hwlab-cloud-api will start a runner Job explicitly without relying on scheduler automation.", - runId, commandId, backendProfile, waitingFor: "agentrun-runner-job-create", valuesPrinted: false, + message: "AgentRun command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.", + runId, commandId, dispatchIntentId: dispatchIntent.id, runnerJobId: dispatchIntent.runnerJobId, + backendProfile, waitingFor: "agentrun-result", valuesPrinted: false, }, backendProfile)); - const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities, backendProfile }); - mapping = agentRunMapping({ env, managerUrl, backendProfile, run, command, runnerJob: null, traceId, startedAt, params: dispatchParams }); + mapping = agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: dispatchParams }); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", - label: "agentrun:runner-job:queued", - message: "AgentRun run/command admission is durable; hwlab-cloud-api will create the runner Job asynchronously so the browser request is not blocked by Kubernetes Job creation.", + label: "agentrun:dispatch-intent:admitted", + message: "AgentRun owns durable runner dispatch; HWLAB has no process-local runner Job creation window.", runId: mapping.runId, commandId: mapping.commandId, backendProfile, - waitingFor: "agentrun-runner-job-create", + dispatchIntentId: dispatchIntent.id, + dispatchIntentState: dispatchIntent.state, + runnerJobId: dispatchIntent.runnerJobId, + waitingFor: "agentrun-result", valuesPrinted: false }, mapping)); - scheduleAgentRunRunnerJobCreateAfterAdmission({ fetchImpl, managerUrl, runnerJobInput, timeoutMs, env, traceStore, traceId, backendProfile, run, command, mapping, options, params: dispatchParams }); finalDispatchParams = dispatchParams; break; } return decorateAgentRunRunningResult({ base: initialAgentRunChatResult({ params: finalDispatchParams, options, traceId }), mapping, traceStore, traceId }); } -function scheduleAgentRunRunnerJobCreateAfterAdmission(args) { - setTimeout(() => { - void ensureAgentRunRunnerJobCreateAfterAdmission(args).catch((error) => { - markAgentRunRunnerJobCreateFailedAfterAdmission(args, error); - }); - }, 0); -} - -async function ensureAgentRunRunnerJobCreateAfterAdmission({ fetchImpl, managerUrl, runnerJobInput, timeoutMs, env = process.env, traceStore = defaultCodeAgentTraceStore, traceId, backendProfile, mapping = {}, options = {}, params = {} }) { - const runId = requiredString(mapping.runId, "mapping.runId"); - const commandId = requiredString(mapping.commandId, "mapping.commandId"); - const runnerJobStartedAt = Date.now(); - const runnerJob = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/runner-jobs", method: "POST", body: runnerJobInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, commandId, stage: "runner-job-create-async" }); - const nextMapping = agentRunMappingWithRunnerJob(mapping, runnerJob); - void emitCodeAgentOtelSpan("agentrun_runner_job_create_async", traceId, env, { startTimeMs: runnerJobStartedAt, attributes: { runId, commandId, backendProfile, managerUrl, jobName: nextMapping.jobName ?? null } }); - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:runner-job:created", - message: `AgentRun runner Job ${nextMapping.jobName ?? "unknown"} created in namespace ${nextMapping.namespace ?? DEFAULT_RUNNER_NAMESPACE}.`, - runId: nextMapping.runId, - commandId: nextMapping.commandId, - attemptId: nextMapping.attemptId, - runnerId: nextMapping.runnerId, - jobName: nextMapping.jobName, - namespace: nextMapping.namespace, - waitingFor: "agentrun-result", - valuesPrinted: false - }, nextMapping)); - const current = options.codeAgentChatResults?.get?.(traceId); - const base = current && typeof current === "object" ? current : initialAgentRunChatResult({ params, options, traceId }); - options.codeAgentChatResults?.set?.(traceId, decorateAgentRunRunningResult({ base: { ...base, agentRun: nextMapping }, mapping: nextMapping, traceStore, traceId })); -} - -function markAgentRunRunnerJobCreateFailedAfterAdmission({ traceStore = defaultCodeAgentTraceStore, traceId, backendProfile, mapping = {}, options = {}, params = {} }, error) { - const failureKind = agentRunDispatchFailureKind(error); - const retryAttempt = Number(error?.retryAttempt ?? 0); - const retryMax = Number(error?.retryMax ?? agentRunDispatchRetryPolicy().maxRetries); - const message = `AgentRun runner Job creation failed after durable run/command admission; Code Agent stopped this turn instead of silently waiting. failureKind=${failureKind}`; - traceStore.append(traceId, agentRunTraceEvent({ - type: "error", - eventType: "error", - status: "failed", - label: `agentrun:runner-job:create-async-failed:${failureKind}`, - errorCode: failureKind, - failureKind, - retryAttempt: Number.isFinite(retryAttempt) ? retryAttempt : 0, - retryMax: Number.isFinite(retryMax) ? retryMax : null, - retryExhausted: error?.retryExhausted === true, - runId: mapping.runId ?? null, - commandId: mapping.commandId ?? null, - waitingFor: "agentrun-runner-job-failed", - terminal: true, - message, - valuesPrinted: false - }, mapping || backendProfile)); - const current = options.codeAgentChatResults?.get?.(traceId); - const base = current && typeof current === "object" - ? current - : decorateAgentRunRunningResult({ base: initialAgentRunChatResult({ params, options, traceId }), mapping, traceStore, traceId }); - const failed = agentRunResultToCodeAgentPayload({ - base: { ...base, agentRun: { ...mapping, status: "runner-job-failed", commandState: "failed", terminalStatus: "failed", failureKind, valuesPrinted: false } }, - result: { terminalStatus: "failed", failureKind, failureMessage: message, runId: mapping.runId ?? null, commandId: mapping.commandId ?? null, valuesPrinted: false }, - traceStore, - traceId, - appendResultEvent: false - }); - options.codeAgentChatResults?.set?.(traceId, failed); -} - async function agentRunDispatchJson({ fetchImpl, managerUrl, path, method = "GET", body = undefined, timeoutMs, env = process.env, traceStore = defaultCodeAgentTraceStore, traceId, backendProfile, runId = null, commandId = null, stage = "agentrun-dispatch", terminalOnFailure = true } = {}) { const policy = agentRunDispatchRetryPolicy(env); let retryAttempt = 0; @@ -677,20 +670,6 @@ function agentRunDispatchRetryDelayMs(policy, retryAttempt) { return Math.min(cap, Math.trunc(base * Math.pow(2, Math.max(0, Number(retryAttempt ?? 1) - 1)))); } -const RETRYABLE_AGENTRUN_TRANSPORT_FAILURE_KINDS = new Set([ - "agentrun-connect-failed", - "agentrun-proxy-exec-failed", - "agentrun-manager-fetch-failed", - "agentrun-timeout", - "timeout", - "fetch-failed", - "network-error" -]); - -function isRetryableAgentRunTransportFailureKind(failureKind) { - return RETRYABLE_AGENTRUN_TRANSPORT_FAILURE_KINDS.has(normalizedFailureKind(failureKind)); -} - function agentRunDispatchRetryable(error, failureKind) { const status = Number(error?.statusCode ?? error?.status ?? error?.agentRunError?.statusCode ?? 0); if ([429, 502, 503, 504].includes(status)) return true; @@ -1261,36 +1240,6 @@ function agentRunCommandMatchesTrace(command, traceId) { return command?.idempotencyKey === traceId || payload.traceId === traceId; } -export function agentRunSessionEvidence(payload = {}) { - if (!payload?.agentRun) return {}; - return { - agentRun: { - adapter: ADAPTER_ID, - runId: payload.agentRun.runId ?? null, - commandId: payload.agentRun.commandId ?? null, - attemptId: payload.agentRun.attemptId ?? null, - runnerId: payload.agentRun.runnerId ?? null, - jobName: payload.agentRun.jobName ?? null, - namespace: payload.agentRun.namespace ?? null, - backendProfile: payload.agentRun.backendProfile ?? null, - managerUrl: payload.agentRun.managerUrl ?? DEFAULT_AGENTRUN_MGR_URL, - repoUrl: payload.agentRun.repoUrl ?? DEFAULT_AGENTRUN_REPO_URL, - status: payload.agentRun.status ?? null, - runStatus: payload.agentRun.runStatus ?? null, - commandState: payload.agentRun.commandState ?? null, - terminalStatus: payload.agentRun.terminalStatus ?? null, - traceId: payload.agentRun.traceId ?? payload.traceId ?? payload.providerTrace?.traceId ?? null, - lastSeq: payload.agentRun.lastSeq ?? 0, - providerId: payload.agentRun.providerId ?? agentRunProviderIdOrNull(), - reuseEligible: payload.agentRun.reuseEligible ?? false, - sessionId: payload.agentRun.sessionId ?? payload.sessionId ?? null, - conversationId: payload.conversationId ?? payload.agentRun.conversationId ?? null, - threadId: payload.threadId ?? payload.agentRun.threadId ?? null, - providerTrace: payload.providerTrace ?? payload.agentRun.providerTrace ?? null, - valuesPrinted: false - } - }; -} function withoutUserBillingReservation(value = {}) { if (!value || typeof value !== "object") return value; @@ -1384,11 +1333,6 @@ function agentRunFreshSessionRequested(params = {}) { return /^(1|true|yes|fresh)$/iu.test(String(value ?? "").trim()); } -function agentRunFailureRequiresFreshSession(failureKind, failureMessage) { - const kind = String(failureKind ?? "").trim().toLowerCase().replace(/_/gu, "-"); - const message = String(failureMessage ?? ""); - return kind === "session-store-evicted" || kind === "thread-resume-failed" || /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|thread\/resume failed/iu.test(message); -} function newSessionIdAfterEviction(baseSessionId, traceId) { const profile = String(traceId).replace(/[^A-Za-z0-9]/gu, "").slice(0, 12) || "fresh"; @@ -1456,7 +1400,6 @@ async function retryAgentRunTurnAfterFreshSessionFailure({ valuesPrinted: false }, failedMapping || mapped?.agentRun || backendProfile)); await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }); - const ownerApiKey = await resolveOwnerApiKey({ params: retryParams, options, now: () => nowIso(options.now) }); const toolCapabilities = await resolveToolCapabilities({ params: retryParams, options }); const runInput = buildAgentRunCreateRunInput({ params: retryParams, env, traceId, backendProfile, sessionId, toolCapabilities }); const run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create-after-session-reset" }); @@ -1471,55 +1414,51 @@ async function retryAgentRunTurnAfterFreshSessionFailure({ waitingFor: "agentrun-command-create", valuesPrinted: false }, backendProfile)); - const commandInput = buildAgentRunCommandInput({ params: retryParams, traceId, backendProfile, sessionId }); + const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }); + const commandInput = buildAgentRunCommandInput({ params: retryParams, traceId, backendProfile, sessionId, dispatchInput }); const command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create-after-session-reset" }); const commandId = requiredString(command?.id, "command.id"); + const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent); traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", label: "agentrun:command:created-after-session-reset", - message: "AgentRun fresh-session retry command " + commandId + " created for the current user turn.", + message: "AgentRun fresh-session retry command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.", runId, commandId, backendProfile, - waitingFor: "agentrun-runner-job-create", + dispatchIntentId: dispatchIntent.id, + runnerJobId: dispatchIntent.runnerJobId, + waitingFor: "agentrun-result", valuesPrinted: false }, backendProfile)); - const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities, backendProfile }); const mapping = { - ...agentRunMapping({ env, managerUrl, backendProfile, run, command, runnerJob: null, traceId, startedAt, params: retryParams }), + ...agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: retryParams }), freshSessionRetryAttempt: 1, freshSessionRetryOf: previous, - status: "runner-job-pending", + status: `dispatch-intent-${dispatchIntent.state}`, valuesPrinted: false }; traceStore.append(traceId, agentRunTraceEvent({ type: "backend", status: "running", - label: "agentrun:runner-job:queued-after-session-reset", - message: "AgentRun fresh-session retry run/command admission is durable; hwlab-cloud-api will create the runner Job asynchronously.", + label: "agentrun:dispatch-intent:admitted-after-session-reset", + message: "AgentRun owns the durable fresh-session runner dispatch; HWLAB has no process-local runner Job creation window.", runId: mapping.runId, commandId: mapping.commandId, backendProfile, - waitingFor: "agentrun-runner-job-create", + dispatchIntentId: dispatchIntent.id, + dispatchIntentState: dispatchIntent.state, + runnerJobId: dispatchIntent.runnerJobId, + waitingFor: "agentrun-result", valuesPrinted: false }, mapping)); - scheduleAgentRunRunnerJobCreateAfterAdmission({ fetchImpl, managerUrl, runnerJobInput, timeoutMs, env, traceStore, traceId, backendProfile, run, command, mapping, options, params: retryParams }); const base = initialAgentRunChatResult({ params: retryParams, options, traceId }); const payload = decorateAgentRunRunningResult({ base: { ...base, freshSessionRetryOf: previous }, mapping, traceStore, traceId }); return { payload, mapping }; } -function agentRunFreshSessionFailure(result = {}) { - const code = result?.failureKind ?? result?.errorCode ?? result?.code ?? (result?.terminalStatus === "failed" ? "agentrun_failed" : null); - const message = result?.failureMessage ?? result?.blocker?.message ?? result?.error?.message ?? result?.message ?? ""; - return { - code: normalizedFailureKind(code) ?? "agentrun_failed", - message, - requiresFreshSession: agentRunFailureRequiresFreshSession(code, message), - valuesPrinted: false - }; -} + function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId, toolCapabilities = null }) { const commitId = requireAgentRunSourceCommit(env); const runtimeAuthority = requireAgentRunRuntimeAuthority(env); @@ -1603,7 +1542,7 @@ function adapterError(code, message, details = {}) { }); } -function buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId }) { +function buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId, dispatchInput = null }) { const prompt = messageAuthorityTextValue(params.message ?? params.prompt ?? ""); const promptEvidence = agentRunPromptMetadataFromText(prompt, traceId, "agentrun-command-payload", { fullText: true }); const threadId = safeOpaqueId(params.threadId); @@ -1626,6 +1565,7 @@ function buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId source: "hwlab-cloud-api", valuesPrinted: false }, + ...(dispatchInput ? { dispatch: { kind: "kubernetes-job", input: dispatchInput } } : {}), idempotencyKey: traceId }; } @@ -1655,25 +1595,14 @@ function buildAgentRunSteerCommandInput({ params, traceId, steerTraceId, mapped }; } -async function resolveOwnerApiKey({ params, options, now }) { - const ownerUserId = text(params.ownerUserId); - if (!ownerUserId) return ""; - const accessController = options.accessController; - if (!accessController?.store?.findActiveDefaultApiKeyForUser) return ""; - const key = await accessController.store.findActiveDefaultApiKeyForUser(ownerUserId) ?? null; - if (!key) return ""; - return text(key.displaySecret ?? ""); -} -function buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities = null, backendProfile }) { +function buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }) { const runtimeAuthority = requireAgentRunRuntimeAuthority(env); const baseTransient = buildAgentRunTransientEnv(env, { providerProfile: backendProfile, parentTraceId: traceId }); assertAgentRunTransientRuntimeAuthority(baseTransient, runtimeAuthority); - const hwpodAllowed = toolCapabilityAllowed(toolCapabilities, "hwpod"); - const transientEnv = ownerApiKey && hwpodAllowed - ? baseTransient.concat([{ name: "HWLAB_API_KEY", value: ownerApiKey, sensitive: true }]) - : baseTransient; + if (baseTransient.some((entry) => entry?.sensitive === true)) { + throw adapterError("agentrun_durable_dispatch_secret_forbidden", "AgentRun durable dispatch input must not persist sensitive transient environment values."); + } return { - commandId, idempotencyKey: traceId, namespace: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE, env.AGENTRUN_RUNTIME_NAMESPACE, DEFAULT_RUNNER_NAMESPACE), ...(firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_IMAGE, env.AGENTRUN_RUNNER_IMAGE) @@ -1682,7 +1611,51 @@ function buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, too ...(firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_SERVICE_ACCOUNT, env.AGENTRUN_RUNNER_SERVICE_ACCOUNT) ? { serviceAccountName: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_SERVICE_ACCOUNT, env.AGENTRUN_RUNNER_SERVICE_ACCOUNT) } : {}), - ...(transientEnv.length > 0 ? { transientEnv } : {}) + ...(baseTransient.length > 0 ? { transientEnv: baseTransient } : {}) + }; +} + +function requireAgentRunDurableDispatchIntent(value) { + const intent = value && typeof value === "object" && !Array.isArray(value) ? value : null; + const admittedStates = new Set(["pending", "dispatching", "retry", "dispatched"]); + const terminalStates = new Set(["failed", "cancelled"]); + const id = typeof intent?.id === "string" ? intent.id.trim() : ""; + const state = typeof intent?.state === "string" ? intent.state.trim() : ""; + const runnerJobId = typeof intent?.runnerJobId === "string" ? intent.runnerJobId.trim() : ""; + const attemptCount = Number(intent?.attemptCount ?? 0); + if ( + !intent || + intent.durable !== true || + (!admittedStates.has(state) && !terminalStates.has(state)) || + !/^dispatch_[A-Za-z0-9_.:-]+$/u.test(id) || + !/^rjob_[A-Za-z0-9_.:-]+$/u.test(runnerJobId) || + !Object.hasOwn(intent, "attemptCount") || + !Number.isInteger(attemptCount) || + attemptCount < 0 + ) { + throw Object.assign(new Error("AgentRun command response did not confirm a durable dispatch intent."), { + code: "agentrun_dispatch_intent_unconfirmed", + statusCode: 502, + valuesPrinted: false + }); + } + if (terminalStates.has(state)) { + throw Object.assign(new Error(`AgentRun durable dispatch intent ${id} is already ${state}; the turn was not admitted as running.`), { + code: `agentrun_dispatch_intent_${state}`, + statusCode: state === "failed" ? 503 : 409, + dispatchIntentId: id, + runnerJobId, + attemptCount, + valuesPrinted: false + }); + } + return { + id, + state, + runnerJobId, + attemptCount, + durable: true, + valuesPrinted: false }; } @@ -1857,7 +1830,7 @@ async function resolveReusableAgentRun({ params = {}, options = {}, env = proces type: "backend", status: "running", label: "agentrun:run:reuse-owner-skipped", - message: "Stored AgentRun session is not visible to the current actor; a new runner Job will be created for this request.", + message: "Stored AgentRun session is not visible to the current actor; a fresh run with manager-owned durable dispatch will be admitted for this request.", sessionId: hwlabSessionId, waitingFor: "agentrun-run-create", valuesPrinted: false @@ -1870,7 +1843,7 @@ async function resolveReusableAgentRun({ params = {}, options = {}, env = proces type: "backend", status: "running", label: "agentrun:run:reuse-skipped", - message: "No reusable AgentRun run was found for this HWLAB session; a new runner Job will be created.", + message: "No reusable AgentRun run was found for this HWLAB session; a fresh run with manager-owned durable dispatch will be admitted.", sessionId: hwlabSessionId, waitingFor: "agentrun-run-create", valuesPrinted: false @@ -1989,1006 +1962,3 @@ function runLeaseExpired(leaseExpiresAt) { const expiresMs = Date.parse(String(leaseExpiresAt ?? "")); return !Number.isFinite(expiresMs) || expiresMs <= Date.now(); } - -function providerCredentialSecretRef(profile, env) { - const profileEnvKey = String(profile ?? "").toUpperCase().replace(/[^A-Z0-9]+/gu, "_"); - return { - profile, - secretRef: { - namespace: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE, DEFAULT_RUNNER_NAMESPACE), - name: firstNonEmpty(env[`HWLAB_CODE_AGENT_AGENTRUN_${profileEnvKey}_SECRET_NAME`], env[`HWLAB_CODE_AGENT_AGENTRUN_${String(profile ?? "").toUpperCase()}_SECRET_NAME`], env.HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_SECRET_NAME, `agentrun-v01-provider-${profile}`), - keys: ["auth.json", "config.toml"], - mountPath: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_CODEX_HOME_MOUNT, "~/.codex") - } - }; -} - -function agentRunProviderTrace({ base = {}, result = {}, terminalStatus = null } = {}) { - return { - transport: "agentrun-v01", - protocol: AGENTRUN_PROVIDER_TRACE_PROTOCOL, - wireApi: AGENTRUN_PROVIDER_TRACE_WIRE_API, - runnerKind: AGENTRUN_RUNNER_KIND, - implementationType: AGENTRUN_IMPLEMENTATION_TYPE, - command: "agentrun.v01.command.turn", - toolName: "agentrun.v01.command.turn", - source: ADAPTER_ID, - backendProfile: base.agentRun?.backendProfile ?? null, - runId: base.agentRun?.runId ?? result?.runId ?? null, - commandId: base.agentRun?.commandId ?? result?.commandId ?? null, - runnerId: base.agentRun?.runnerId ?? result?.runnerId ?? null, - jobName: base.agentRun?.jobName ?? result?.jobName ?? null, - namespace: base.agentRun?.namespace ?? result?.namespace ?? null, - traceId: base.traceId ?? base.agentRun?.traceId ?? null, - threadId: result?.sessionRef?.threadId ?? agentRunEffectiveThreadId(base), - turnId: result?.turnId ?? null, - terminalStatus: terminalStatus ?? result?.terminalStatus ?? null, - failureKind: result?.failureKind ?? null, - failureMessage: result?.failureMessage ?? result?.blocker?.message ?? null, - willRetry: result?.willRetry === true ? true : undefined, - valuesPrinted: false - }; -} - -function agentRunFailureAttribution({ code, message, canceled = false } = {}) { - if (canceled) { - return { - category: "canceled", - retryable: true, - userMessage: "AgentRun 请求已取消。", - summary: message || "AgentRun command was canceled." - }; - } - if (code === "provider-invalid-tool-call" || /invalid function arguments json string|invalid_prompt|tool_call_id/iu.test(String(message ?? ""))) { - return { - category: "provider_invalid_tool_call", - retryable: true, - userMessage: "AgentRun/provider 返回了无效 tool-call arguments JSON；这不是 HWPOD 或 Cloud API 端点失败，请查看 providerTrace.failureKind 和 runnerTrace 定位上游 tool_call_id。", - summary: message || "AgentRun/provider returned invalid tool-call arguments JSON." - }; - } - if (code === "thread-resume-failed") { - return { - category: "thread_resume_failed", - retryable: true, - userMessage: "AgentRun 复用的 thread 已失效；当前 turn 会按失败终止，Workbench 会清理 stale continuation 指针，下一轮自动从新 session/thread 启动。", - summary: message || "AgentRun thread/resume failed for an existing thread." - }; - } - return { - category: "agentrun_failed", - retryable: true, - userMessage: "AgentRun 请求失败；请查看 runnerTrace、providerTrace 和 agentRun 字段定位 runId/commandId/jobName。", - summary: message || "AgentRun command failed." - }; -} - -function agentRunLongLivedSessionGate(base = {}) { - const session = agentRunSessionSummary(base, "idle"); - return { - status: "pass", - pass: true, - requiredCapability: AGENTRUN_CAPABILITY_LEVEL, - currentCapability: AGENTRUN_IMPLEMENTATION_TYPE, - sessionId: session.sessionId ?? null, - sessionStatus: session.status ?? null, - runnerKind: AGENTRUN_RUNNER_KIND, - provider: providerForBackendProfile(base.agentRun?.backendProfile ?? "deepseek"), - sessionMode: AGENTRUN_SESSION_MODE, - implementationType: AGENTRUN_IMPLEMENTATION_TYPE, - adapter: ADAPTER_ID, - delegatedToAgentRun: true, - codexStdio: false, - blockers: [], - valuesPrinted: false - }; -} - -function agentRunToolCalls(result = {}, status = "completed", replyTextOverride = null) { - const replyText = messageAuthorityTextValue(replyTextOverride) || agentRunReplyText(result); - return [{ - name: "agentrun.v01.command.turn", - status, - runId: result?.runId ?? null, - commandId: result?.commandId ?? null, - outputSummary: replyText ? `assistantChars=${replyText.length}` : null, - valuesPrinted: false - }]; -} - -function agentRunReplyText(result = {}) { - for (const value of [result?.reply, result?.assistantMessage, result?.finalResponse, result?.content, result?.text, result?.message]) { - const text = messageAuthorityTextValue(value); - if (text) return text; - } - return ""; -} - -function agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent = true }) { - const terminalStatus = String(result?.terminalStatus ?? ""); - const normalizedTerminalStatus = terminalStatus.trim().toLowerCase().replace(/_/gu, "-"); - const retryableInterruption = normalizedTerminalStatus !== "completed" && isRetryableProviderInterruptionPayload(result); - const terminal = !retryableInterruption && ["completed", "failed", "blocked", "cancelled", "canceled"].includes(normalizedTerminalStatus); - if (!terminal) { - if (retryableInterruption && appendResultEvent) { - traceStore.append(traceId, agentRunProviderRetryEvent({ base, payload: result, status: "retrying", label: "agentrun:provider-retry:provider-stream-disconnected" }), base.agentRun); - } - const runningMapping = retryableInterruption ? { - ...base.agentRun, - status: "retrying", - commandState: "retrying", - failureKind: normalizedFailureKind(result?.failureKind), - willRetry: true - } : base.agentRun; - const running = decorateAgentRunRunningResult({ base, mapping: runningMapping, traceStore, traceId }); - if (!retryableInterruption) return running; - return { - ...running, - status: "retrying", - providerTrace: agentRunProviderTrace({ base, result, terminalStatus }), - transient: { failureKind: normalizedFailureKind(result?.failureKind), willRetry: true, valuesRedacted: true }, - valuesPrinted: false - }; - } - const now = nowIso(); - const runnerTrace = traceStore.snapshot(traceId, agentRunTraceMeta({}, {})); - const terminalEventCreatedAt = agentRunResultTraceCreatedAt(runnerTrace, now); - const providerTrace = agentRunProviderTrace({ base, result, terminalStatus }); - const replyText = agentRunReplyText(result); - if (normalizedTerminalStatus === "completed") { - const traceReplyText = agentRunTraceTerminalAssistantText(runnerTrace); - const completedReplyText = replyText || traceReplyText; - const finalResponse = completedReplyText ? agentRunCompletedFinalResponse({ base, result, traceId, now, replyText: completedReplyText }) : null; - const traceSummary = agentRunCompletedTraceSummary({ base, runnerTrace, finalResponse, traceId }); - if (appendResultEvent) { - traceStore.append(traceId, agentRunTraceEvent({ - type: "result", - status: "completed", - label: "agentrun:result:completed", - createdAt: terminalEventCreatedAt, - message: "AgentRun result is ready for HWLAB short-connection polling.", - runId: base.agentRun.runId, - commandId: base.agentRun.commandId, - terminal: true, - valuesPrinted: false - }, base.agentRun)); - } - return { - ...base, - status: "completed", - messageId: finalResponse?.messageId ?? base.messageId ?? `msg_${traceId.slice(4)}`, - threadId: agentRunEffectiveThreadId(base), - updatedAt: now, - workspace: base.workspace ?? "/home/agentrun/workspace", - sandbox: base.sandbox ?? "danger-full-access", - session: agentRunSessionSummary(base, "idle"), - sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true), - runner: agentRunRunnerSummary(base.agentRun), - runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})), - toolCalls: agentRunToolCalls(result, "completed", completedReplyText), - skills: { status: "delegated", provider: ADAPTER_ID, count: 0, items: [], valuesPrinted: false }, - longLivedSessionGate: agentRunLongLivedSessionGate(base), - providerTrace, - finalResponse: finalResponse ?? null, - traceSummary, - reply: finalResponse ? { - messageId: finalResponse.messageId, - role: "assistant", - content: finalResponse.text, - createdAt: finalResponse.createdAt - } : null, - usage: null, - agentRun: { ...base.agentRun, terminalStatus, completed: true, reuseEligible: true, providerTrace, valuesPrinted: false }, - valuesPrinted: false - }; - } - const canceled = terminalStatus === "cancelled"; - const code = canceled ? "agentrun_canceled" : result?.failureKind ?? (terminalStatus === "blocked" ? "agentrun_blocked" : "agentrun_failed"); - const message = result?.failureMessage ?? result?.blocker?.message ?? (canceled ? "AgentRun command was canceled" : "AgentRun command failed"); - const attribution = agentRunFailureAttribution({ code, message, canceled }); - const finalResponse = agentRunTerminalFailureFinalResponse({ - base, - traceId, - now, - status: canceled ? "canceled" : "failed", - text: firstNonEmpty(message, attribution.summary, attribution.userMessage) - }); - const traceSummary = agentRunTerminalFailureTraceSummary({ base, runnerTrace, finalResponse, traceId, terminalStatus: canceled ? "canceled" : terminalStatus || "failed" }); - if (appendResultEvent) { - traceStore.append(traceId, agentRunTraceEvent({ - type: "result", - status: canceled ? "canceled" : "failed", - label: `agentrun:result:${canceled ? "canceled" : terminalStatus || "failed"}`, - createdAt: terminalEventCreatedAt, - errorCode: code, - message: attribution.summary, - runId: base.agentRun.runId, - commandId: base.agentRun.commandId, - terminal: true, - valuesPrinted: false - }, base.agentRun)); - } - const partialContext = partialAgentRunContext(runnerTrace); - const freshSessionRequired = !canceled && agentRunFailureRequiresFreshSession(code, message); - const failedSessionStatus = freshSessionRequired ? "thread-resume-failed" : "failed"; - const resumableAfterFailure = !canceled && !freshSessionRequired && Boolean(agentRunEffectiveThreadId(base)); - return { - ...base, - status: canceled ? "canceled" : "failed", - messageId: finalResponse.messageId, - threadId: agentRunEffectiveThreadId(base), - canceled, - updatedAt: now, - session: agentRunSessionSummary(base, canceled ? "canceled" : failedSessionStatus), - sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true, { status: canceled ? undefined : freshSessionRequired ? "thread-resume-failed" : resumableAfterFailure ? "failed-resumable" : "failed-requires-new-session" }), - runner: agentRunRunnerSummary(base.agentRun), - runnerTrace: partialContext ? { ...runnerTrace, partialContext } : runnerTrace, - toolCalls: agentRunToolCalls(result, canceled ? "canceled" : "failed"), - skills: { status: "delegated", provider: ADAPTER_ID, count: 0, items: [], valuesPrinted: false }, - providerTrace, - finalResponse, - traceSummary, - error: { - code, - layer: "agentrun", - category: attribution.category, - retryable: attribution.retryable, - message, - userMessage: attribution.userMessage, - traceId, - route: "/v1/agent/chat", - toolName: "agentrun.manual-dispatch" - }, - blocker: canceled ? null : { - code, - layer: "agentrun", - category: attribution.category, - retryable: attribution.retryable, - summary: attribution.summary, - traceId, - route: "/v1/agent/chat", - toolName: "agentrun.manual-dispatch" - }, - agentRun: { ...base.agentRun, terminalStatus, completed: false, reuseEligible: resumableAfterFailure, providerTrace, valuesPrinted: false }, - reuseEligible: resumableAfterFailure, - ...(partialContext ? { partialContext } : {}), - valuesPrinted: false - }; -} - -function agentRunCompletedFinalResponse({ base, result, traceId, now, replyText = null }) { - const textValue = messageAuthorityTextValue(replyText) || agentRunReplyText(result); - return { - text: textValue, - textChars: textValue.length, - role: "assistant", - status: "completed", - traceId, - messageId: base.messageId ?? base.reply?.messageId ?? `msg_${traceId.slice(4)}`, - createdAt: base.reply?.createdAt ?? base.createdAt ?? now, - updatedAt: now, - valuesPrinted: false - }; -} - -function agentRunTraceTerminalAssistantText(runnerTrace = {}) { - const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; - for (let index = events.length - 1; index >= 0; index -= 1) { - const event = events[index]; - if (!event || typeof event !== "object") continue; - if (!isAgentRunAssistantTraceEvent(event)) continue; - if (!(event.terminal === true || event.final === true || event.replyAuthority === true)) continue; - const textValue = messageAuthorityTextValue(event.finalResponse ?? event.text ?? event.content ?? event.message ?? event.summary ?? event.payload?.text ?? event.payload?.content ?? event.payload?.message); - if (textValue) return textValue; - } - return ""; -} - -function isAgentRunAssistantTraceEvent(event = {}) { - const type = String(event.type ?? event.eventType ?? "").trim().toLowerCase(); - if (type === "assistant" || type === "assistant_message") return true; - return /assistant:message|assistant_message/u.test(String(event.label ?? "").toLowerCase()); -} - -function agentRunTerminalFailureFinalResponse({ base, traceId, now, status, text }) { - const textValue = String(text ?? "").trim(); - return { - text: textValue || "AgentRun command failed.", - textChars: (textValue || "AgentRun command failed.").length, - role: "assistant", - status, - traceId, - messageId: base.messageId ?? base.reply?.messageId ?? `msg_${traceId.slice(4)}`, - createdAt: base.reply?.createdAt ?? base.createdAt ?? now, - updatedAt: now, - valuesPrinted: false - }; -} - -function agentRunTerminalFailureTraceSummary({ base, runnerTrace, finalResponse, traceId, terminalStatus }) { - const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; - const lastSeq = Math.max(0, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); - return { - traceId, - source: "agentrun-command-result", - sourceEventCount: Number(runnerTrace?.eventCount ?? events.length ?? 0), - terminalStatus, - finalAssistantRow: { - role: finalResponse.role, - status: finalResponse.status, - textChars: finalResponse.textChars, - textPreview: finalResponse.text.slice(0, 240), - messageId: finalResponse.messageId, - valuesPrinted: false - }, - agentRun: { - runId: base.agentRun?.runId ?? null, - commandId: base.agentRun?.commandId ?? null, - lastSeq, - valuesPrinted: false - }, - valuesPrinted: false - }; -} - -function agentRunCompletedTraceSummary({ base, runnerTrace, finalResponse, traceId }) { - const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; - const lastSeq = Math.max(0, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); - return { - traceId, - source: "agentrun-command-result", - sourceEventCount: Number(runnerTrace?.eventCount ?? events.length ?? 0), - terminalStatus: "completed", - finalAssistantRow: finalResponse ? { - role: finalResponse.role, - status: finalResponse.status, - textChars: finalResponse.textChars, - textPreview: finalResponse.text.slice(0, 240), - messageId: finalResponse.messageId, - valuesPrinted: false - } : null, - agentRun: { - runId: base.agentRun?.runId ?? null, - commandId: base.agentRun?.commandId ?? null, - lastSeq, - valuesPrinted: false - }, - valuesPrinted: false - }; -} - -function partialAgentRunContext(runnerTrace = {}) { - const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; - const assistantMessages = events - .filter((event) => event?.type === "assistant" && String(event.text ?? event.message ?? "").trim()) - .map((event) => String(event.text ?? event.message).trim()) - .slice(-4); - const toolEvidence = events - .filter((event) => event?.type === "tool_call" && String(event.status ?? "") === "completed") - .map((event) => [event.toolName ?? event.label ?? "tool", event.command ?? event.outputSummary ?? event.stdoutSummary ?? event.message ?? ""].filter(Boolean).join(": ")) - .filter(Boolean) - .slice(-6); - if (assistantMessages.length === 0 && toolEvidence.length === 0) return null; - return { - status: "partial-before-terminal", - summary: "AgentRun terminal 前已有可延续的 assistant/tool 上下文；后续同 conversation/session/thread 轮次必须能继续使用。", - assistantMessages, - toolEvidence, - valuesPrinted: false - }; -} - -function agentRunResultTraceCreatedAt(runnerTrace = {}, fallback) { - const last = String(runnerTrace?.lastEvent?.createdAt ?? runnerTrace?.updatedAt ?? ""); - return Number.isFinite(Date.parse(last)) ? last : fallback; -} - -function decorateAgentRunRunningResult({ base, mapping, traceStore, traceId }) { - return { - ...base, - status: "running", - threadId: agentRunEffectiveThreadId({ agentRun: mapping }), - accepted: true, - shortConnection: true, - updatedAt: nowIso(), - session: agentRunSessionSummary({ ...base, agentRun: mapping }, "running"), - sessionReuse: agentRunSessionReuseSummary({ ...base, agentRun: mapping }, mapping.reused === true), - runner: agentRunRunnerSummary(mapping), - runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})), - agentRun: { ...mapping, adapter: ADAPTER_ID, valuesPrinted: false }, - valuesPrinted: false - }; -} - -function appendAgentRunEventsToTrace(traceStore, traceId, events, mapping = {}) { - for (const event of events) { - if (isForeignAgentRunCommandEvent(event, mapping)) continue; - const normalized = mapAgentRunEvent(event, mapping); - if (normalized) traceStore.append(traceId, normalized, agentRunTraceMeta({}, {})); - } -} - -function isForeignAgentRunCommandEvent(event, mapping = {}) { - const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : ""; - if (!currentCommandId) return false; - const eventCommandId = agentRunEventCommandId(event); - return Boolean(eventCommandId && eventCommandId !== currentCommandId); -} - -function mapAgentRunEvent(event, mapping = {}) { - if (!event || typeof event !== "object") return null; - const payload = event.payload && typeof event.payload === "object" ? event.payload : {}; - const base = { - createdAt: event.createdAt ?? null, - source: "agentrun", - sourceSeq: event.seq ?? null, - backend: backendForBackendProfile(mapping.backendProfile ?? payload.backendProfile ?? payload.providerProfile ?? "deepseek"), - runId: event.runId ?? mapping.runId ?? null, - commandId: payload.commandId ?? mapping.commandId ?? null, - attemptId: payload.attemptId ?? mapping.attemptId ?? null, - runnerId: payload.runnerId ?? mapping.runnerId ?? null, - jobName: payload.jobName ?? mapping.jobName ?? null, - namespace: payload.namespace ?? mapping.namespace ?? null, - valuesPrinted: false - }; - if (event.type === "backend_status") { - const phase = String(payload.phase ?? "status"); - return { - ...base, - type: "backend", - eventType: "backend", - status: "running", - label: `agentrun:backend:${phase}`, - message: textPayload(payload, phase), - waitingFor: waitingForForPhase(phase), - details: agentRunBackendStatusDetails(phase, payload) - }; - } - if (event.type === "assistant_message") { - const terminal = payload.replyAuthority === true || payload.final === true; - return { - ...base, - type: "assistant", - eventType: "assistant", - status: terminal ? "completed" : "running", - label: "agentrun:assistant:message", - message: textPayload(payload, "assistant message"), - text: textPayload(payload, ""), - itemId: payload.itemId ?? null, - messageIndex: payload.messageIndex ?? null, - messageCount: payload.messageCount ?? null, - replyAuthority: payload.replyAuthority === true, - final: payload.final === true, - terminal - }; - } - if (event.type === "tool_call") { - const commandExecution = agentRunCommandExecutionEvent(base, payload); - if (commandExecution) return commandExecution; - const preview = agentRunToolCallPreviewSummary(payload); - const toolName = firstNonEmpty( - payload.toolName, - payload.name, - payload.item?.type, - payload.type, - preview.toolName, - preview.name, - preview.type, - isProtocolToolLifecycleMethod(payload.method) ? null : payload.method, - "tool call" - ); - const command = firstNonEmpty(payload.command, payload.commandLine, payload.item?.command, payload.item?.commandLine, preview.command, preview.commandLine); - const output = firstNonEmpty(payload.outputSummary, payload.stdoutSummary, preview.outputSummary, preview.stdoutSummary, payload.summary?.text, payload.itemPreview); - return { - ...base, - type: "tool_call", - eventType: "tool_call", - status: String(payload.status ?? preview.status ?? (payload.method === "item/completed" ? "completed" : "running")), - label: `agentrun:tool:${String(toolName)}`, - itemId: payload.item?.id ?? payload.itemId ?? preview.itemId ?? null, - toolName, - command, - exitCode: Number.isInteger(payload.item?.exitCode) ? payload.item.exitCode : Number.isInteger(payload.exitCode) ? payload.exitCode : undefined, - durationMs: typeof payload.item?.durationMs === "number" ? payload.item.durationMs : typeof payload.durationMs === "number" ? payload.durationMs : undefined, - stdoutSummary: output, - outputSummary: output, - message: output || command || textPayload(payload, "tool call"), - outputBytes: typeof payload.outputBytes === "number" ? payload.outputBytes : payload.summary?.outputBytes, - outputTruncated: payload.outputTruncated === true || payload.summary?.outputTruncated === true - }; - } - if (event.type === "command_output") { - const stream = payload.stream === "stderr" ? "stderr" : "stdout"; - return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true }; - } - if (event.type === "diff") { - return { ...base, type: "diff", eventType: "status", status: "running", label: "agentrun:diff", message: textPayload(payload, "diff") }; - } - if (event.type === "error") { - const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? "backend") ?? "backend"; - const normalizedPayload = { ...payload, failureKind, errorCode: failureKind }; - if (isRetryableProviderInterruptionPayload(normalizedPayload, { activeError: true })) return agentRunProviderRetryEvent({ base, payload: { ...normalizedPayload, willRetry: true }, status: "retrying", label: `agentrun:provider-retry:${failureKind}` }); - return { ...base, type: "error", eventType: "error", status: "failed", label: `agentrun:error:${failureKind}`, errorCode: failureKind, failureKind, willRetry: payload.willRetry === true ? true : undefined, message: textPayload(payload, "AgentRun error") }; - } - if (event.type === "terminal_status") { - const terminalStatus = String(payload.terminalStatus ?? event.terminalStatus ?? payload.status ?? event.status ?? "failed"); - const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? event.failureKind ?? event.errorCode) ?? null; - const normalizedPayload = failureKind ? { ...payload, failureKind, errorCode: failureKind } : payload; - if (isRetryableProviderInterruptionPayload(normalizedPayload)) return agentRunProviderRetryEvent({ base, payload: normalizedPayload, status: "retrying", label: `agentrun:provider-retry:${terminalStatus}` }); - return { ...base, type: "result", eventType: "terminal", status: terminalStatus === "cancelled" ? "canceled" : terminalStatus, label: `agentrun:terminal:${terminalStatus}`, errorCode: failureKind, failureKind, willRetry: payload.willRetry === true || event.willRetry === true ? true : undefined, message: textPayload({ ...event, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true }; - } - return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${String(event.type ?? "unknown")}`, message: textPayload(payload, "AgentRun event") }; -} - -function agentRunProviderRetryEvent({ base = {}, payload = {}, status = "retrying", label = "agentrun:provider-retry" } = {}) { - const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? "provider-stream-disconnected"); - const retryMessage = providerRetryMessage(textPayload(payload, "Provider stream disconnected; AgentRun will retry."), payload); - return { - ...base, - type: "provider_retry", - eventType: "backend", - status, - label, - errorCode: failureKind, - failureKind, - willRetry: payload.willRetry === true, - retryable: true, - transient: true, - terminal: false, - retryAttempt: numberOrNull(payload.retryAttempt), - retryMax: numberOrNull(payload.retryMax), - retryDelayMs: numberOrNull(payload.retryDelayMs), - nextRetryAt: firstNonEmpty(payload.nextRetryAt) || null, - retryOf: payload.retryOf ?? null, - terminalStatus: payload.terminalStatus ?? null, - message: retryMessage, - valuesPrinted: false - }; -} - -function providerRetryMessage(message, payload = {}) { - const attempt = numberOrNull(payload.retryAttempt); - const max = numberOrNull(payload.retryMax); - const delayMs = numberOrNull(payload.retryDelayMs); - const nextRetryAt = firstNonEmpty(payload.nextRetryAt) || null; - const parts = []; - if (attempt !== null && max !== null) parts.push(`retry ${attempt}/${max}`); - else if (attempt !== null) parts.push(`retry attempt ${attempt}`); - else if (max !== null) parts.push(`retry max ${max}`); - if (delayMs !== null) parts.push(`after ${delayMs}ms`); - if (nextRetryAt) parts.push(`next=${nextRetryAt}`); - if (parts.length === 0) return message; - const text = String(message || "Provider stream disconnected; AgentRun will retry.").trim(); - if (attempt !== null && max !== null && text.includes(`${attempt}/${max}`)) return text; - return text.replace(/[.。]\s*$/u, "") + `; ${parts.join(" ")}.`; -} - -function isRetryableProviderInterruptionPayload(payload = {}, options = {}) { - const kind = normalizedFailureKind(payload?.failureKind ?? payload?.errorCode ?? payload?.code); - if (kind === "provider-stream-disconnected") return payload?.willRetry === true; - if (kind === "provider-unavailable") return payload?.willRetry === true || (options.activeError === true && payload?.terminal !== true && payload?.final !== true); - if (isRetryableAgentRunTransportFailureKind(kind)) return payload?.willRetry === true; - return false; -} - -function normalizedFailureKind(value) { - const normalized = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-") || null; - if (normalized === "agentrun-unreachable") return "agentrun-connect-failed"; - return normalized; -} - -function numberOrNull(value) { - const number = Number(value); - return Number.isFinite(number) ? number : null; -} - -function agentRunTraceEvent(event = {}, backendSource = {}) { - const source = backendSource && typeof backendSource === "object" ? backendSource : { backendProfile: backendSource }; - const backendProfile = firstNonEmpty(event.backendProfile, source.backendProfile, source.providerProfile, source.agentRun?.backendProfile); - return { - ...event, - eventType: event.eventType ?? agentRunManualEventType(event), - backend: event.backend ?? (backendProfile ? backendForBackendProfile(backendProfile) : null), - backendProfile: event.backendProfile ?? backendProfile ?? null, - valuesPrinted: event.valuesPrinted ?? false - }; -} - -function agentRunManualEventType(event = {}) { - const type = String(event.type ?? "").trim().toLowerCase(); - if (type === "request") return "request"; - if (type === "cancel" || type === "result") return "terminal"; - if (type === "assistant") return "assistant"; - if (type === "tool" || type === "tool_call") return "tool_call"; - if (type === "error") return "error"; - if (type === "backend") return "backend"; - const raw = String(`${event.label ?? ""} ${event.status ?? ""}`).toLowerCase(); - if (/request|accepted/u.test(raw)) return "request"; - if (/cancel|terminal|complete|final/u.test(raw)) return "terminal"; - if (/assistant|message|reply/u.test(raw)) return "assistant"; - if (/error|fail|timeout/u.test(raw)) return "error"; - if (/backend|runner|agentrun/u.test(raw)) return "backend"; - return "status"; -} - -function isProtocolToolLifecycleMethod(value) { - return /^item\/(started|completed)$/u.test(String(value ?? "")); -} - -function agentRunToolCallPreviewSummary(payload = {}) { - const text = firstNonEmpty(payload.itemPreview, payload.summary?.text); - return { - itemId: jsonPreviewStringField(text, "itemId"), - type: jsonPreviewStringField(text, "type"), - toolName: jsonPreviewStringField(text, "toolName"), - name: jsonPreviewStringField(text, "name"), - status: jsonPreviewStringField(text, "status"), - command: jsonPreviewStringField(text, "command"), - commandLine: jsonPreviewStringField(text, "commandLine"), - outputSummary: jsonPreviewStringField(text, "outputSummary"), - stdoutSummary: jsonPreviewStringField(text, "stdoutSummary") - }; -} - -function jsonPreviewStringField(text, key) { - const body = String(text ?? ""); - if (!body || !/^[A-Za-z][A-Za-z0-9_]*$/u.test(key)) return null; - const match = body.match(new RegExp(`"${key}"\\s*:\\s*"((?:\\\\.|[^"\\\\])*)"`, "u")); - if (!match) return null; - try { - const parsed = JSON.parse(`"${match[1]}"`); - return typeof parsed === "string" && parsed.trim() ? parsed.trim() : null; - } catch { - return match[1]?.trim() || null; - } -} - -function agentRunBackendStatusDetails(phase, payload = {}) { - if (phase === "initial-prompt-assembly") { - return compactObject({ - initialPromptInjected: payload.initialPromptInjected === true, - reason: firstNonEmpty(payload.reason), - initialPrompt: agentRunResourceSummary(payload.initialPrompt), - valuesPrinted: false - }); - } - if (phase === "resource-bundle-materialized") { - return compactObject({ - kind: firstNonEmpty(payload.kind), - commitId: firstNonEmpty(payload.commitId), - bundles: agentRunResourceSummary(payload.bundles), - tools: agentRunResourceSummary(payload.tools), - promptRefs: agentRunResourceSummary(payload.promptRefs), - skillDirs: agentRunResourceSummary(payload.skillDirs), - initialPrompt: agentRunResourceSummary(payload.initialPrompt), - valuesPrinted: false - }); - } - return undefined; -} - -function agentRunResourceSummary(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; - return compactObject({ - available: typeof value.available === "boolean" ? value.available : undefined, - count: Number.isInteger(value.count) ? value.count : undefined, - bytes: Number.isInteger(value.bytes) ? value.bytes : undefined, - sha256: firstNonEmpty(value.sha256), - promptRefCount: Number.isInteger(value.promptRefCount) ? value.promptRefCount : undefined, - skillCount: Number.isInteger(value.skillCount) ? value.skillCount : undefined, - toolCount: Number.isInteger(value.toolCount) ? value.toolCount : undefined, - names: Array.isArray(value.names) ? value.names.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, - targetPaths: Array.isArray(value.targetPaths) ? value.targetPaths.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, - skillNames: Array.isArray(value.skillNames) ? value.skillNames.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, - requiredMissing: Array.isArray(value.requiredMissing) ? value.requiredMissing.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, - valuesPrinted: false - }); -} - -function compactObject(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; - const result = {}; - for (const [key, item] of Object.entries(value)) { - if (item === undefined || item === null) continue; - if (Array.isArray(item) && item.length === 0) continue; - if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue; - result[key] = item; - } - return Object.keys(result).length > 0 ? result : undefined; -} - -function agentRunCommandExecutionEvent(base, payload = {}) { - const item = payload.item && typeof payload.item === "object" ? payload.item : null; - const payloadToolType = firstNonEmpty(payload.toolName, payload.type, payload.name); - if (item?.type !== "commandExecution" && payloadToolType !== "commandExecution") return null; - const method = String(payload.method ?? ""); - const completed = method === "item/completed" || payload.status === "completed" || item?.status === "completed"; - const output = firstNonEmpty(item?.aggregatedOutput, item?.stdout, item?.output, payload.outputSummary, payload.stdoutSummary, payload.summary?.text, payload.itemPreview); - return { - ...base, - type: "tool_call", - status: completed ? "completed" : "started", - label: completed ? "item/commandExecution:completed" : "item/commandExecution:started", - toolName: "commandExecution", - itemId: item?.id ?? payload.itemId ?? null, - command: firstNonEmpty(item?.command, item?.commandLine, payload.command, payload.commandLine), - exitCode: Number.isInteger(item?.exitCode) ? item.exitCode : Number.isInteger(payload.exitCode) ? payload.exitCode : undefined, - durationMs: typeof item?.durationMs === "number" ? item.durationMs : typeof payload.durationMs === "number" ? payload.durationMs : undefined, - outputBytes: typeof payload.outputBytes === "number" ? payload.outputBytes : payload.summary?.outputBytes, - stdoutSummary: output, - outputSummary: output, - outputTruncated: payload.outputTruncated === true || payload.summary?.outputTruncated === true, - message: output || textPayload(payload, "commandExecution") - }; -} - -function nowMs() { - if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); - return Date.now(); -} - -function agentRunMapping({ env, managerUrl, backendProfile, run, command, runnerJob, traceId, startedAt, params = {} }) { - const threadReused = Boolean(safeOpaqueId(params.threadId)); - const runnerJobCreated = Boolean(runnerJob); - return { - adapter: ADAPTER_ID, - managerUrl, - backendProfile, - providerId: resolveAgentRunProviderId(env), - runId: run.id, - commandId: command.id, - attemptId: runnerJob?.attemptId ?? runnerJob?.runner?.attemptId ?? null, - runnerId: runnerJob?.runnerId ?? runnerJob?.runner?.runnerId ?? null, - runnerJobId: runnerJob?.id ?? null, - jobName: runnerJob?.jobName ?? runnerJob?.jobIdentity?.name ?? null, - namespace: runnerJob?.namespace ?? runnerJob?.jobIdentity?.namespace ?? DEFAULT_RUNNER_NAMESPACE, - status: runnerJobCreated ? "runner-job-created" : "runner-job-pending", - runStatus: run.status ?? null, - commandState: command.state ?? null, - terminalStatus: null, - lastSeq: 0, - traceId, - ...codeAgentOtelTraceFields(traceId, env), - sessionId: run?.sessionRef?.sessionId ?? null, - conversationId: run?.sessionRef?.conversationId ?? null, - threadId: run?.sessionRef?.threadId ?? null, - runnerReused: false, - runnerJobCount: runnerJobCreated ? 1 : 0, - threadReused, - persistentResume: threadReused, - reused: false, - reuseEligible: true, - createdAt: startedAt, - updatedAt: nowIso(), - valuesPrinted: false - }; -} - -function agentRunMappingWithRunnerJob(mapping = {}, runnerJob = null) { - return { - ...mapping, - attemptId: runnerJob?.attemptId ?? runnerJob?.runner?.attemptId ?? mapping.attemptId ?? null, - runnerId: runnerJob?.runnerId ?? runnerJob?.runner?.runnerId ?? mapping.runnerId ?? null, - runnerJobId: runnerJob?.id ?? mapping.runnerJobId ?? null, - jobName: runnerJob?.jobName ?? runnerJob?.jobIdentity?.name ?? mapping.jobName ?? null, - namespace: runnerJob?.namespace ?? runnerJob?.jobIdentity?.namespace ?? mapping.namespace ?? DEFAULT_RUNNER_NAMESPACE, - status: "runner-job-created", - runnerJobCount: Math.max(1, Number(mapping.runnerJobCount ?? 0) + 1), - updatedAt: nowIso(), - valuesPrinted: false - }; -} - -function agentRunReusedMapping({ previous = {}, run = {}, command = {}, runnerJob = null, traceId, startedAt, backendProfile, managerUrl, env }) { - const threadReused = Boolean(safeOpaqueId(run?.sessionRef?.threadId ?? previous.threadId)); - return { - ...previous, - adapter: ADAPTER_ID, - managerUrl, - backendProfile, - providerId: previous.providerId ?? resolveAgentRunProviderId(env), - runId: previous.runId ?? run?.id, - commandId: command.id, - attemptId: runnerJob?.attemptId ?? runnerJob?.runner?.attemptId ?? previous.attemptId ?? null, - runnerId: runnerJob?.runnerId ?? runnerJob?.runner?.runnerId ?? previous.runnerId ?? null, - runnerJobId: runnerJob?.id ?? previous.runnerJobId ?? null, - jobName: runnerJob?.jobName ?? runnerJob?.jobIdentity?.name ?? previous.jobName ?? null, - namespace: runnerJob?.namespace ?? runnerJob?.jobIdentity?.namespace ?? previous.namespace ?? DEFAULT_RUNNER_NAMESPACE, - status: runnerJob ? "runner-job-ensured" : "runner-job-reused", - runStatus: run?.status ?? previous.runStatus ?? null, - commandState: command.state ?? null, - terminalStatus: null, - lastSeq: previous.lastSeq ?? 0, - runnerJobCount: runnerJob ? Math.max(1, Number(previous.runnerJobCount ?? 0) + 1) : Number(previous.runnerJobCount ?? 0), - traceId, - sessionId: run?.sessionRef?.sessionId ?? previous.sessionId ?? null, - conversationId: run?.sessionRef?.conversationId ?? previous.conversationId ?? null, - threadId: run?.sessionRef?.threadId ?? previous.threadId ?? null, - runnerReused: true, - threadReused, - persistentResume: threadReused, - reused: true, - reuseEligible: true, - createdAt: previous.createdAt ?? startedAt, - updatedAt: nowIso(), - valuesPrinted: false - }; -} - -function agentRunResultRefs(result = {}) { - const refs = {}; - for (const key of ["attemptId", "runnerId", "jobName", "namespace", "runnerJobCount"]) { - if (result?.[key] !== undefined && result?.[key] !== null) refs[key] = result[key]; - } - if (result?.sessionRef && typeof result.sessionRef === "object") { - if (result.sessionRef.sessionId) refs.sessionId = result.sessionRef.sessionId; - if (result.sessionRef.conversationId) refs.conversationId = result.sessionRef.conversationId; - if (result.sessionRef.threadId) refs.threadId = result.sessionRef.threadId; - } - return refs; -} - -function withAgentRunRunnerJobCount(mapping = {}) { - if (!mapping || typeof mapping !== "object") return mapping; - const count = Number(mapping.runnerJobCount); - if (Number.isFinite(count) && count >= 0) return mapping; - const runnerJobEnsured = mapping.reused === true || mapping.status === "runner-job-ensured"; - if (runnerJobEnsured && (mapping.runnerJobId || mapping.jobName || mapping.attemptId || mapping.runnerId)) { - return { ...mapping, runnerJobCount: 1 }; - } - return mapping; -} - -function hwlabSessionIdForParams(params = {}) { - return safeSessionId(params.sessionId) || null; -} - -function scopedAgentRunSessionIdForParams(params = {}, traceId) { - const baseSessionId = hwlabSessionIdForParams(params, traceId); - if (!safeSessionId(baseSessionId)) { - throw adapterError( - "agentrun_session_id_required", - "AgentRun persistent session/PVC requires params.sessionId; refusing to derive sessionRef.sessionId from traceId because that would create a new PVC per turn." - ); - } - const base = String(baseSessionId).replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session"; - return `ses_agentrun_${base}`; -} - -function agentRunSessionId(traceId) { - return `ses_agentrun_${String(safeTraceId(traceId) || `trc_${randomUUID()}`).slice(4)}`; -} - -function agentRunEffectiveThreadId(base = {}) { - const agentRun = base?.agentRun && typeof base.agentRun === "object" ? base.agentRun : null; - if (agentRun && Object.hasOwn(agentRun, "threadId")) return safeOpaqueId(agentRun.threadId) || null; - return safeOpaqueId(base?.threadId) || null; -} - -function agentRunSessionSummary(base, status) { - return decorateCodeAgentSession({ - sessionId: base.sessionId ?? base.agentRun?.sessionId ?? null, - conversationId: base.conversationId ?? base.agentRun?.conversationId ?? null, - threadId: agentRunEffectiveThreadId(base), - status, - sessionMode: AGENTRUN_SESSION_MODE, - runnerKind: AGENTRUN_RUNNER_KIND, - lastTraceId: base.traceId ?? base.agentRun?.traceId ?? null, - longLivedSession: true, - codexStdio: false, - delegatedToAgentRun: true, - writeCapable: true, - durable: true, - durableSession: true, - idleTimeoutMs: parsePositiveInteger(base.agentRun?.idleTimeoutMs, 600000), - secretMaterialStored: false, - valuesRedacted: true - }); -} - -function agentRunThreadReused(base = {}, options = {}) { - if (options.status === "failed-requires-new-session") return false; - if (base.agentRun?.threadReused === true || base.agentRun?.persistentResume === true) return true; - return Boolean(agentRunEffectiveThreadId(base)); -} - -function agentRunSessionReuseSummary(base, reused, options = {}) { - const runnerReused = Boolean(reused); - const threadReused = agentRunThreadReused(base, options); - const persistentResume = threadReused; - const sessionReused = runnerReused || persistentResume; - return { - sessionId: base.sessionId ?? base.agentRun?.sessionId ?? null, - conversationId: base.conversationId ?? base.agentRun?.conversationId ?? null, - threadId: agentRunEffectiveThreadId(base), - reused: sessionReused, - status: options.status ?? (runnerReused ? "reused" : persistentResume ? "thread-resumed" : "new"), - runnerReused, - threadReused, - persistentResume, - valuesRedacted: true - }; -} - -function agentRunRunnerSummary(mapping = {}) { - return { - kind: AGENTRUN_RUNNER_KIND, - adapter: ADAPTER_ID, - runId: mapping.runId ?? null, - commandId: mapping.commandId ?? null, - attemptId: mapping.attemptId ?? null, - runnerId: mapping.runnerId ?? null, - jobName: mapping.jobName ?? null, - namespace: mapping.namespace ?? null, - backendProfile: mapping.backendProfile ?? null, - provider: providerForBackendProfile(mapping.backendProfile ?? "deepseek"), - sessionMode: AGENTRUN_SESSION_MODE, - implementationType: AGENTRUN_IMPLEMENTATION_TYPE, - capabilityLevel: AGENTRUN_CAPABILITY_LEVEL, - codexStdio: false, - delegatedToAgentRun: true, - writeCapable: true, - durableSession: true, - longLivedSession: true, - sandbox: "danger-full-access", - valuesPrinted: false - }; -} - -function hostForUrl(value) { - if (!value) return null; - try { - return new URL(value).hostname.toLowerCase(); - } catch { - return null; - } -} - -function waitingForForPhase(phase) { - if (phase.includes("runner-job")) return "agentrun-runner"; - if (phase.includes("resource-bundle")) return "agentrun-resource-bundle"; - if (phase.includes("backend")) return "agentrun-backend"; - return "agentrun-result"; -} - -function textPayload(payload, fallback) { - for (const key of ["message", "text", "content", "delta", "summary", "phase"]) { - if (typeof payload?.[key] === "string" && payload[key].length > 0) return payload[key]; - } - return fallback; -} - -function requiredString(value, fieldName) { - if (typeof value === "string" && value.trim()) return value.trim(); - throw Object.assign(new Error(`AgentRun response missing ${fieldName}`), { code: "agentrun_response_invalid", statusCode: 502 }); -} - -function firstNonEmpty(...values) { - for (const value of values) { - const text = String(value ?? "").trim(); - if (text) return text; - } - return null; -} - -function nowIso(now) { - return typeof now === "function" ? now() : new Date().toISOString(); -} - -function timestampIsoOrNow(value = null) { - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); -} - -function redactUrl(value) { - try { - const url = new URL(value); - if (url.username || url.password) { - url.username = "***"; - url.password = ""; - } - return url.toString(); - } catch { - return ""; - } -} diff --git a/internal/cloud/code-agent-agentrun-durable-admission.test.ts b/internal/cloud/code-agent-agentrun-durable-admission.test.ts new file mode 100644 index 00000000..890825a9 --- /dev/null +++ b/internal/cloud/code-agent-agentrun-durable-admission.test.ts @@ -0,0 +1,354 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; + +const sourceCommit = "0123456789abcdef0123456789abcdef01234567"; + +test("fresh gpt.pika turn returns running only after durable dispatch intent is confirmed", async () => { + const traceId = "trc_gpt_pika_durable_admission"; + const calls = []; + const traceStore = createCodeAgentTraceStore(); + const result = await submitAgentRunChatTurn({ + traceId, + traceStore, + params: { + ownerUserId: "usr_gpt_pika", + projectId: "pikasTech/HWLAB", + conversationId: "cnv_gpt_pika_durable", + sessionId: "ses_gpt_pika_durable", + message: "verify durable admission", + providerProfile: "gpt.pika" + }, + options: { + env: agentRunTestEnv(), + accessController: { + async codeAgentToolCapabilitiesForOwner() { + return { tools: { hwpod: { allowed: true } } }; + }, + store: { + async findActiveDefaultApiKeyForUser() { + return { displaySecret: "hwl_live_must_not_be_persisted" }; + } + } + }, + fetchImpl: createAgentRunFetch(calls, { + dispatchIntent: { + id: "dispatch_gpt_pika_durable", + state: "pending", + runnerJobId: "rjob_gpt_pika_durable", + attemptCount: 0, + durable: true + } + }) + } + }); + + assert.equal(result.status, "running"); + assert.equal(result.agentRun.runId, "run_gpt_pika_durable"); + assert.equal(result.agentRun.commandId, "cmd_gpt_pika_durable"); + assert.equal(result.agentRun.dispatchIntentId, "dispatch_gpt_pika_durable"); + assert.equal(result.agentRun.dispatchIntentState, "pending"); + assert.equal(result.agentRun.runnerJobId, "rjob_gpt_pika_durable"); + assert.equal(result.agentRun.durableDispatch, true); + assert.equal(result.agentRun.status, "dispatch-intent-pending"); + + const commandCall = calls.find((call) => call.path.endsWith("/commands")); + assert.ok(commandCall); + assert.equal(commandCall.body.idempotencyKey, traceId); + assert.equal(commandCall.body.payload.traceId, traceId); + assert.equal(commandCall.body.payload.sessionId, "ses_agentrun_gpt_pika_durable"); + assert.equal(commandCall.body.payload.hwlabSessionId, "ses_gpt_pika_durable"); + assert.equal(commandCall.body.payload.conversationId, "cnv_gpt_pika_durable"); + assert.equal(commandCall.body.dispatch.kind, "kubernetes-job"); + assert.equal(Object.hasOwn(commandCall.body.dispatch.input, "commandId"), false); + assert.equal(commandCall.body.dispatch.input.idempotencyKey, traceId); + assert.equal(commandCall.body.dispatch.input.transientEnv.every((entry) => entry.sensitive === false), true); + assert.equal(JSON.stringify(commandCall.body).includes("hwl_live_must_not_be_persisted"), false); + assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); + assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:dispatch-intent:admitted")); +}); + +test("fresh turn rejects an unconfirmed dispatch intent instead of returning false running", async () => { + const calls = []; + await assert.rejects( + submitAgentRunChatTurn({ + traceId: "trc_dispatch_intent_missing", + traceStore: createCodeAgentTraceStore(), + params: { + sessionId: "ses_dispatch_intent_missing", + conversationId: "cnv_dispatch_intent_missing", + message: "must fail admission", + providerProfile: "gpt.pika" + }, + options: { + env: agentRunTestEnv(), + fetchImpl: createAgentRunFetch(calls, { dispatchIntent: null }) + } + }), + (error) => error?.code === "agentrun_dispatch_intent_unconfirmed" && error?.statusCode === 502 + ); + assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); +}); + +test("fresh turn rejects a durable dispatch intent with missing contract fields", async () => { + await assert.rejects( + submitAgentRunChatTurn({ + traceId: "trc_dispatch_intent_attempt_missing", + traceStore: createCodeAgentTraceStore(), + params: { + sessionId: "ses_dispatch_intent_attempt_missing", + message: "must require explicit attempt count", + providerProfile: "gpt.pika" + }, + options: { + env: agentRunTestEnv(), + fetchImpl: createAgentRunFetch([], { + dispatchIntent: { + id: "dispatch_attempt_missing", + state: "pending", + runnerJobId: "rjob_attempt_missing", + durable: true + } + }) + } + }), + (error) => error?.code === "agentrun_dispatch_intent_unconfirmed" && error?.statusCode === 502 + ); +}); + +for (const terminal of [ + { state: "failed", code: "agentrun_dispatch_intent_failed", statusCode: 503 }, + { state: "cancelled", code: "agentrun_dispatch_intent_cancelled", statusCode: 409 } +]) { + test(`fresh turn exposes ${terminal.state} durable dispatch replay instead of returning running`, async () => { + await assert.rejects( + submitAgentRunChatTurn({ + traceId: `trc_dispatch_intent_${terminal.state}`, + traceStore: createCodeAgentTraceStore(), + params: { + sessionId: `ses_dispatch_intent_${terminal.state}`, + message: `surface ${terminal.state} admission`, + providerProfile: "gpt.pika" + }, + options: { + env: agentRunTestEnv(), + fetchImpl: createAgentRunFetch([], { + dispatchIntent: { + id: `dispatch_terminal_${terminal.state}`, + state: terminal.state, + runnerJobId: `rjob_terminal_${terminal.state}`, + attemptCount: 2, + durable: true + } + }) + } + }), + (error) => error?.code === terminal.code && error?.statusCode === terminal.statusCode + ); + }); +} + +test("gpt.pika session-eviction retry uses the same durable dispatch admission", async () => { + const traceId = "trc_gpt_pika_durable_retry"; + const calls = []; + const traceStore = createCodeAgentTraceStore(); + const synced = await syncAgentRunChatResult({ + traceId, + traceStore, + refreshEvents: false, + forceResultSync: true, + retryParams: { + ownerUserId: "usr_gpt_pika_retry", + projectId: "pikasTech/HWLAB", + conversationId: "cnv_gpt_pika_retry", + sessionId: "ses_gpt_pika_retry", + threadId: "thread_evicted", + message: "retry the evicted gpt.pika turn", + providerProfile: "gpt.pika" + }, + currentResult: { + traceId, + status: "running", + conversationId: "cnv_gpt_pika_retry", + sessionId: "ses_gpt_pika_retry", + threadId: "thread_evicted", + providerProfile: "gpt.pika", + agentRun: { + adapter: "agentrun-v01", + runId: "run_gpt_pika_evicted", + commandId: "cmd_gpt_pika_evicted", + sessionId: "ses_agentrun_gpt_pika_retry", + threadId: "thread_evicted", + backendProfile: "gpt.pika", + managerUrl: "http://127.0.0.1:65535", + status: "running", + traceId, + lastSeq: 0, + providerTrace: { traceId, valuesPrinted: false }, + valuesPrinted: false + }, + valuesPrinted: false + }, + options: { + env: agentRunTestEnv(), + fetchImpl: createAgentRunRetryFetch(calls, traceId) + } + }); + + assert.equal(synced.freshSessionRetried, true); + assert.equal(synced.result.status, "completed"); + assert.equal(synced.result.agentRun.runId, "run_gpt_pika_retry"); + assert.equal(synced.result.agentRun.commandId, "cmd_gpt_pika_retry"); + assert.equal(synced.result.agentRun.freshSessionRetryOf.runId, "run_gpt_pika_evicted"); + const retryCommand = calls.find((call) => call.path === "/api/v1/runs/run_gpt_pika_retry/commands"); + assert.ok(retryCommand); + assert.equal(retryCommand.body.idempotencyKey, traceId); + assert.equal(retryCommand.body.payload.threadId, null); + assert.equal(retryCommand.body.dispatch.kind, "kubernetes-job"); + assert.equal(Object.hasOwn(retryCommand.body.dispatch.input, "commandId"), false); + assert.equal(retryCommand.body.dispatch.input.transientEnv.every((entry) => entry.sensitive === false), true); + assert.equal(JSON.stringify(retryCommand.body).includes("HWLAB_API_KEY"), false); + assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); + assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:dispatch-intent:admitted-after-session-reset")); +}); + +function agentRunTestEnv() { + return { + AGENTRUN_MGR_URL: "http://127.0.0.1:65535", + HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", + HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX: "0", + HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "NC01", + HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: sourceCommit, + HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "gpt.pika", + HWLAB_RUNTIME_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667", + HWLAB_RUNTIME_WEB_URL: "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080", + HWLAB_RUNTIME_NAMESPACE: "hwlab-v03", + HWLAB_RUNTIME_LANE: "v03", + HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", + HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", + UNIDESK_MAIN_SERVER_IP: "http://127.0.0.1:18081" + }; +} + +function createAgentRunFetch(calls, { dispatchIntent }) { + return async (url, init = {}) => { + const parsed = new URL(url); + const method = init.method ?? "GET"; + const body = init.body ? JSON.parse(String(init.body)) : null; + calls.push({ method, path: parsed.pathname, body }); + const send = (data) => new Response(JSON.stringify({ ok: true, data }), { + status: 200, + headers: { "content-type": "application/json" } + }); + if (method === "POST" && parsed.pathname === "/api/v1/sessions") { + return send({ session: { id: "ses_agentrun_gpt_pika_durable", metadata: {} } }); + } + if (method === "POST" && parsed.pathname === "/api/v1/runs") { + return send({ + id: "run_gpt_pika_durable", + status: "pending", + sessionRef: body.sessionRef, + backendProfile: body.backendProfile + }); + } + if (method === "POST" && parsed.pathname === "/api/v1/runs/run_gpt_pika_durable/commands") { + return send({ + id: "cmd_gpt_pika_durable", + runId: "run_gpt_pika_durable", + state: "pending", + type: "turn", + seq: 1, + dispatchIntent + }); + } + throw new Error(`unexpected AgentRun call ${method} ${parsed.pathname}`); + }; +} + +function createAgentRunRetryFetch(calls, traceId) { + return async (url, init = {}) => { + const parsed = new URL(url); + const method = init.method ?? "GET"; + const body = init.body ? JSON.parse(String(init.body)) : null; + calls.push({ method, path: parsed.pathname, body }); + const send = (data) => new Response(JSON.stringify({ ok: true, data }), { + status: 200, + headers: { "content-type": "application/json" } + }); + if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_evicted/commands/cmd_gpt_pika_evicted/result") { + return send({ + runId: "run_gpt_pika_evicted", + commandId: "cmd_gpt_pika_evicted", + status: "failed", + runStatus: "failed", + commandState: "failed", + terminalStatus: "failed", + failureKind: "session-store-evicted", + failureMessage: "session storage was evicted while resuming the thread", + completed: false, + lastSeq: 1 + }); + } + if (method === "POST" && parsed.pathname === "/api/v1/sessions") { + return send({ session: { id: body.sessionId, metadata: {} } }); + } + if (method === "POST" && parsed.pathname === "/api/v1/runs") { + return send({ + id: "run_gpt_pika_retry", + status: "pending", + sessionRef: body.sessionRef, + backendProfile: body.backendProfile + }); + } + if (method === "POST" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") { + return send({ + id: "cmd_gpt_pika_retry", + runId: "run_gpt_pika_retry", + state: "pending", + type: "turn", + seq: 1, + dispatchIntent: { + id: "dispatch_gpt_pika_retry", + state: "pending", + runnerJobId: "rjob_gpt_pika_retry", + attemptCount: 0, + durable: true + } + }); + } + if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/events") { + return send({ items: [] }); + } + if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") { + return send({ items: [{ + id: "cmd_gpt_pika_retry", + runId: "run_gpt_pika_retry", + state: "completed", + idempotencyKey: traceId, + payload: { traceId } + }] }); + } + if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands/cmd_gpt_pika_retry/result") { + return send({ + runId: "run_gpt_pika_retry", + commandId: "cmd_gpt_pika_retry", + status: "completed", + runStatus: "completed", + commandState: "completed", + terminalStatus: "completed", + completed: true, + reply: "GPT_PIKA_RETRY_OK", + lastSeq: 2, + sessionRef: { + sessionId: "ses_agentrun_gpt_pika_retry-reset-trcgptpikadu", + conversationId: "cnv_gpt_pika_retry", + threadId: "thread_gpt_pika_retry" + }, + traceId + }); + } + throw new Error(`unexpected AgentRun retry call ${method} ${parsed.pathname}`); + }; +} diff --git a/internal/cloud/code-agent-agentrun-result.ts b/internal/cloud/code-agent-agentrun-result.ts new file mode 100644 index 00000000..b629af2c --- /dev/null +++ b/internal/cloud/code-agent-agentrun-result.ts @@ -0,0 +1,1154 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: AgentRun command-result/event mapping, terminal projection, and trace observability helpers. + +import { randomUUID } from "node:crypto"; + +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { decorateCodeAgentSession } from "./code-agent-session-lifecycle.ts"; +import { + parsePositiveInteger, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + text +} from "./server-http-utils.ts"; +import { currentBackendPerformanceContext } from "./backend-performance.ts"; +import { + DEFAULT_AGENTRUN_MGR_URL, + agentRunJson, + isAgentRunManagerInternalServiceHost, + resolveAgentRunManagerUrl +} from "./code-agent-agentrun-runtime.ts"; +import { + agentRunErrorWithPolling, + agentRunEventCommandId, + agentRunMetricStatusForError, + agentRunPollingRetryableError, + agentRunPollingRootCause, + agentRunProjectionPollCount, + agentRunProjectionPollingState, + agentRunTerminalConsistencyGap, + agentRunTerminalResultRefreshSatisfied, + agentRunTerminalStatusFromEvents, + agentRunTraceCursorSeq, + emitAgentRunProjectionPollingSpan, + fetchAgentRunEventsForTrace, + measuredAgentRunEventsFetch, + measuredAgentRunResultFetch, + nonNegativeNumber, + recordAgentRunEventsProjectionMetrics +} from "./code-agent-agentrun-projection-polling.ts"; +import { + agentRunAlreadyTerminalCancelPayload, + agentRunCancelBlockedPayload, + agentRunCancelTerminalStatus +} from "./code-agent-agentrun-cancel.ts"; +import { + DEFAULT_AGENTRUN_REPO_URL, + agentRunAvailabilityBlocker, + agentRunProviderIdOrNull, + agentRunTraceMeta, + backendForBackendProfile, + modelForBackendProfile, + providerForBackendProfile, + requireAgentRunSourceCommit, + resolveAgentRunBackendProfile, + resolveAgentRunProviderId, + resolveAgentRunRepoUrl +} from "./code-agent-agentrun-profile.ts"; +import { + agentRunPromptEventFields, + agentRunPromptFields, + agentRunPromptMetadataFromParams, + agentRunPromptMetadataFromSession, + agentRunPromptMetadataFromText, + messageAuthorityTextValue +} from "./code-agent-agentrun-prompt.ts"; +import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; +import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts"; + +const ADAPTER_ID = "agentrun-v01"; +const ADAPTER_IDS = new Set([ADAPTER_ID, "agentrun-v02"]); +const ADAPTER_ALIASES = new Set(["agentrun", "agentrun-v0.1", "agentrun-v0.2"]); +const DEFAULT_TENANT_ID = "hwlab"; +const DEFAULT_PROJECT_ID = "pikasTech/HWLAB"; +const DEFAULT_RUNNER_NAMESPACE = "agentrun-v01"; +const DEFAULT_TIMEOUT_MS = 1_200_000; +const AGENTRUN_RUNNER_KIND = "agentrun-v01-shared-runner"; +const AGENTRUN_SESSION_MODE = "agentrun-v01-durable-session"; +const AGENTRUN_IMPLEMENTATION_TYPE = "agentrun-v01-shared-execution-infra"; +const AGENTRUN_CAPABILITY_LEVEL = "agentrun-v01-shared-code-agent-session"; +const AGENTRUN_PROVIDER_TRACE_PROTOCOL = "agentrun-v01-jsonrpc"; +const AGENTRUN_PROVIDER_TRACE_WIRE_API = "agentrun-v01-command-result"; +const THREAD_CONTINUITY_POLICY = "hwlab-agentrun-v01-reuse-runner-thread"; +const SESSION_POLICY_RUN_LOCAL = "hwlab-agentrun-v01-session-runner-reuse"; +const TERMINAL_RUN_STATUSES = new Set(["completed", "failed", "blocked", "cancelled", "canceled"]); +const HWLAB_RESOURCE_GIT_BUNDLES = Object.freeze([ + Object.freeze({ name: "hwlab-tools", subpath: "tools", target_path: "tools" }), + Object.freeze({ name: "hwlab-agent-skills", subpath: "skills", target_path: ".agents/skills" }) +]); +const HWLAB_RESOURCE_PROMPT_REFS = Object.freeze([ + Object.freeze({ name: "hwlab-v02-runtime", path: "internal/agent/prompts/hwlab-v02-runtime.md", inject: "thread-start", required: true }) +]); + +const RETRYABLE_AGENTRUN_TRANSPORT_FAILURE_KINDS = new Set([ + "agentrun-connect-failed", + "agentrun-proxy-exec-failed", + "agentrun-manager-fetch-failed", + "agentrun-timeout", + "timeout", + "fetch-failed", + "network-error" +]); + +export function isRetryableAgentRunTransportFailureKind(failureKind) { + return RETRYABLE_AGENTRUN_TRANSPORT_FAILURE_KINDS.has(normalizedFailureKind(failureKind)); +} + + +export function agentRunSessionEvidence(payload = {}) { + if (!payload?.agentRun) return {}; + return { + agentRun: { + adapter: ADAPTER_ID, + runId: payload.agentRun.runId ?? null, + commandId: payload.agentRun.commandId ?? null, + attemptId: payload.agentRun.attemptId ?? null, + runnerId: payload.agentRun.runnerId ?? null, + jobName: payload.agentRun.jobName ?? null, + namespace: payload.agentRun.namespace ?? null, + backendProfile: payload.agentRun.backendProfile ?? null, + managerUrl: payload.agentRun.managerUrl ?? DEFAULT_AGENTRUN_MGR_URL, + repoUrl: payload.agentRun.repoUrl ?? DEFAULT_AGENTRUN_REPO_URL, + status: payload.agentRun.status ?? null, + runStatus: payload.agentRun.runStatus ?? null, + commandState: payload.agentRun.commandState ?? null, + terminalStatus: payload.agentRun.terminalStatus ?? null, + traceId: payload.agentRun.traceId ?? payload.traceId ?? payload.providerTrace?.traceId ?? null, + lastSeq: payload.agentRun.lastSeq ?? 0, + providerId: payload.agentRun.providerId ?? agentRunProviderIdOrNull(), + reuseEligible: payload.agentRun.reuseEligible ?? false, + sessionId: payload.agentRun.sessionId ?? payload.sessionId ?? null, + conversationId: payload.conversationId ?? payload.agentRun.conversationId ?? null, + threadId: payload.threadId ?? payload.agentRun.threadId ?? null, + providerTrace: payload.providerTrace ?? payload.agentRun.providerTrace ?? null, + valuesPrinted: false + } + }; +} + + +export function agentRunFailureRequiresFreshSession(failureKind, failureMessage) { + const kind = String(failureKind ?? "").trim().toLowerCase().replace(/_/gu, "-"); + const message = String(failureMessage ?? ""); + return kind === "session-store-evicted" || kind === "thread-resume-failed" || /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|thread\/resume failed/iu.test(message); +} + + +export function agentRunFreshSessionFailure(result = {}) { + const code = result?.failureKind ?? result?.errorCode ?? result?.code ?? (result?.terminalStatus === "failed" ? "agentrun_failed" : null); + const message = result?.failureMessage ?? result?.blocker?.message ?? result?.error?.message ?? result?.message ?? ""; + return { + code: normalizedFailureKind(code) ?? "agentrun_failed", + message, + requiresFreshSession: agentRunFailureRequiresFreshSession(code, message), + valuesPrinted: false + }; +} + +export function providerCredentialSecretRef(profile, env) { + const profileEnvKey = String(profile ?? "").toUpperCase().replace(/[^A-Z0-9]+/gu, "_"); + return { + profile, + secretRef: { + namespace: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE, DEFAULT_RUNNER_NAMESPACE), + name: firstNonEmpty(env[`HWLAB_CODE_AGENT_AGENTRUN_${profileEnvKey}_SECRET_NAME`], env[`HWLAB_CODE_AGENT_AGENTRUN_${String(profile ?? "").toUpperCase()}_SECRET_NAME`], env.HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_SECRET_NAME, `agentrun-v01-provider-${profile}`), + keys: ["auth.json", "config.toml"], + mountPath: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_CODEX_HOME_MOUNT, "~/.codex") + } + }; +} + +export function agentRunProviderTrace({ base = {}, result = {}, terminalStatus = null } = {}) { + return { + transport: "agentrun-v01", + protocol: AGENTRUN_PROVIDER_TRACE_PROTOCOL, + wireApi: AGENTRUN_PROVIDER_TRACE_WIRE_API, + runnerKind: AGENTRUN_RUNNER_KIND, + implementationType: AGENTRUN_IMPLEMENTATION_TYPE, + command: "agentrun.v01.command.turn", + toolName: "agentrun.v01.command.turn", + source: ADAPTER_ID, + backendProfile: base.agentRun?.backendProfile ?? null, + runId: base.agentRun?.runId ?? result?.runId ?? null, + commandId: base.agentRun?.commandId ?? result?.commandId ?? null, + runnerId: base.agentRun?.runnerId ?? result?.runnerId ?? null, + jobName: base.agentRun?.jobName ?? result?.jobName ?? null, + namespace: base.agentRun?.namespace ?? result?.namespace ?? null, + traceId: base.traceId ?? base.agentRun?.traceId ?? null, + threadId: result?.sessionRef?.threadId ?? agentRunEffectiveThreadId(base), + turnId: result?.turnId ?? null, + terminalStatus: terminalStatus ?? result?.terminalStatus ?? null, + failureKind: result?.failureKind ?? null, + failureMessage: result?.failureMessage ?? result?.blocker?.message ?? null, + willRetry: result?.willRetry === true ? true : undefined, + valuesPrinted: false + }; +} + +export function agentRunFailureAttribution({ code, message, canceled = false } = {}) { + if (canceled) { + return { + category: "canceled", + retryable: true, + userMessage: "AgentRun 请求已取消。", + summary: message || "AgentRun command was canceled." + }; + } + if (code === "provider-invalid-tool-call" || /invalid function arguments json string|invalid_prompt|tool_call_id/iu.test(String(message ?? ""))) { + return { + category: "provider_invalid_tool_call", + retryable: true, + userMessage: "AgentRun/provider 返回了无效 tool-call arguments JSON；这不是 HWPOD 或 Cloud API 端点失败，请查看 providerTrace.failureKind 和 runnerTrace 定位上游 tool_call_id。", + summary: message || "AgentRun/provider returned invalid tool-call arguments JSON." + }; + } + if (code === "thread-resume-failed") { + return { + category: "thread_resume_failed", + retryable: true, + userMessage: "AgentRun 复用的 thread 已失效；当前 turn 会按失败终止，Workbench 会清理 stale continuation 指针，下一轮自动从新 session/thread 启动。", + summary: message || "AgentRun thread/resume failed for an existing thread." + }; + } + return { + category: "agentrun_failed", + retryable: true, + userMessage: "AgentRun 请求失败；请查看 runnerTrace、providerTrace 和 agentRun 字段定位 runId/commandId/jobName。", + summary: message || "AgentRun command failed." + }; +} + +export function agentRunLongLivedSessionGate(base = {}) { + const session = agentRunSessionSummary(base, "idle"); + return { + status: "pass", + pass: true, + requiredCapability: AGENTRUN_CAPABILITY_LEVEL, + currentCapability: AGENTRUN_IMPLEMENTATION_TYPE, + sessionId: session.sessionId ?? null, + sessionStatus: session.status ?? null, + runnerKind: AGENTRUN_RUNNER_KIND, + provider: providerForBackendProfile(base.agentRun?.backendProfile ?? "deepseek"), + sessionMode: AGENTRUN_SESSION_MODE, + implementationType: AGENTRUN_IMPLEMENTATION_TYPE, + adapter: ADAPTER_ID, + delegatedToAgentRun: true, + codexStdio: false, + blockers: [], + valuesPrinted: false + }; +} + +export function agentRunToolCalls(result = {}, status = "completed", replyTextOverride = null) { + const replyText = messageAuthorityTextValue(replyTextOverride) || agentRunReplyText(result); + return [{ + name: "agentrun.v01.command.turn", + status, + runId: result?.runId ?? null, + commandId: result?.commandId ?? null, + outputSummary: replyText ? `assistantChars=${replyText.length}` : null, + valuesPrinted: false + }]; +} + +export function agentRunReplyText(result = {}) { + for (const value of [result?.reply, result?.assistantMessage, result?.finalResponse, result?.content, result?.text, result?.message]) { + const text = messageAuthorityTextValue(value); + if (text) return text; + } + return ""; +} + +export function agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent = true }) { + const terminalStatus = String(result?.terminalStatus ?? ""); + const normalizedTerminalStatus = terminalStatus.trim().toLowerCase().replace(/_/gu, "-"); + const retryableInterruption = normalizedTerminalStatus !== "completed" && isRetryableProviderInterruptionPayload(result); + const terminal = !retryableInterruption && ["completed", "failed", "blocked", "cancelled", "canceled"].includes(normalizedTerminalStatus); + if (!terminal) { + if (retryableInterruption && appendResultEvent) { + traceStore.append(traceId, agentRunProviderRetryEvent({ base, payload: result, status: "retrying", label: "agentrun:provider-retry:provider-stream-disconnected" }), base.agentRun); + } + const runningMapping = retryableInterruption ? { + ...base.agentRun, + status: "retrying", + commandState: "retrying", + failureKind: normalizedFailureKind(result?.failureKind), + willRetry: true + } : base.agentRun; + const running = decorateAgentRunRunningResult({ base, mapping: runningMapping, traceStore, traceId }); + if (!retryableInterruption) return running; + return { + ...running, + status: "retrying", + providerTrace: agentRunProviderTrace({ base, result, terminalStatus }), + transient: { failureKind: normalizedFailureKind(result?.failureKind), willRetry: true, valuesRedacted: true }, + valuesPrinted: false + }; + } + const now = nowIso(); + const runnerTrace = traceStore.snapshot(traceId, agentRunTraceMeta({}, {})); + const terminalEventCreatedAt = agentRunResultTraceCreatedAt(runnerTrace, now); + const providerTrace = agentRunProviderTrace({ base, result, terminalStatus }); + const replyText = agentRunReplyText(result); + if (normalizedTerminalStatus === "completed") { + const traceReplyText = agentRunTraceTerminalAssistantText(runnerTrace); + const completedReplyText = replyText || traceReplyText; + const finalResponse = completedReplyText ? agentRunCompletedFinalResponse({ base, result, traceId, now, replyText: completedReplyText }) : null; + const traceSummary = agentRunCompletedTraceSummary({ base, runnerTrace, finalResponse, traceId }); + if (appendResultEvent) { + traceStore.append(traceId, agentRunTraceEvent({ + type: "result", + status: "completed", + label: "agentrun:result:completed", + createdAt: terminalEventCreatedAt, + message: "AgentRun result is ready for HWLAB short-connection polling.", + runId: base.agentRun.runId, + commandId: base.agentRun.commandId, + terminal: true, + valuesPrinted: false + }, base.agentRun)); + } + return { + ...base, + status: "completed", + messageId: finalResponse?.messageId ?? base.messageId ?? `msg_${traceId.slice(4)}`, + threadId: agentRunEffectiveThreadId(base), + updatedAt: now, + workspace: base.workspace ?? "/home/agentrun/workspace", + sandbox: base.sandbox ?? "danger-full-access", + session: agentRunSessionSummary(base, "idle"), + sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true), + runner: agentRunRunnerSummary(base.agentRun), + runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})), + toolCalls: agentRunToolCalls(result, "completed", completedReplyText), + skills: { status: "delegated", provider: ADAPTER_ID, count: 0, items: [], valuesPrinted: false }, + longLivedSessionGate: agentRunLongLivedSessionGate(base), + providerTrace, + finalResponse: finalResponse ?? null, + traceSummary, + reply: finalResponse ? { + messageId: finalResponse.messageId, + role: "assistant", + content: finalResponse.text, + createdAt: finalResponse.createdAt + } : null, + usage: null, + agentRun: { ...base.agentRun, terminalStatus, completed: true, reuseEligible: true, providerTrace, valuesPrinted: false }, + valuesPrinted: false + }; + } + const canceled = terminalStatus === "cancelled"; + const code = canceled ? "agentrun_canceled" : result?.failureKind ?? (terminalStatus === "blocked" ? "agentrun_blocked" : "agentrun_failed"); + const message = result?.failureMessage ?? result?.blocker?.message ?? (canceled ? "AgentRun command was canceled" : "AgentRun command failed"); + const attribution = agentRunFailureAttribution({ code, message, canceled }); + const finalResponse = agentRunTerminalFailureFinalResponse({ + base, + traceId, + now, + status: canceled ? "canceled" : "failed", + text: firstNonEmpty(message, attribution.summary, attribution.userMessage) + }); + const traceSummary = agentRunTerminalFailureTraceSummary({ base, runnerTrace, finalResponse, traceId, terminalStatus: canceled ? "canceled" : terminalStatus || "failed" }); + if (appendResultEvent) { + traceStore.append(traceId, agentRunTraceEvent({ + type: "result", + status: canceled ? "canceled" : "failed", + label: `agentrun:result:${canceled ? "canceled" : terminalStatus || "failed"}`, + createdAt: terminalEventCreatedAt, + errorCode: code, + message: attribution.summary, + runId: base.agentRun.runId, + commandId: base.agentRun.commandId, + terminal: true, + valuesPrinted: false + }, base.agentRun)); + } + const partialContext = partialAgentRunContext(runnerTrace); + const freshSessionRequired = !canceled && agentRunFailureRequiresFreshSession(code, message); + const failedSessionStatus = freshSessionRequired ? "thread-resume-failed" : "failed"; + const resumableAfterFailure = !canceled && !freshSessionRequired && Boolean(agentRunEffectiveThreadId(base)); + return { + ...base, + status: canceled ? "canceled" : "failed", + messageId: finalResponse.messageId, + threadId: agentRunEffectiveThreadId(base), + canceled, + updatedAt: now, + session: agentRunSessionSummary(base, canceled ? "canceled" : failedSessionStatus), + sessionReuse: agentRunSessionReuseSummary(base, base.agentRun.reused === true, { status: canceled ? undefined : freshSessionRequired ? "thread-resume-failed" : resumableAfterFailure ? "failed-resumable" : "failed-requires-new-session" }), + runner: agentRunRunnerSummary(base.agentRun), + runnerTrace: partialContext ? { ...runnerTrace, partialContext } : runnerTrace, + toolCalls: agentRunToolCalls(result, canceled ? "canceled" : "failed"), + skills: { status: "delegated", provider: ADAPTER_ID, count: 0, items: [], valuesPrinted: false }, + providerTrace, + finalResponse, + traceSummary, + error: { + code, + layer: "agentrun", + category: attribution.category, + retryable: attribution.retryable, + message, + userMessage: attribution.userMessage, + traceId, + route: "/v1/agent/chat", + toolName: "agentrun.manual-dispatch" + }, + blocker: canceled ? null : { + code, + layer: "agentrun", + category: attribution.category, + retryable: attribution.retryable, + summary: attribution.summary, + traceId, + route: "/v1/agent/chat", + toolName: "agentrun.manual-dispatch" + }, + agentRun: { ...base.agentRun, terminalStatus, completed: false, reuseEligible: resumableAfterFailure, providerTrace, valuesPrinted: false }, + reuseEligible: resumableAfterFailure, + ...(partialContext ? { partialContext } : {}), + valuesPrinted: false + }; +} + +export function agentRunCompletedFinalResponse({ base, result, traceId, now, replyText = null }) { + const textValue = messageAuthorityTextValue(replyText) || agentRunReplyText(result); + return { + text: textValue, + textChars: textValue.length, + role: "assistant", + status: "completed", + traceId, + messageId: base.messageId ?? base.reply?.messageId ?? `msg_${traceId.slice(4)}`, + createdAt: base.reply?.createdAt ?? base.createdAt ?? now, + updatedAt: now, + valuesPrinted: false + }; +} + +export function agentRunTraceTerminalAssistantText(runnerTrace = {}) { + const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; + for (let index = events.length - 1; index >= 0; index -= 1) { + const event = events[index]; + if (!event || typeof event !== "object") continue; + if (!isAgentRunAssistantTraceEvent(event)) continue; + if (!(event.terminal === true || event.final === true || event.replyAuthority === true)) continue; + const textValue = messageAuthorityTextValue(event.finalResponse ?? event.text ?? event.content ?? event.message ?? event.summary ?? event.payload?.text ?? event.payload?.content ?? event.payload?.message); + if (textValue) return textValue; + } + return ""; +} + +export function isAgentRunAssistantTraceEvent(event = {}) { + const type = String(event.type ?? event.eventType ?? "").trim().toLowerCase(); + if (type === "assistant" || type === "assistant_message") return true; + return /assistant:message|assistant_message/u.test(String(event.label ?? "").toLowerCase()); +} + +export function agentRunTerminalFailureFinalResponse({ base, traceId, now, status, text }) { + const textValue = String(text ?? "").trim(); + return { + text: textValue || "AgentRun command failed.", + textChars: (textValue || "AgentRun command failed.").length, + role: "assistant", + status, + traceId, + messageId: base.messageId ?? base.reply?.messageId ?? `msg_${traceId.slice(4)}`, + createdAt: base.reply?.createdAt ?? base.createdAt ?? now, + updatedAt: now, + valuesPrinted: false + }; +} + +export function agentRunTerminalFailureTraceSummary({ base, runnerTrace, finalResponse, traceId, terminalStatus }) { + const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; + const lastSeq = Math.max(0, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); + return { + traceId, + source: "agentrun-command-result", + sourceEventCount: Number(runnerTrace?.eventCount ?? events.length ?? 0), + terminalStatus, + finalAssistantRow: { + role: finalResponse.role, + status: finalResponse.status, + textChars: finalResponse.textChars, + textPreview: finalResponse.text.slice(0, 240), + messageId: finalResponse.messageId, + valuesPrinted: false + }, + agentRun: { + runId: base.agentRun?.runId ?? null, + commandId: base.agentRun?.commandId ?? null, + lastSeq, + valuesPrinted: false + }, + valuesPrinted: false + }; +} + +export function agentRunCompletedTraceSummary({ base, runnerTrace, finalResponse, traceId }) { + const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; + const lastSeq = Math.max(0, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); + return { + traceId, + source: "agentrun-command-result", + sourceEventCount: Number(runnerTrace?.eventCount ?? events.length ?? 0), + terminalStatus: "completed", + finalAssistantRow: finalResponse ? { + role: finalResponse.role, + status: finalResponse.status, + textChars: finalResponse.textChars, + textPreview: finalResponse.text.slice(0, 240), + messageId: finalResponse.messageId, + valuesPrinted: false + } : null, + agentRun: { + runId: base.agentRun?.runId ?? null, + commandId: base.agentRun?.commandId ?? null, + lastSeq, + valuesPrinted: false + }, + valuesPrinted: false + }; +} + +export function partialAgentRunContext(runnerTrace = {}) { + const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; + const assistantMessages = events + .filter((event) => event?.type === "assistant" && String(event.text ?? event.message ?? "").trim()) + .map((event) => String(event.text ?? event.message).trim()) + .slice(-4); + const toolEvidence = events + .filter((event) => event?.type === "tool_call" && String(event.status ?? "") === "completed") + .map((event) => [event.toolName ?? event.label ?? "tool", event.command ?? event.outputSummary ?? event.stdoutSummary ?? event.message ?? ""].filter(Boolean).join(": ")) + .filter(Boolean) + .slice(-6); + if (assistantMessages.length === 0 && toolEvidence.length === 0) return null; + return { + status: "partial-before-terminal", + summary: "AgentRun terminal 前已有可延续的 assistant/tool 上下文；后续同 conversation/session/thread 轮次必须能继续使用。", + assistantMessages, + toolEvidence, + valuesPrinted: false + }; +} + +export function agentRunResultTraceCreatedAt(runnerTrace = {}, fallback) { + const last = String(runnerTrace?.lastEvent?.createdAt ?? runnerTrace?.updatedAt ?? ""); + return Number.isFinite(Date.parse(last)) ? last : fallback; +} + +export function decorateAgentRunRunningResult({ base, mapping, traceStore, traceId }) { + return { + ...base, + status: "running", + threadId: agentRunEffectiveThreadId({ agentRun: mapping }), + accepted: true, + shortConnection: true, + updatedAt: nowIso(), + session: agentRunSessionSummary({ ...base, agentRun: mapping }, "running"), + sessionReuse: agentRunSessionReuseSummary({ ...base, agentRun: mapping }, mapping.reused === true), + runner: agentRunRunnerSummary(mapping), + runnerTrace: traceStore.snapshot(traceId, agentRunTraceMeta({}, {})), + agentRun: { ...mapping, adapter: ADAPTER_ID, valuesPrinted: false }, + valuesPrinted: false + }; +} + +export function appendAgentRunEventsToTrace(traceStore, traceId, events, mapping = {}) { + for (const event of events) { + if (isForeignAgentRunCommandEvent(event, mapping)) continue; + const normalized = mapAgentRunEvent(event, mapping); + if (normalized) traceStore.append(traceId, normalized, agentRunTraceMeta({}, {})); + } +} + +export function isForeignAgentRunCommandEvent(event, mapping = {}) { + const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : ""; + if (!currentCommandId) return false; + const eventCommandId = agentRunEventCommandId(event); + return Boolean(eventCommandId && eventCommandId !== currentCommandId); +} + +export function mapAgentRunEvent(event, mapping = {}) { + if (!event || typeof event !== "object") return null; + const payload = event.payload && typeof event.payload === "object" ? event.payload : {}; + const base = { + createdAt: event.createdAt ?? null, + source: "agentrun", + sourceSeq: event.seq ?? null, + backend: backendForBackendProfile(mapping.backendProfile ?? payload.backendProfile ?? payload.providerProfile ?? "deepseek"), + runId: event.runId ?? mapping.runId ?? null, + commandId: payload.commandId ?? mapping.commandId ?? null, + attemptId: payload.attemptId ?? mapping.attemptId ?? null, + runnerId: payload.runnerId ?? mapping.runnerId ?? null, + jobName: payload.jobName ?? mapping.jobName ?? null, + namespace: payload.namespace ?? mapping.namespace ?? null, + valuesPrinted: false + }; + if (event.type === "backend_status") { + const phase = String(payload.phase ?? "status"); + return { + ...base, + type: "backend", + eventType: "backend", + status: "running", + label: `agentrun:backend:${phase}`, + message: textPayload(payload, phase), + waitingFor: waitingForForPhase(phase), + details: agentRunBackendStatusDetails(phase, payload) + }; + } + if (event.type === "assistant_message") { + const terminal = payload.replyAuthority === true || payload.final === true; + return { + ...base, + type: "assistant", + eventType: "assistant", + status: terminal ? "completed" : "running", + label: "agentrun:assistant:message", + message: textPayload(payload, "assistant message"), + text: textPayload(payload, ""), + itemId: payload.itemId ?? null, + messageIndex: payload.messageIndex ?? null, + messageCount: payload.messageCount ?? null, + replyAuthority: payload.replyAuthority === true, + final: payload.final === true, + terminal + }; + } + if (event.type === "tool_call") { + const commandExecution = agentRunCommandExecutionEvent(base, payload); + if (commandExecution) return commandExecution; + const preview = agentRunToolCallPreviewSummary(payload); + const toolName = firstNonEmpty( + payload.toolName, + payload.name, + payload.item?.type, + payload.type, + preview.toolName, + preview.name, + preview.type, + isProtocolToolLifecycleMethod(payload.method) ? null : payload.method, + "tool call" + ); + const command = firstNonEmpty(payload.command, payload.commandLine, payload.item?.command, payload.item?.commandLine, preview.command, preview.commandLine); + const output = firstNonEmpty(payload.outputSummary, payload.stdoutSummary, preview.outputSummary, preview.stdoutSummary, payload.summary?.text, payload.itemPreview); + return { + ...base, + type: "tool_call", + eventType: "tool_call", + status: String(payload.status ?? preview.status ?? (payload.method === "item/completed" ? "completed" : "running")), + label: `agentrun:tool:${String(toolName)}`, + itemId: payload.item?.id ?? payload.itemId ?? preview.itemId ?? null, + toolName, + command, + exitCode: Number.isInteger(payload.item?.exitCode) ? payload.item.exitCode : Number.isInteger(payload.exitCode) ? payload.exitCode : undefined, + durationMs: typeof payload.item?.durationMs === "number" ? payload.item.durationMs : typeof payload.durationMs === "number" ? payload.durationMs : undefined, + stdoutSummary: output, + outputSummary: output, + message: output || command || textPayload(payload, "tool call"), + outputBytes: typeof payload.outputBytes === "number" ? payload.outputBytes : payload.summary?.outputBytes, + outputTruncated: payload.outputTruncated === true || payload.summary?.outputTruncated === true + }; + } + if (event.type === "command_output") { + const stream = payload.stream === "stderr" ? "stderr" : "stdout"; + return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true }; + } + if (event.type === "diff") { + return { ...base, type: "diff", eventType: "status", status: "running", label: "agentrun:diff", message: textPayload(payload, "diff") }; + } + if (event.type === "error") { + const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? "backend") ?? "backend"; + const normalizedPayload = { ...payload, failureKind, errorCode: failureKind }; + if (isRetryableProviderInterruptionPayload(normalizedPayload, { activeError: true })) return agentRunProviderRetryEvent({ base, payload: { ...normalizedPayload, willRetry: true }, status: "retrying", label: `agentrun:provider-retry:${failureKind}` }); + return { ...base, type: "error", eventType: "error", status: "failed", label: `agentrun:error:${failureKind}`, errorCode: failureKind, failureKind, willRetry: payload.willRetry === true ? true : undefined, message: textPayload(payload, "AgentRun error") }; + } + if (event.type === "terminal_status") { + const terminalStatus = String(payload.terminalStatus ?? event.terminalStatus ?? payload.status ?? event.status ?? "failed"); + const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? event.failureKind ?? event.errorCode) ?? null; + const normalizedPayload = failureKind ? { ...payload, failureKind, errorCode: failureKind } : payload; + if (isRetryableProviderInterruptionPayload(normalizedPayload)) return agentRunProviderRetryEvent({ base, payload: normalizedPayload, status: "retrying", label: `agentrun:provider-retry:${terminalStatus}` }); + return { ...base, type: "result", eventType: "terminal", status: terminalStatus === "cancelled" ? "canceled" : terminalStatus, label: `agentrun:terminal:${terminalStatus}`, errorCode: failureKind, failureKind, willRetry: payload.willRetry === true || event.willRetry === true ? true : undefined, message: textPayload({ ...event, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true }; + } + return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${String(event.type ?? "unknown")}`, message: textPayload(payload, "AgentRun event") }; +} + +export function agentRunProviderRetryEvent({ base = {}, payload = {}, status = "retrying", label = "agentrun:provider-retry" } = {}) { + const failureKind = normalizedFailureKind(payload.failureKind ?? payload.errorCode ?? "provider-stream-disconnected"); + const retryMessage = providerRetryMessage(textPayload(payload, "Provider stream disconnected; AgentRun will retry."), payload); + return { + ...base, + type: "provider_retry", + eventType: "backend", + status, + label, + errorCode: failureKind, + failureKind, + willRetry: payload.willRetry === true, + retryable: true, + transient: true, + terminal: false, + retryAttempt: numberOrNull(payload.retryAttempt), + retryMax: numberOrNull(payload.retryMax), + retryDelayMs: numberOrNull(payload.retryDelayMs), + nextRetryAt: firstNonEmpty(payload.nextRetryAt) || null, + retryOf: payload.retryOf ?? null, + terminalStatus: payload.terminalStatus ?? null, + message: retryMessage, + valuesPrinted: false + }; +} + +export function providerRetryMessage(message, payload = {}) { + const attempt = numberOrNull(payload.retryAttempt); + const max = numberOrNull(payload.retryMax); + const delayMs = numberOrNull(payload.retryDelayMs); + const nextRetryAt = firstNonEmpty(payload.nextRetryAt) || null; + const parts = []; + if (attempt !== null && max !== null) parts.push(`retry ${attempt}/${max}`); + else if (attempt !== null) parts.push(`retry attempt ${attempt}`); + else if (max !== null) parts.push(`retry max ${max}`); + if (delayMs !== null) parts.push(`after ${delayMs}ms`); + if (nextRetryAt) parts.push(`next=${nextRetryAt}`); + if (parts.length === 0) return message; + const text = String(message || "Provider stream disconnected; AgentRun will retry.").trim(); + if (attempt !== null && max !== null && text.includes(`${attempt}/${max}`)) return text; + return text.replace(/[.。]\s*$/u, "") + `; ${parts.join(" ")}.`; +} + +export function isRetryableProviderInterruptionPayload(payload = {}, options = {}) { + const kind = normalizedFailureKind(payload?.failureKind ?? payload?.errorCode ?? payload?.code); + if (kind === "provider-stream-disconnected") return payload?.willRetry === true; + if (kind === "provider-unavailable") return payload?.willRetry === true || (options.activeError === true && payload?.terminal !== true && payload?.final !== true); + if (isRetryableAgentRunTransportFailureKind(kind)) return payload?.willRetry === true; + return false; +} + +export function normalizedFailureKind(value) { + const normalized = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-") || null; + if (normalized === "agentrun-unreachable") return "agentrun-connect-failed"; + return normalized; +} + +export function numberOrNull(value) { + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +export function agentRunTraceEvent(event = {}, backendSource = {}) { + const source = backendSource && typeof backendSource === "object" ? backendSource : { backendProfile: backendSource }; + const backendProfile = firstNonEmpty(event.backendProfile, source.backendProfile, source.providerProfile, source.agentRun?.backendProfile); + return { + ...event, + eventType: event.eventType ?? agentRunManualEventType(event), + backend: event.backend ?? (backendProfile ? backendForBackendProfile(backendProfile) : null), + backendProfile: event.backendProfile ?? backendProfile ?? null, + valuesPrinted: event.valuesPrinted ?? false + }; +} + +export function agentRunManualEventType(event = {}) { + const type = String(event.type ?? "").trim().toLowerCase(); + if (type === "request") return "request"; + if (type === "cancel" || type === "result") return "terminal"; + if (type === "assistant") return "assistant"; + if (type === "tool" || type === "tool_call") return "tool_call"; + if (type === "error") return "error"; + if (type === "backend") return "backend"; + const raw = String(`${event.label ?? ""} ${event.status ?? ""}`).toLowerCase(); + if (/request|accepted/u.test(raw)) return "request"; + if (/cancel|terminal|complete|final/u.test(raw)) return "terminal"; + if (/assistant|message|reply/u.test(raw)) return "assistant"; + if (/error|fail|timeout/u.test(raw)) return "error"; + if (/backend|runner|agentrun/u.test(raw)) return "backend"; + return "status"; +} + +export function isProtocolToolLifecycleMethod(value) { + return /^item\/(started|completed)$/u.test(String(value ?? "")); +} + +export function agentRunToolCallPreviewSummary(payload = {}) { + const text = firstNonEmpty(payload.itemPreview, payload.summary?.text); + return { + itemId: jsonPreviewStringField(text, "itemId"), + type: jsonPreviewStringField(text, "type"), + toolName: jsonPreviewStringField(text, "toolName"), + name: jsonPreviewStringField(text, "name"), + status: jsonPreviewStringField(text, "status"), + command: jsonPreviewStringField(text, "command"), + commandLine: jsonPreviewStringField(text, "commandLine"), + outputSummary: jsonPreviewStringField(text, "outputSummary"), + stdoutSummary: jsonPreviewStringField(text, "stdoutSummary") + }; +} + +export function jsonPreviewStringField(text, key) { + const body = String(text ?? ""); + if (!body || !/^[A-Za-z][A-Za-z0-9_]*$/u.test(key)) return null; + const match = body.match(new RegExp(`"${key}"\\s*:\\s*"((?:\\\\.|[^"\\\\])*)"`, "u")); + if (!match) return null; + try { + const parsed = JSON.parse(`"${match[1]}"`); + return typeof parsed === "string" && parsed.trim() ? parsed.trim() : null; + } catch { + return match[1]?.trim() || null; + } +} + +export function agentRunBackendStatusDetails(phase, payload = {}) { + if (phase === "initial-prompt-assembly") { + return compactObject({ + initialPromptInjected: payload.initialPromptInjected === true, + reason: firstNonEmpty(payload.reason), + initialPrompt: agentRunResourceSummary(payload.initialPrompt), + valuesPrinted: false + }); + } + if (phase === "resource-bundle-materialized") { + return compactObject({ + kind: firstNonEmpty(payload.kind), + commitId: firstNonEmpty(payload.commitId), + bundles: agentRunResourceSummary(payload.bundles), + tools: agentRunResourceSummary(payload.tools), + promptRefs: agentRunResourceSummary(payload.promptRefs), + skillDirs: agentRunResourceSummary(payload.skillDirs), + initialPrompt: agentRunResourceSummary(payload.initialPrompt), + valuesPrinted: false + }); + } + return undefined; +} + +export function agentRunResourceSummary(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + return compactObject({ + available: typeof value.available === "boolean" ? value.available : undefined, + count: Number.isInteger(value.count) ? value.count : undefined, + bytes: Number.isInteger(value.bytes) ? value.bytes : undefined, + sha256: firstNonEmpty(value.sha256), + promptRefCount: Number.isInteger(value.promptRefCount) ? value.promptRefCount : undefined, + skillCount: Number.isInteger(value.skillCount) ? value.skillCount : undefined, + toolCount: Number.isInteger(value.toolCount) ? value.toolCount : undefined, + names: Array.isArray(value.names) ? value.names.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, + targetPaths: Array.isArray(value.targetPaths) ? value.targetPaths.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, + skillNames: Array.isArray(value.skillNames) ? value.skillNames.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, + requiredMissing: Array.isArray(value.requiredMissing) ? value.requiredMissing.map((item) => firstNonEmpty(item)).filter(Boolean).slice(0, 20) : undefined, + valuesPrinted: false + }); +} + +export function compactObject(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const result = {}; + for (const [key, item] of Object.entries(value)) { + if (item === undefined || item === null) continue; + if (Array.isArray(item) && item.length === 0) continue; + if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue; + result[key] = item; + } + return Object.keys(result).length > 0 ? result : undefined; +} + +export function agentRunCommandExecutionEvent(base, payload = {}) { + const item = payload.item && typeof payload.item === "object" ? payload.item : null; + const payloadToolType = firstNonEmpty(payload.toolName, payload.type, payload.name); + if (item?.type !== "commandExecution" && payloadToolType !== "commandExecution") return null; + const method = String(payload.method ?? ""); + const completed = method === "item/completed" || payload.status === "completed" || item?.status === "completed"; + const output = firstNonEmpty(item?.aggregatedOutput, item?.stdout, item?.output, payload.outputSummary, payload.stdoutSummary, payload.summary?.text, payload.itemPreview); + return { + ...base, + type: "tool_call", + status: completed ? "completed" : "started", + label: completed ? "item/commandExecution:completed" : "item/commandExecution:started", + toolName: "commandExecution", + itemId: item?.id ?? payload.itemId ?? null, + command: firstNonEmpty(item?.command, item?.commandLine, payload.command, payload.commandLine), + exitCode: Number.isInteger(item?.exitCode) ? item.exitCode : Number.isInteger(payload.exitCode) ? payload.exitCode : undefined, + durationMs: typeof item?.durationMs === "number" ? item.durationMs : typeof payload.durationMs === "number" ? payload.durationMs : undefined, + outputBytes: typeof payload.outputBytes === "number" ? payload.outputBytes : payload.summary?.outputBytes, + stdoutSummary: output, + outputSummary: output, + outputTruncated: payload.outputTruncated === true || payload.summary?.outputTruncated === true, + message: output || textPayload(payload, "commandExecution") + }; +} + +export function nowMs() { + if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); + return Date.now(); +} + +export function agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params = {} }) { + if (dispatchIntent?.durable !== true) { + throw new Error("AgentRun mapping requires a confirmed durable dispatch intent."); + } + const threadReused = Boolean(safeOpaqueId(params.threadId)); + return { + adapter: ADAPTER_ID, + managerUrl, + backendProfile, + providerId: resolveAgentRunProviderId(env), + runId: run.id, + commandId: command.id, + attemptId: null, + runnerId: null, + runnerJobId: dispatchIntent.runnerJobId, + jobName: null, + namespace: firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE, env.AGENTRUN_RUNTIME_NAMESPACE, DEFAULT_RUNNER_NAMESPACE), + status: `dispatch-intent-${dispatchIntent.state}`, + dispatchIntentId: dispatchIntent.id, + dispatchIntentState: dispatchIntent.state, + durableDispatch: true, + runStatus: run.status ?? null, + commandState: command.state ?? null, + terminalStatus: null, + lastSeq: 0, + traceId, + ...codeAgentOtelTraceFields(traceId, env), + sessionId: run?.sessionRef?.sessionId ?? null, + conversationId: run?.sessionRef?.conversationId ?? null, + threadId: run?.sessionRef?.threadId ?? null, + runnerReused: false, + runnerJobCount: dispatchIntent.state === "dispatched" ? 1 : 0, + threadReused, + persistentResume: threadReused, + reused: false, + reuseEligible: true, + createdAt: startedAt, + updatedAt: nowIso(), + valuesPrinted: false + }; +} + +export function agentRunReusedMapping({ previous = {}, run = {}, command = {}, traceId, startedAt, backendProfile, managerUrl, env }) { + const threadReused = Boolean(safeOpaqueId(run?.sessionRef?.threadId ?? previous.threadId)); + return { + ...previous, + adapter: ADAPTER_ID, + managerUrl, + backendProfile, + providerId: previous.providerId ?? resolveAgentRunProviderId(env), + runId: previous.runId ?? run?.id, + commandId: command.id, + attemptId: previous.attemptId ?? null, + runnerId: previous.runnerId ?? null, + runnerJobId: previous.runnerJobId ?? null, + jobName: previous.jobName ?? null, + namespace: previous.namespace ?? DEFAULT_RUNNER_NAMESPACE, + status: "runner-job-reused", + runStatus: run?.status ?? previous.runStatus ?? null, + commandState: command.state ?? null, + terminalStatus: null, + lastSeq: previous.lastSeq ?? 0, + runnerJobCount: Number(previous.runnerJobCount ?? 0), + traceId, + sessionId: run?.sessionRef?.sessionId ?? previous.sessionId ?? null, + conversationId: run?.sessionRef?.conversationId ?? previous.conversationId ?? null, + threadId: run?.sessionRef?.threadId ?? previous.threadId ?? null, + runnerReused: true, + threadReused, + persistentResume: threadReused, + reused: true, + reuseEligible: true, + createdAt: previous.createdAt ?? startedAt, + updatedAt: nowIso(), + valuesPrinted: false + }; +} + +export function agentRunResultRefs(result = {}) { + const refs = {}; + for (const key of ["attemptId", "runnerId", "jobName", "namespace", "runnerJobCount"]) { + if (result?.[key] !== undefined && result?.[key] !== null) refs[key] = result[key]; + } + if (result?.sessionRef && typeof result.sessionRef === "object") { + if (result.sessionRef.sessionId) refs.sessionId = result.sessionRef.sessionId; + if (result.sessionRef.conversationId) refs.conversationId = result.sessionRef.conversationId; + if (result.sessionRef.threadId) refs.threadId = result.sessionRef.threadId; + } + return refs; +} + +export function withAgentRunRunnerJobCount(mapping = {}) { + if (!mapping || typeof mapping !== "object") return mapping; + const count = Number(mapping.runnerJobCount); + if (Number.isFinite(count) && count >= 0) return mapping; + const runnerJobEnsured = mapping.reused === true || mapping.status === "runner-job-ensured"; + if (runnerJobEnsured && (mapping.runnerJobId || mapping.jobName || mapping.attemptId || mapping.runnerId)) { + return { ...mapping, runnerJobCount: 1 }; + } + return mapping; +} + +export function hwlabSessionIdForParams(params = {}) { + return safeSessionId(params.sessionId) || null; +} + +export function scopedAgentRunSessionIdForParams(params = {}, traceId) { + const baseSessionId = hwlabSessionIdForParams(params, traceId); + if (!safeSessionId(baseSessionId)) { + throw adapterError( + "agentrun_session_id_required", + "AgentRun persistent session/PVC requires params.sessionId; refusing to derive sessionRef.sessionId from traceId because that would create a new PVC per turn." + ); + } + const base = String(baseSessionId).replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session"; + return `ses_agentrun_${base}`; +} + +export function agentRunSessionId(traceId) { + return `ses_agentrun_${String(safeTraceId(traceId) || `trc_${randomUUID()}`).slice(4)}`; +} + +export function agentRunEffectiveThreadId(base = {}) { + const agentRun = base?.agentRun && typeof base.agentRun === "object" ? base.agentRun : null; + if (agentRun && Object.hasOwn(agentRun, "threadId")) return safeOpaqueId(agentRun.threadId) || null; + return safeOpaqueId(base?.threadId) || null; +} + +export function agentRunSessionSummary(base, status) { + return decorateCodeAgentSession({ + sessionId: base.sessionId ?? base.agentRun?.sessionId ?? null, + conversationId: base.conversationId ?? base.agentRun?.conversationId ?? null, + threadId: agentRunEffectiveThreadId(base), + status, + sessionMode: AGENTRUN_SESSION_MODE, + runnerKind: AGENTRUN_RUNNER_KIND, + lastTraceId: base.traceId ?? base.agentRun?.traceId ?? null, + longLivedSession: true, + codexStdio: false, + delegatedToAgentRun: true, + writeCapable: true, + durable: true, + durableSession: true, + idleTimeoutMs: parsePositiveInteger(base.agentRun?.idleTimeoutMs, 600000), + secretMaterialStored: false, + valuesRedacted: true + }); +} + +export function agentRunThreadReused(base = {}, options = {}) { + if (options.status === "failed-requires-new-session") return false; + if (base.agentRun?.threadReused === true || base.agentRun?.persistentResume === true) return true; + return Boolean(agentRunEffectiveThreadId(base)); +} + +export function agentRunSessionReuseSummary(base, reused, options = {}) { + const runnerReused = Boolean(reused); + const threadReused = agentRunThreadReused(base, options); + const persistentResume = threadReused; + const sessionReused = runnerReused || persistentResume; + return { + sessionId: base.sessionId ?? base.agentRun?.sessionId ?? null, + conversationId: base.conversationId ?? base.agentRun?.conversationId ?? null, + threadId: agentRunEffectiveThreadId(base), + reused: sessionReused, + status: options.status ?? (runnerReused ? "reused" : persistentResume ? "thread-resumed" : "new"), + runnerReused, + threadReused, + persistentResume, + valuesRedacted: true + }; +} + +export function agentRunRunnerSummary(mapping = {}) { + return { + kind: AGENTRUN_RUNNER_KIND, + adapter: ADAPTER_ID, + runId: mapping.runId ?? null, + commandId: mapping.commandId ?? null, + attemptId: mapping.attemptId ?? null, + runnerId: mapping.runnerId ?? null, + jobName: mapping.jobName ?? null, + namespace: mapping.namespace ?? null, + dispatchIntentId: mapping.dispatchIntentId ?? null, + dispatchIntentState: mapping.dispatchIntentState ?? null, + durableDispatch: mapping.durableDispatch === true, + backendProfile: mapping.backendProfile ?? null, + provider: providerForBackendProfile(mapping.backendProfile ?? "deepseek"), + sessionMode: AGENTRUN_SESSION_MODE, + implementationType: AGENTRUN_IMPLEMENTATION_TYPE, + capabilityLevel: AGENTRUN_CAPABILITY_LEVEL, + codexStdio: false, + delegatedToAgentRun: true, + writeCapable: true, + durableSession: true, + longLivedSession: true, + sandbox: "danger-full-access", + valuesPrinted: false + }; +} + +export function hostForUrl(value) { + if (!value) return null; + try { + return new URL(value).hostname.toLowerCase(); + } catch { + return null; + } +} + +export function waitingForForPhase(phase) { + if (phase.includes("runner-job")) return "agentrun-runner"; + if (phase.includes("resource-bundle")) return "agentrun-resource-bundle"; + if (phase.includes("backend")) return "agentrun-backend"; + return "agentrun-result"; +} + +export function textPayload(payload, fallback) { + for (const key of ["message", "text", "content", "delta", "summary", "phase"]) { + if (typeof payload?.[key] === "string" && payload[key].length > 0) return payload[key]; + } + return fallback; +} + +export function requiredString(value, fieldName) { + if (typeof value === "string" && value.trim()) return value.trim(); + throw Object.assign(new Error(`AgentRun response missing ${fieldName}`), { code: "agentrun_response_invalid", statusCode: 502 }); +} + +export function firstNonEmpty(...values) { + for (const value of values) { + const text = String(value ?? "").trim(); + if (text) return text; + } + return null; +} + +export function nowIso(now) { + return typeof now === "function" ? now() : new Date().toISOString(); +} + +export function timestampIsoOrNow(value = null) { + const parsed = Date.parse(String(value ?? "")); + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); +} + +export function redactUrl(value) { + try { + const url = new URL(value); + if (url.username || url.password) { + url.username = "***"; + url.password = ""; + } + return url.toString(); + } catch { + return ""; + } +} diff --git a/internal/cloud/server-workbench-facts.ts b/internal/cloud/server-workbench-facts.ts new file mode 100644 index 00000000..97ce0952 --- /dev/null +++ b/internal/cloud/server-workbench-facts.ts @@ -0,0 +1,990 @@ +/* + * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-read-model-timeline-contract; draft-2026-06-28-p0-d518-session-timeline-consistency; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-api-contract; PJ2026-010401 Web工作台 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-model-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 + * 职责: Workbench durable fact normalization and DTO projection helpers; no HTTP or realtime transport ownership. + */ +import { createHash } from "node:crypto"; + +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { + parsePositiveInteger, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson +} from "./server-http-utils.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; +import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; +import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; +import { openKafkaEventStream } from "./kafka-event-bridge.ts"; +import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; +import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; + +function projectionText(...values) { + for (const value of values) { + if (value && typeof value === "object") { + const nested = messageAuthorityTextValue(value.text ?? value.content ?? value.message ?? value.summary ?? value.preview ?? value.title); + if (nested) return nested; + continue; + } + const text = messageAuthorityTextValue(value); + if (text) return text; + } + return null; +} + +function workbenchError(code, message, extra = {}) { + return { + ok: false, + status: "failed", + error: { code, message, ...extra }, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function normalizeRole(role) { + const value = textValue(role).toLowerCase(); + return value === "agent" ? "assistant" : value; +} + +function partFact(part, index, messageId, traceId) { + const type = textValue(part?.type) || "text"; + const text = messageAuthorityTextValue(part?.text ?? part?.content ?? part?.message); + return { + partId: safePartId(part?.partId ?? part?.id) || `prt_${hash(`${messageId}:${index}:${type}`).slice(0, 24)}`, + messageId, + traceId, + type, + text: text || null, + status: normalizeStatus(part?.status ?? "completed"), + toolName: textValue(part?.toolName ?? part?.name) || null, + createdAt: textValue(part?.createdAt ?? part?.timestamp) || null, + valuesRedacted: part?.valuesRedacted !== false + }; +} + +export function factSessionSummary(session, facts = {}) { + const sessionId = factSessionId(session); + if (!sessionId) return null; + const currentTrace = factCurrentTraceForSession(facts, sessionId, session); + const traceId = currentTrace?.traceId ?? factLastTraceId(session) ?? factLatestTraceIdForSession(facts, sessionId); + const projection = traceId ? factProjectionForTrace(facts, traceId) : null; + const turn = traceId ? factTurnForTrace(facts, traceId) : null; + const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null; + const trace = traceId ? factTraceSnapshot(facts, traceId) : null; + const messages = factMessagesForSession(session, facts); + const terminalMessage = traceId ? factTerminalMessageForTrace(messages, traceId) : null; + const traceStatus = normalizeTerminalStatus(trace?.status); + const checkpointStatus = normalizeTerminalStatus(checkpoint?.status); + const messageTerminalStatus = factTerminalStatusFromMessageProjection(terminalMessage); + const turnStatus = normalizeStatus(turn?.status); + const launchContext = compactLaunchContext(session?.sessionJson?.launchContext); + const status = normalizeStatus(checkpointStatus ?? traceStatus ?? messageTerminalStatus ?? nonUnknownStatus(turnStatus) ?? nonUnknownStatus(currentTrace?.status) ?? session?.status); + const timing = traceId + ? factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? terminalMessage : null, turn, trace) + : factTimingProjection(session, status); + const title = sessionTitleFromMessages(messages); + const preview = sessionPreviewFromMessages(messages); + return { + sessionId, + threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), + agentId: session?.agentId ?? "hwlab-code-agent", + title, + preview, + titleSource: title ? "message-projection" : null, + previewSource: preview ? "message-projection" : null, + status, + running: RUNNING_STATUSES.has(status), + terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status), + lastTraceId: traceId, + projection, + projectionStatus: projection?.projectionStatus ?? null, + projectionHealth: projection?.projectionHealth ?? null, + staleMs: projection?.staleMs ?? null, + blocker: projection?.blocker ?? null, + providerProfile: textValue(session?.providerProfile ?? session?.sessionJson?.providerProfile) || null, + launchContext, + messageCount: messages.length, + firstUserMessagePreview: firstUserPreview(messages), + updatedAt: factUpdatedAt(session), + turnSummary: turn ? { + turnId: factTurnId(turn, traceId), + traceId, + status, + running: RUNNING_STATUSES.has(status), + terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status), + eventCount: projection?.lastProjectedSeq ?? trace?.eventCount, + projection, + projectionStatus: projection?.projectionStatus ?? null, + projectionHealth: projection?.projectionHealth ?? null, + staleMs: projection?.staleMs ?? null, + blocker: projection?.blocker ?? null, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + updatedAt: factUpdatedAt(turn) + } : null, + valuesRedacted: true + }; +} + +export function factSessionDetail(session, facts = {}, options = {}) { + const sessionId = factSessionId(session); + return buildWorkbenchSessionDetail({ + summary: factSessionSummary(session, facts), + session, + sessionId, + includeMessages: options.includeMessages === true, + messages: factMessagesForSession(session, facts) + }); +} + +export function factMessagesForSession(session, facts = {}) { + const sessionId = factSessionId(session); + if (!sessionId) return []; + const partsByMessageId = new Map(); + for (const part of factArray(facts.parts)) { + if (part.sessionId !== sessionId) continue; + const messageId = textValue(part.messageId); + if (!messageId) continue; + const existing = partsByMessageId.get(messageId) ?? []; + existing.push(part); + partsByMessageId.set(messageId, existing); + } + const messageGroups = canonicalFactMessageGroupsForSession( + factArray(facts.messages) + .filter((message) => message.sessionId === sessionId) + .sort((left, right) => compareFactMessagesForSessionAsc(left, right, facts)), + partsByMessageId, + facts + ); + return messageGroups + .sort((left, right) => compareFactMessagesForSessionAsc(left.message, right.message, facts)) + .map((group) => factMessageDto(group.message, group.parts, facts)) + .filter(Boolean); +} + +export function canonicalFactMessageGroupsForSession(messages = [], partsByMessageId = new Map(), facts = {}) { + const groups = new Map(); + for (const message of messages) { + const key = factMessageCanonicalGroupKey(message); + const existing = groups.get(key) ?? []; + existing.push(message); + groups.set(key, existing); + } + return [...groups.values()].map((groupMessages) => { + const message = selectCanonicalFactMessage(groupMessages, partsByMessageId, facts); + const aliasIds = uniqueText(groupMessages.map((item) => item?.messageId)); + const parts = aliasIds.flatMap((messageId) => partsByMessageId.get(messageId) ?? []); + return { message, parts }; + }); +} + +export function factMessageCanonicalGroupKey(message) { + const messageId = textValue(message?.messageId) || textValue(message?.id); + const role = textValue(message?.role) || "agent"; + const traceId = safeTraceId(message?.traceId); + const sessionId = factSessionId(message) ?? textValue(message?.sessionId) ?? "session"; + if (traceId && isAssistantLikeRole(role)) return `assistant:${sessionId}:${traceId}`; + return `message:${messageId || sessionId}:${role}:${traceId || "none"}`; +} + +export function selectCanonicalFactMessage(messages = [], partsByMessageId = new Map(), facts = {}) { + if (messages.length <= 1) return messages[0] ?? null; + const traceId = safeTraceId(messages[0]?.traceId); + const preferredMessageId = traceId ? workbenchLifecycleMessageId(traceId, "agent") : null; + const turnMessageId = traceId ? textValue(factTurnForTrace(facts, traceId)?.messageId) || null : null; + return [...messages].sort((left, right) => { + const score = factMessageCanonicalScore(right, partsByMessageId, preferredMessageId, turnMessageId) - factMessageCanonicalScore(left, partsByMessageId, preferredMessageId, turnMessageId); + return score || compareFactRecordsDesc(left, right); + })[0] ?? messages[0] ?? null; +} + +export function factMessageCanonicalScore(message, partsByMessageId = new Map(), preferredMessageId = null, turnMessageId = null) { + const messageId = textValue(message?.messageId) || textValue(message?.id); + let score = 0; + if (preferredMessageId && messageId === preferredMessageId) score += 10000; + if (turnMessageId && messageId === turnMessageId) score += 1000; + if (factMessageHasFinalResponsePart(message, partsByMessageId)) score += 100; + if (message?.sealed === true || message?.terminal === true) score += 50; + if (isTerminalProjectionStatus(normalizeStatus(message?.status))) score += 25; + score += Math.min(10, factSeq(message) ?? 0); + return score; +} + +export function factMessageHasFinalResponsePart(message, partsByMessageId = new Map()) { + const messageId = textValue(message?.messageId) || textValue(message?.id); + if (!messageId) return false; + return (partsByMessageId.get(messageId) ?? []).some((part) => (textValue(part?.partType ?? part?.type) === "final_response") && Boolean(projectionText(part?.text, part?.content, part?.message))); +} + +export function workbenchLifecycleMessageId(traceId, role = "agent") { + const traceSuffix = (safeTraceId(traceId) || String(traceId || "trace")) + .replace(/^trc_/u, "") + .replace(/[^A-Za-z0-9_.:-]/gu, "_") + .slice(0, 48) || "trace"; + return `msg_${traceSuffix}_${role === "user" ? "user" : "agent"}`; +} + +export function factMessageDto(message, parts = [], facts = {}) { + const messageId = safeMessageId(message?.messageId) || textValue(message?.messageId); + if (!messageId) return null; + const traceId = safeTraceId(message?.traceId) ?? null; + const role = textValue(message?.role) || "agent"; + const turn = traceId ? factTurnForTrace(facts, traceId) : null; + const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null; + const messageStatus = normalizeStatus(message?.status); + const assistantLike = isAssistantLikeRole(role); + const legacyText = projectionText(message?.text, message?.content, message?.message, message?.finalResponse); + const baseParts = parts.length > 0 + ? [...parts].sort(compareFactPartsAsc).map((part) => factPartDto(part, messageId, traceId)).filter(Boolean) + : !assistantLike && legacyText ? [partFact({ type: "text", text: legacyText, status: message?.status }, 0, messageId, traceId)] : []; + const checkpointStatus = assistantLike ? normalizeTerminalStatus(checkpoint?.status) : null; + const preliminaryStatus = normalizeStatus(checkpointStatus ?? (assistantLike ? turn?.status : null) ?? messageStatus); + const syntheticTerminalFinalResponse = assistantLike && !factPartsHaveFinalResponse(baseParts) + ? factSyntheticTerminalFinalResponse(preliminaryStatus, traceId, message, checkpoint, turn) + : null; + const normalizedParts = syntheticTerminalFinalResponse + ? [...baseParts, partFact({ type: "final_response", text: syntheticTerminalFinalResponse.text, status: syntheticTerminalFinalResponse.status }, baseParts.length, messageId, traceId)] + : baseParts; + const messageTerminalStatus = assistantLike ? factTerminalStatusFromMessageProjection(message, normalizedParts) : null; + const status = normalizeStatus(checkpointStatus ?? messageTerminalStatus ?? (assistantLike ? turn?.status : null) ?? messageStatus); + const timing = assistantLike + ? factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? { ...message, parts: normalizedParts } : null, turn, message) + : factTimingProjection(message, status); + const text = factMessageAuthorityText({ role, status, parts: normalizedParts, legacyText }); + return { + messageId, + role, + sessionId: (safeSessionId(message?.sessionId) ?? textValue(message?.sessionId)) || null, + traceId, + turnId: safeTurnId(message?.turnId) || traceId, + status, + parts: normalizedParts, + text: text || "", + textPreview: text ? text.slice(0, 240) : null, + createdAt: textValue(message?.createdAt) || null, + updatedAt: factUpdatedAt(message), + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + projectionStatus: null, + projectionHealth: null, + valuesRedacted: message?.valuesRedacted !== false + }; +} + +export function factMessageAuthorityText({ role, status, parts = [], legacyText = null } = {}) { + if (!isAssistantLikeRole(role)) return firstFactPartText(parts) ?? legacyText ?? ""; + if (!isTerminalProjectionStatus(status)) return ""; + return firstFactPartText(parts, "final_response") ?? ""; +} + +export function factPartsHaveFinalResponse(parts = []) { + return parts.some((part) => part?.type === "final_response" && Boolean(projectionText(part?.text))); +} + +export function factSyntheticTerminalFinalResponse(status, traceId = null, ...records) { + const terminalStatus = normalizeTerminalStatus(status); + if (!terminalStatus || terminalStatus === "completed") return null; + return terminalFinalResponse(terminalStatus, { traceId, status: terminalStatus, records }, { evidence: records }); +} + +export function isTerminalProjectionStatus(status) { + return TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status); +} + +export function firstFactPartText(parts = [], type = null) { + for (const part of parts) { + if (type && part?.type !== type) continue; + const text = projectionText(part?.text); + if (text) return text; + } + return null; +} + +export function factMessageFinalResponseText(message) { + if (!message || !isAssistantLikeRole(message.role) || !isTerminalProjectionStatus(message.status)) return null; + return firstFactPartText(message.parts, "final_response"); +} + +export function factTerminalMessageForTrace(messages = [], traceId = null) { + const safeTrace = safeTraceId(traceId); + return [...factArray(messages)].reverse().find((message) => { + if (!message || !isAssistantLikeRole(message.role)) return false; + if (safeTrace && message.traceId !== safeTrace) return false; + return Boolean(factTerminalStatusFromMessageProjection(message)); + }) ?? null; +} + +export function factTerminalStatusFromMessageProjection(message = null, parts = null) { + if (!message || !isAssistantLikeRole(message.role)) return null; + const sourceParts = Array.isArray(parts) ? parts : factArray(message.parts); + if (!firstFactPartText(sourceParts, "final_response")) return null; + const messageStatus = normalizeTerminalStatus(message.status); + if (messageStatus) return messageStatus; + for (const part of sourceParts) { + if (textValue(part?.partType ?? part?.type) !== "final_response") continue; + if (!projectionText(part?.text, part?.content, part?.message)) continue; + const partStatus = normalizeTerminalStatus(part?.status); + if (partStatus) return partStatus; + if (part?.terminal === true || part?.sealed === true) return "completed"; + } + return message.terminal === true || message.sealed === true ? "completed" : null; +} + +export function factPartDto(part, messageId, traceId) { + const partId = safePartId(part?.partId) || textValue(part?.partId) || `prt_${hash(`${messageId}:${part?.partIndex ?? 0}:${part?.partType ?? part?.type ?? "text"}`).slice(0, 24)}`; + const text = projectionText(part?.text, part?.content, part?.message); + return { + partId, + messageId, + traceId: safeTraceId(part?.traceId) ?? traceId, + type: textValue(part?.partType ?? part?.type) || "text", + text: text || null, + status: normalizeStatus(part?.status), + toolName: textValue(part?.toolName ?? part?.name) || null, + createdAt: textValue(part?.createdAt ?? part?.occurredAt) || null, + valuesRedacted: part?.valuesRedacted !== false + }; +} + +export function factTurnForTrace(facts = {}, traceId, turnId = null) { + const safeTrace = safeTraceId(traceId); + if (!safeTrace) return null; + const requestedTurn = safeTurnId(turnId); + const matches = factArray(facts.turns) + .filter((turn) => turn.traceId === safeTrace && (!requestedTurn || turn.turnId === requestedTurn || turn.turnId === safeTrace)) + .sort(compareFactRecordsDesc); + return matches[0] ?? null; +} + +export function factTurnSnapshot({ turn = null, session = null, facts = {}, traceId, turnId = null } = {}) { + const safeTrace = safeTraceId(traceId); + const resolvedTurnId = factTurnId(turn, safeTrace) ?? safeTurnId(turnId) ?? safeTrace; + const messages = session ? factMessagesForSession(session, facts) : []; + const userMessage = messages.find((message) => message.role === "user") ?? null; + const assistantMessage = [...messages].reverse().find((message) => isAssistantLikeRole(message.role)) ?? null; + const terminalMessage = safeTrace ? factTerminalMessageForTrace(messages, safeTrace) : null; + const checkpoint = safeTrace ? factCheckpointForTrace(facts, safeTrace) : null; + const trace = factTraceSnapshot(facts, safeTrace); + const checkpointStatus = normalizeTerminalStatus(checkpoint?.status); + const traceStatus = normalizeTerminalStatus(trace.status); + const messageTerminalStatus = factTerminalStatusFromMessageProjection(terminalMessage); + const status = normalizeStatus(checkpointStatus ?? traceStatus ?? messageTerminalStatus ?? turn?.status ?? session?.status); + const terminal = isTerminalProjectionStatus(status); + const assistantText = terminal ? factMessageFinalResponseText(assistantMessage) : null; + const timing = factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? terminalMessage : null, turn, trace); + return { + turnId: resolvedTurnId, + traceId: safeTrace, + status, + running: RUNNING_STATUSES.has(status), + terminal, + sessionId: factSessionId(session) ?? turn?.sessionId ?? checkpoint?.sessionId ?? null, + threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), + userMessageId: userMessage?.messageId ?? null, + assistantMessageId: assistantMessage?.messageId ?? turn?.messageId ?? null, + assistantText: assistantText ?? null, + finalResponse: assistantText ? { text: assistantText, sealed: true, source: "message-part" } : null, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + agentRun: checkpoint ? { + runId: textValue(checkpoint.runId) || null, + commandId: textValue(checkpoint.commandId) || null, + status, + lastSeq: factSeq(checkpoint), + valuesRedacted: true + } : null, + trace: { + traceId: safeTrace, + status: trace.status, + eventCount: trace.eventCount, + timing: trace.timing, + startedAt: trace.startedAt, + lastEventAt: trace.lastEventAt, + finishedAt: trace.finishedAt, + durationMs: trace.durationMs, + updatedAt: trace.updatedAt + }, + urls: { + self: resolvedTurnId ? `/v1/workbench/turns/${encodeURIComponent(resolvedTurnId)}` : null, + traceEvents: safeTrace ? `/v1/workbench/traces/${encodeURIComponent(safeTrace)}/events` : null + } + }; +} + +export function factTraceSnapshot(facts = {}, traceId) { + const safeTrace = safeTraceId(traceId); + const events = factArray(facts.traceEvents) + .filter((event) => !safeTrace || event.traceId === safeTrace) + .sort(compareFactTraceEventsAsc) + .map(factTraceEventDto) + .filter(Boolean); + const status = durableTraceStatus(events); + const lastEvent = events.at(-1) ?? null; + const checkpoint = factCheckpointForTrace(facts, safeTrace); + const timing = factTraceTimingProjection(events, checkpoint, status); + return { + traceId: safeTrace, + status, + eventCount: events.length, + events, + lastEvent, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + updatedAt: lastEvent?.updatedAt ?? lastEvent?.createdAt ?? checkpoint?.updatedAt ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +export function factTraceTimingProjection(events = [], checkpoint = null, status = null) { + const checkpointTiming = factTimingProjection(checkpoint, status); + const normalizedStatus = normalizeStatus(status ?? checkpoint?.status); + const terminal = TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus); + const observedAt = new Date().toISOString(); + const eventStartTimes = factArray(events).flatMap((event) => [event?.createdAt, event?.occurredAt]); + const eventActivityTimes = factArray(events).flatMap((event) => [event?.createdAt, event?.occurredAt, event?.updatedAt]); + const terminalEventTimes = factArray(events) + .filter(factTraceEventIsTerminalAuthority) + .flatMap(factTraceEventTerminalTimes); + const startedAt = firstTimestampIso(checkpointTiming.startedAt, ...eventStartTimes); + const finishedAt = terminal ? latestTimestampIso(checkpointTiming.finishedAt, ...terminalEventTimes) : null; + const lastEventAt = terminal && finishedAt ? finishedAt : latestTimestampIso(checkpointTiming.lastEventAt, ...eventActivityTimes); + const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt); + const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt); + return { + ...checkpointTiming, + startedAt, + lastEventAt, + finishedAt, + durationMs, + observedAt: terminal ? null : observedAt, + lastEventAgeMs, + valuesRedacted: true + }; +} + +export function factTraceEventIsTerminalAuthority(event = null) { + const eventType = textValue(event?.eventType ?? event?.type); + return event?.terminal === true || event?.sealed === true || eventType === "terminal"; +} + +export function factTraceEventTerminalTimes(event = null) { + const source = objectValue(event?.timing); + return [source?.finishedAt, event?.finishedAt, source?.lastEventAt, event?.lastEventAt, event?.occurredAt, event?.createdAt]; +} + +export function factTraceEventDto(event, index) { + const seq = factProjectedSeq(event); + if (!seq) return null; + const sourceSeq = Number.isFinite(Number(event?.sourceSeq)) && Number(event.sourceSeq) >= 0 ? Math.trunc(Number(event.sourceSeq)) : null; + return { + ...event, + id: textValue(event?.id) || textValue(event?.sourceEventId) || null, + seq, + projectedSeq: seq, + sourceSeq, + type: textValue(event?.type ?? event?.eventType) || "event", + label: textValue(event?.label ?? event?.eventType ?? event?.type) || null, + status: normalizeStatus(event?.status), + createdAt: textValue(event?.createdAt ?? event?.occurredAt) || null, + updatedAt: factUpdatedAt(event), + valuesRedacted: event?.valuesRedacted !== false + }; +} + +export function factProjectionForTrace(facts = {}, traceId) { + const checkpoint = factCheckpointForTrace(facts, traceId); + if (!checkpoint) { + return { + projectionStatus: "unknown", + projectionHealth: "unknown", + lastProjectedSeq: null, + sourceRunId: null, + sourceCommandId: null, + staleMs: null, + blocker: null, + updatedAt: null, + valuesRedacted: true + }; + } + const projectionStatus = normalizeProjectionStatus(checkpoint.projectionStatus); + const projectionHealth = normalizeProjectionHealth(checkpoint.projectionHealth, projectionStatus); + const diagnostic = objectValue(checkpoint.diagnostic); + const timing = factTimingProjection(checkpoint, normalizeStatus(checkpoint.status)); + return { + projectionStatus, + projectionHealth, + lastProjectedSeq: factSeq(checkpoint), + sourceRunId: textValue(checkpoint.runId ?? checkpoint.sourceRunId) || null, + sourceCommandId: textValue(checkpoint.commandId ?? checkpoint.sourceCommandId) || null, + staleMs: null, + blocker: diagnostic.blocker ?? checkpoint.blocker ?? null, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + updatedAt: factUpdatedAt(checkpoint), + valuesRedacted: true + }; +} + +export function factTimingProjection(record = null, status = null) { + const source = objectValue(record?.timing); + const observedAt = new Date().toISOString(); + const startedAt = timestampIso(source?.startedAt ?? record?.startedAt ?? record?.createdAt); + const lastEventAt = timestampIso(source?.lastEventAt ?? record?.lastEventAt ?? record?.updatedAt ?? record?.createdAt); + const normalizedStatus = normalizeStatus(status ?? record?.status); + const terminal = record?.terminal === true || TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus); + const finishedAt = terminal ? timestampIso(source?.finishedAt ?? record?.finishedAt ?? record?.completedAt) : null; + const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt); + const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt); + return { startedAt, lastEventAt, finishedAt, durationMs, observedAt: terminal ? null : observedAt, lastEventAgeMs, valuesRedacted: source?.valuesRedacted !== false }; +} + +export function factCombinedTimingProjection(status = null, ...records) { + const normalizedStatus = normalizeStatus(status ?? records.find((record) => record?.status)?.status); + const terminal = TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus); + const observedAt = new Date().toISOString(); + const timings = records.map((record) => factTimingSource(record)).filter(Boolean); + const startedAt = firstTimestampIso(...timings.map((timing) => timing.startedAt)); + const finishedAt = terminal ? latestTimestampIso(...timings.map((timing) => timing.finishedAt)) : null; + const lastEventAt = terminal && finishedAt ? finishedAt : latestTimestampIso(...timings.map((timing) => timing.lastEventAt)); + const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt); + const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt); + return { startedAt, lastEventAt, finishedAt, durationMs, observedAt: terminal ? null : observedAt, lastEventAgeMs, valuesRedacted: true }; +} + +export function factTimingSource(record = null) { + if (!record) return null; + const source = objectValue(record?.timing); + return { + startedAt: timestampIso(source?.startedAt ?? record?.startedAt ?? record?.createdAt), + lastEventAt: timestampIso(source?.lastEventAt ?? record?.lastEventAt ?? record?.updatedAt ?? record?.createdAt), + finishedAt: timestampIso(source?.finishedAt ?? record?.finishedAt ?? record?.completedAt) + }; +} + +export function firstTimestampIso(...values) { + for (const value of values) { + const timestamp = timestampIso(value); + if (timestamp) return timestamp; + } + return null; +} + +export function latestTimestampIso(...values) { + let latest = null; + let latestMs = Number.NEGATIVE_INFINITY; + for (const value of values) { + const timestamp = timestampIso(value); + if (!timestamp) continue; + const ms = Date.parse(timestamp); + if (!Number.isFinite(ms) || ms < latestMs) continue; + latest = timestamp; + latestMs = ms; + } + return latest; +} + +export function elapsedFactMs(startedAt, endedAt) { + const start = Date.parse(String(startedAt ?? "")); + const end = Date.parse(String(endedAt ?? "")); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null; + return Math.trunc(end - start); +} + +export function timestampIso(value) { + const ms = Date.parse(String(value ?? "")); + return Number.isFinite(ms) ? new Date(ms).toISOString() : null; +} + +export function factCheckpointForTrace(facts = {}, traceId) { + const safeTrace = safeTraceId(traceId); + if (!safeTrace) return null; + return factArray(facts.checkpoints) + .filter((checkpoint) => checkpoint.traceId === safeTrace) + .sort(compareFactRecordsDesc)[0] ?? null; +} + +export function workbenchProjectionStoreError(error) { + const data = objectValue(error?.data) ?? {}; + const retryAfterNumber = Number(data.retryAfterMs); + const retryAfterMs = Number.isFinite(retryAfterNumber) && retryAfterNumber >= 0 ? Math.trunc(retryAfterNumber) : null; + const retryAttempt = nonNegativeIntegerOrNull(data.retryAttempt ?? data.retryAttempts); + const retryMax = nonNegativeIntegerOrNull(data.retryMax); + const retryDelaysMs = Array.isArray(data.retryDelaysMs) + ? data.retryDelaysMs.map(nonNegativeIntegerOrNull).filter((value) => value !== null) + : []; + return workbenchError("projection_store_unavailable", "Workbench durable projection store is unavailable.", { + causeCode: error?.code ?? null, + queryResult: textValue(data.queryResult) || null, + blockedLayer: textValue(data.blockedLayer) || null, + retryable: data.retryable !== false, + transient: data.transient === true || data.retryable === true, + retryAfterMs: retryAfterMs ?? null, + retryAttempt, + retryMax, + retryAttempts: retryAttempt, + retryExhausted: data.retryExhausted === true, + retryDelaysMs: retryDelaysMs.length > 0 ? retryDelaysMs : null + }); +} + +export function nonNegativeIntegerOrNull(value) { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null; +} + +export function factSessionId(session) { + return safeSessionId(session?.sessionId ?? session?.id) ?? null; +} + +export function factLastTraceId(session) { + return safeTraceId(session?.lastTraceId ?? session?.traceId) ?? null; +} + +export function factLatestTraceIdForSession(facts = {}, sessionId) { + const turn = factArray(facts.turns).filter((item) => item.sessionId === sessionId).sort(compareFactRecordsDesc)[0] ?? null; + return safeTraceId(turn?.traceId) ?? null; +} + +export function factCurrentTraceForSession(facts = {}, sessionId, session = null) { + const candidates = []; + const sessionTraceId = factLastTraceId(session); + if (sessionTraceId) candidates.push(factTraceCandidate(session, sessionTraceId, normalizeStatus(session?.status), 0)); + for (const message of factArray(facts.messages)) { + if (message?.sessionId !== sessionId || !safeTraceId(message?.traceId)) continue; + candidates.push(factMessageTraceCandidate(message)); + } + for (const turn of factArray(facts.turns)) { + if (turn?.sessionId !== sessionId || !safeTraceId(turn?.traceId)) continue; + candidates.push(factTraceCandidate(turn, safeTraceId(turn.traceId), normalizeStatus(turn?.status), 3)); + } + return candidates.sort(compareTraceCandidatesDesc)[0] ?? null; +} + +export function factMessageTraceCandidate(message) { + const role = textValue(message?.role); + let status = normalizeStatus(message?.status); + let priority = 1; + if (isAssistantLikeRole(role)) { + priority = 2; + if (status === "unknown") status = "running"; + } else if (safeTraceId(message?.traceId)) { + status = "running"; + } + return factTraceCandidate(message, safeTraceId(message?.traceId), status, priority); +} + +export function factTraceCandidate(record, traceId, status, priority) { + return { traceId: safeTraceId(traceId) ?? null, status: normalizeStatus(status), seq: factSeq(record), updatedAt: factUpdatedAt(record), priority }; +} + +export function compareTraceCandidatesDesc(left, right) { + return compareNumberDesc(left?.seq, right?.seq) || compareTimestampDesc(left?.updatedAt, right?.updatedAt) || compareNumberDesc(left?.priority, right?.priority); +} + +export function nonUnknownStatus(status) { + return normalizeStatus(status) === "unknown" ? null : status; +} + +export function factTurnId(turn, fallbackTraceId = null) { + return safeTurnId(turn?.turnId) ?? safeTraceId(turn?.turnId) ?? safeTraceId(fallbackTraceId) ?? null; +} + +export function factUpdatedAt(record) { + return textValue(record?.updatedAt ?? record?.occurredAt ?? record?.createdAt) || null; +} + +export function factSeq(record) { + for (const value of [record?.projectedSeq, record?.sourceSeq, record?.seq]) { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed >= 0) return Math.trunc(parsed); + } + return null; +} + +export function factProjectedSeq(record) { + const parsed = Number(record?.projectedSeq); + return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : null; +} + +export function normalizeProjectionStatus(value) { + const status = normalizeStatus(value); + if (status === "caught-up") return "caught-up"; + if (status === "caughtup") return "caught-up"; + return ["projecting", "blocked", "stalled", "unknown"].includes(status) ? status : "unknown"; +} + +export function normalizeProjectionHealth(value, projectionStatus = "unknown") { + const health = normalizeStatus(value); + if (health === "healthy" && projectionStatus === "caught-up") return "caught-up"; + if (health === "healthy" && projectionStatus === "projecting") return "projecting"; + if (["caught-up", "projecting", "degraded", "stalled", "unavailable", "unknown"].includes(health)) return health; + return projectionStatus === "caught-up" || projectionStatus === "projecting" ? projectionStatus : "unknown"; +} + +export function compareFactSessionsDesc(left, right) { + return compareTimestampDesc(factUpdatedAt(left), factUpdatedAt(right)) || compareText(factSessionId(left), factSessionId(right)); +} + +export function compareFactMessagesAsc(left, right) { + return compareNumberAsc(factSeq(left), factSeq(right)) || compareTimestampAsc(left?.createdAt ?? left?.updatedAt, right?.createdAt ?? right?.updatedAt) || compareText(left?.messageId, right?.messageId); +} + +export function compareFactMessagesForSessionAsc(left, right, facts = {}) { + const leftKey = factMessageTimelineKey(left, facts); + const rightKey = factMessageTimelineKey(right, facts); + return compareOptionalTimestampAsc(leftKey.timelineAnchorAt, rightKey.timelineAnchorAt) + || compareOptionalNumberAsc(leftKey.turnSeq, rightKey.turnSeq) + || compareTimestampAsc(leftKey.turnStartedAt, rightKey.turnStartedAt) + || compareText(leftKey.traceId, rightKey.traceId) + || compareOptionalNumberAsc(leftKey.roleRank, rightKey.roleRank) + || compareTimestampAsc(leftKey.messageCreatedAt, rightKey.messageCreatedAt) + || compareOptionalNumberAsc(leftKey.messageSeq, rightKey.messageSeq) + || compareText(left?.messageId, right?.messageId); +} + +export function factMessageTimelineKey(message, facts = {}) { + const traceId = safeTraceId(message?.traceId) ?? null; + const turn = traceId ? factTurnForTrace(facts, traceId) : null; + const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null; + const anchorMessage = traceId ? factTimelineAnchorMessageForTrace(facts, traceId) : null; + return { + traceId: traceId ?? "", + timelineAnchorAt: firstTimestampIso(anchorMessage?.createdAt, anchorMessage?.updatedAt), + turnSeq: firstFiniteNumber(factSeq(turn), factSeq(checkpoint)), + turnStartedAt: firstTimestampIso(turn?.startedAt, checkpoint?.startedAt, message?.createdAt, message?.updatedAt), + roleRank: factMessageTimelineRoleRank(message?.role), + messageCreatedAt: timestampIso(message?.createdAt ?? message?.updatedAt), + messageSeq: factSeq(message) + }; +} + +export function factTimelineAnchorMessageForTrace(facts = {}, traceId) { + const safeTrace = safeTraceId(traceId); + if (!safeTrace) return null; + const messages = factArray(facts.messages).filter((message) => message.traceId === safeTrace); + const userMessages = messages.filter((message) => normalizeRole(message?.role) === "user"); + const candidates = userMessages.length > 0 ? userMessages : messages; + return [...candidates].sort((left, right) => compareOptionalTimestampAsc(left?.createdAt ?? left?.updatedAt, right?.createdAt ?? right?.updatedAt) || compareOptionalNumberAsc(factSeq(left), factSeq(right)) || compareText(left?.messageId, right?.messageId))[0] ?? null; +} + +export function factMessageTimelineRoleRank(role) { + const normalized = normalizeRole(role); + if (normalized === "user") return 0; + if (isAssistantLikeRole(normalized)) return 1; + return 2; +} + +export function firstFiniteNumber(...values) { + for (const value of values) { + const parsed = Number(value); + if (Number.isFinite(parsed)) return Math.trunc(parsed); + } + return null; +} + +export function compareOptionalNumberAsc(left, right) { + const leftNumber = Number(left); + const rightNumber = Number(right); + const leftComparable = Number.isFinite(leftNumber) ? leftNumber : Number.MAX_SAFE_INTEGER; + const rightComparable = Number.isFinite(rightNumber) ? rightNumber : Number.MAX_SAFE_INTEGER; + return leftComparable - rightComparable; +} + +export function compareFactPartsAsc(left, right) { + return compareNumberAsc(left?.partIndex, right?.partIndex) || compareNumberAsc(factSeq(left), factSeq(right)) || compareText(left?.partId, right?.partId); +} + +export function compareFactTraceEventsAsc(left, right) { + return compareNumberAsc(factProjectedSeq(left), factProjectedSeq(right)); +} + +export function compareFactRecordsDesc(left, right) { + return compareNumberDesc(factSeq(left), factSeq(right)) || compareTimestampDesc(factUpdatedAt(left), factUpdatedAt(right)); +} + +export function compareTimestampAsc(left, right) { + return timestampMs(left) - timestampMs(right); +} + +export function compareTimestampDesc(left, right) { + return timestampMs(right) - timestampMs(left); +} + +export function compareOptionalTimestampAsc(left, right) { + const leftMs = optionalTimestampMs(left); + const rightMs = optionalTimestampMs(right); + return (leftMs ?? Number.MAX_SAFE_INTEGER) - (rightMs ?? Number.MAX_SAFE_INTEGER); +} + +export function compareNumberAsc(left, right) { + return numericValue(left, Number.MAX_SAFE_INTEGER) - numericValue(right, Number.MAX_SAFE_INTEGER); +} + +export function compareNumberDesc(left, right) { + return numericValue(right, -1) - numericValue(left, -1); +} + +export function compareText(left, right) { + return textValue(left).localeCompare(textValue(right)); +} + +export function timestampMs(value) { + const parsed = Date.parse(String(value ?? "")); + return Number.isFinite(parsed) ? parsed : 0; +} + +export function optionalTimestampMs(value) { + const parsed = Date.parse(String(value ?? "")); + return Number.isFinite(parsed) ? parsed : null; +} + +export function numericValue(value, fallback) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +export function mergeFactSets(...sets) { + const merged = emptyFactSet(); + const keys = { + sessions: "sessionId", + messages: "messageId", + parts: "partId", + turns: "turnId", + traceEvents: "id", + checkpoints: "traceId" + }; + for (const set of sets) { + for (const [key, idKey] of Object.entries(keys)) { + const seen = new Set(merged[key].map((item) => textValue(item?.[idKey]))); + for (const item of factArray(set?.[key])) { + const id = textValue(item?.[idKey]); + if (id && seen.has(id)) continue; + merged[key].push(item); + if (id) seen.add(id); + } + } + } + return merged; +} + +export function emptyFactSet() { + return { + sessions: [], + messages: [], + parts: [], + turns: [], + traceEvents: [], + checkpoints: [] + }; +} + +export function factArray(value) { + return Array.isArray(value) ? value : []; +} + +export function uniqueText(values = []) { + return [...new Set(values.map((value) => textValue(value)).filter(Boolean))]; +} + + +export function eventSeq(event, index) { + const seq = Number(event?.seq); + return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; +} + +export function normalizeStatus(value) { + const text = textValue(value).toLowerCase().replace(/_/gu, "-"); + if (text === "cancelled") return "canceled"; + return text || "unknown"; +} + +export function normalizeTerminalStatus(value) { + const status = normalizeStatus(value); + return TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status) ? status : null; +} + +export function firstUserPreview(messages) { + return messages.find((message) => message.role === "user")?.textPreview ?? null; +} + +export function latestUserPreview(messages) { + return [...messages].reverse().find((message) => message.role === "user")?.textPreview ?? null; +} + +export function latestValidMessagePreview(messages) { + return [...messages].reverse().find((message) => message.role === "user" || isAssistantLikeRole(message.role))?.textPreview ?? null; +} + +export function latestAssistantPreview(messages) { + return [...messages].reverse().find((message) => isAssistantLikeRole(message.role))?.textPreview ?? null; +} + +export function sessionTitleFromMessages(messages = []) { + return boundedPreviewText(latestUserPreview(messages) ?? firstUserPreview(messages) ?? latestAssistantPreview(messages)); +} + +export function sessionPreviewFromMessages(messages = []) { + return boundedPreviewText(latestValidMessagePreview(messages) ?? firstUserPreview(messages) ?? latestAssistantPreview(messages)); +} + +export function boundedPreviewText(value, maxLength = 160) { + const text = textValue(value).replace(/\s+/gu, " ").trim(); + if (!text) return null; + return text.length > maxLength ? `${text.slice(0, maxLength - 3)}...` : text; +} + +export function isAssistantLikeRole(role) { + return role === "assistant" || role === "agent"; +} + +export function objectValue(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +export function textValue(value) { + return String(value ?? "").trim(); +} + +export function messageAuthorityTextValue(value) { + const text = String(value ?? "").replace(/\r\n?/gu, "\n"); + if (!text.trim() || text.trim() === "[object Object]") return ""; + return text; +} + +export function safeTurnId(value) { + const text = textValue(value); + return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; +} + +export function safeMessageId(value) { + const text = textValue(value); + return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; +} + +export function safePartId(value) { + const text = textValue(value); + return /^prt_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; +} + +export function hash(value) { + return createHash("sha256").update(String(value)).digest("hex"); +} diff --git a/internal/cloud/server-workbench-http-test-helpers.ts b/internal/cloud/server-workbench-http-test-helpers.ts new file mode 100644 index 00000000..ba95ccd6 --- /dev/null +++ b/internal/cloud/server-workbench-http-test-helpers.ts @@ -0,0 +1,368 @@ +import assert from "node:assert/strict"; + +export const ACTOR = { id: "usr_workbench_reader", username: "reader", displayName: "Reader", role: "user", status: "active" }; + +export async function getJson(port, path) { + const response = await fetch(`http://127.0.0.1:${port}${path}`); + return { + status: response.status, + body: await response.json() + }; +} + +export async function waitForCondition(predicate, timeoutMs = 500) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + assert.fail("condition was not met before timeout"); +} + +export function createDurableFactsRuntimeStore({ sessions = [], queryError = null, queries = [] } = {}) { + const facts = emptyFacts(); + for (const item of sessions) mergeFacts(facts, buildDurableFactsForSession(item)); + return createRuntimeStoreFromFacts(facts, { queryError, queries }); +} + +export function createRuntimeStoreFromFacts(facts, { queryError = null, queries = [] } = {}) { + return { + queries, + async queryWorkbenchFacts(params = {}) { + queries.push(params); + if (queryError) throw queryError; + const filtered = filterFacts(facts, params); + return { + facts: filtered, + count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), + persistence: { adapter: "test-durable-workbench-facts", durable: true } + }; + } + }; +} + +export function buildDurableFactsForSession({ session, events = [], status = null, finalText = null, assistantText = null, runId = null, commandId = null, projectionStatus = null, projectionHealth = "healthy", lastProjectedSeq = null, omitSessions = false } = {}) { + const traceId = session.lastTraceId ?? session.session?.lastTraceId; + const projectedStatus = normalizeTestStatus(status ?? session.status); + const terminal = testTerminalStatuses.has(projectedStatus); + const normalizedEvents = normalizeTestEvents(events, session, { traceId }); + const timing = testTimingProjection({ session, events: normalizedEvents, status: projectedStatus, terminal }); + const normalizedMessages = normalizeTestMessages(session, { traceId, status: projectedStatus, finalText, terminal, timing }); + const explicitLastProjectedSeq = lastProjectedSeq !== null && lastProjectedSeq !== undefined ? Number(lastProjectedSeq) : NaN; + const projectedSeq = Number.isFinite(explicitLastProjectedSeq) ? explicitLastProjectedSeq : Math.max(normalizedEvents.length, normalizedMessages.length, 0); + const checkpointStatus = projectionStatus ?? (terminal ? "caught_up" : "projecting"); + return { + sessions: omitSessions ? [] : [{ + sessionId: session.id, + ownerUserId: session.ownerUserId, + ownerRole: session.ownerRole ?? null, + projectId: session.projectId ?? null, + conversationId: session.conversationId ?? null, + threadId: session.threadId ?? null, + agentId: session.agentId ?? "hwlab-code-agent", + status: projectedStatus, + lastTraceId: traceId, + providerProfile: session.session?.providerProfile ?? null, + projectedSeq, + sourceSeq: projectedSeq, + sourceEventId: traceId, + terminal, + sealed: terminal, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + createdAt: session.startedAt ?? session.createdAt ?? session.updatedAt ?? null, + updatedAt: session.updatedAt ?? session.endedAt ?? session.startedAt ?? null, + valuesRedacted: true + }], + messages: normalizedMessages, + parts: normalizedMessages.filter((message) => message.text).map((message, index) => { + const partType = message.role !== "user" && message.terminal ? "final_response" : "text"; + return { + partId: `prt_${session.id}_${index}`, + messageId: message.messageId, + sessionId: session.id, + turnId: message.turnId, + traceId: message.traceId, + partIndex: 0, + partType, + status: message.status, + text: message.text, + projectedSeq: message.projectedSeq, + sourceSeq: message.sourceSeq, + sourceEventId: message.sourceEventId, + terminal: message.terminal, + sealed: partType === "final_response" ? true : message.sealed, + updatedAt: message.updatedAt, + valuesRedacted: true + }; + }), + turns: traceId ? [{ + turnId: traceId, + sessionId: session.id, + traceId, + messageId: normalizedMessages.find((message) => message.role !== "user")?.messageId ?? null, + status: projectedStatus, + projectedSeq, + sourceSeq: projectedSeq, + sourceEventId: traceId, + terminal, + sealed: terminal, + assistantText: assistantText ?? null, + finalResponse: finalText ? { text: finalText, status: projectedStatus, traceId, valuesPrinted: false } : null, + diagnostic: { projectionStatus: checkpointStatus, projectionHealth, valuesRedacted: true }, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + createdAt: session.startedAt ?? session.updatedAt ?? null, + updatedAt: session.updatedAt ?? null, + valuesRedacted: true + }] : [], + traceEvents: normalizedEvents, + checkpoints: traceId ? [{ + traceId, + sessionId: session.id, + turnId: traceId, + runId, + commandId, + projectedSeq, + sourceSeq: projectedSeq, + sourceEventId: traceId, + projectionStatus: checkpointStatus, + projectionHealth, + terminal, + sealed: terminal, + diagnostic: { projectionStatus: checkpointStatus, projectionHealth, valuesRedacted: true }, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + createdAt: session.startedAt ?? session.updatedAt ?? null, + updatedAt: session.updatedAt ?? null, + valuesRedacted: true + }] : [] + }; +} + +export function normalizeTestMessages(session, { traceId, status, finalText, terminal, timing }) { + const raw = Array.isArray(session.session?.messages) ? session.session.messages : []; + return raw.map((message, index) => { + const role = message.role ?? (index % 2 === 0 ? "user" : "agent"); + const isAssistant = role === "agent" || role === "assistant"; + const text = isAssistant && finalText ? finalText : String(message.text ?? message.content ?? ""); + const messageStatus = isAssistant && finalText ? status : normalizeTestStatus(message.status ?? (role === "user" ? "sent" : status)); + const messageTraceId = message.traceId ?? traceId; + const projectedSeq = Number.isFinite(Number(message.projectedSeq)) ? Number(message.projectedSeq) : index + 1; + const sourceSeq = Number.isFinite(Number(message.sourceSeq)) ? Number(message.sourceSeq) : projectedSeq; + return { + messageId: message.messageId ?? `msg_${session.id}_${index}`, + sessionId: session.id, + turnId: message.turnId ?? messageTraceId, + traceId: messageTraceId, + role, + status: messageStatus, + projectedSeq, + sourceSeq, + sourceEventId: `${session.id}:message:${index}`, + terminal: isAssistant && terminal, + sealed: isAssistant && terminal, + text, + ...(isAssistant ? { timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs } : {}), + createdAt: message.createdAt ?? session.startedAt ?? session.updatedAt ?? null, + updatedAt: message.updatedAt ?? session.updatedAt ?? null, + valuesRedacted: true + }; + }); +} + +export function testTimingProjection({ session, events = [], status, terminal }) { + const startedAt = timestampIso(session.startedAt ?? session.createdAt ?? events[0]?.createdAt ?? events[0]?.occurredAt); + const lastEvent = events.at(-1) ?? null; + const lastEventAt = timestampIso(lastEvent?.updatedAt ?? lastEvent?.createdAt ?? lastEvent?.occurredAt ?? session.updatedAt); + const finishedAt = terminal ? timestampIso(session.endedAt ?? lastEventAt ?? session.updatedAt) : null; + const durationMs = terminal ? elapsedMs(startedAt, finishedAt ?? lastEventAt) : null; + return { startedAt, lastEventAt, finishedAt, durationMs, status, valuesRedacted: true }; +} + +export function timestampIso(value) { + const ms = Date.parse(String(value ?? "")); + return Number.isFinite(ms) ? new Date(ms).toISOString() : null; +} + +export function elapsedMs(startedAt, endedAt) { + const start = Date.parse(String(startedAt ?? "")); + const end = Date.parse(String(endedAt ?? "")); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null; + return end - start; +} + +export function normalizeTestEvents(events, session, { traceId }) { + return events.map((event, index) => { + const projectedSeq = Number.isFinite(Number(event.projectedSeq)) ? Number(event.projectedSeq) : Number.isFinite(Number(event.seq)) ? Number(event.seq) : index + 1; + const sourceSeq = Number.isFinite(Number(event.sourceSeq)) ? Number(event.sourceSeq) : null; + return { + ...event, + id: event.id ?? `wte_${session.id}_${index}`, + traceId: event.traceId ?? traceId, + sessionId: event.sessionId ?? session.id, + turnId: event.turnId ?? traceId, + messageId: event.messageId ?? null, + sourceSeq, + sourceEventId: event.sourceEventId ?? `${traceId}:${sourceSeq ?? projectedSeq}:${index}`, + projectedSeq, + eventType: event.eventType ?? event.type ?? event.label ?? "event", + terminal: event.terminal === true, + sealed: event.terminal === true, + occurredAt: event.occurredAt ?? event.createdAt ?? session.updatedAt ?? null, + updatedAt: event.updatedAt ?? event.createdAt ?? session.updatedAt ?? null, + valuesRedacted: true + }; + }); +} + +export function filterFacts(facts, params = {}) { + const families = testFactFamilySet(params); + const afterProjectedSeq = Number.parseInt(params.afterProjectedSeq ?? "", 10); + const filtered = { + sessions: families.has("sessions") ? facts.sessions.filter((record) => matchesFact(record, params, ["sessionId", "ownerUserId", "projectId", "conversationId", "threadId", "status"]) && matchesTraceFact(record.lastTraceId, params)) : [], + messages: families.has("messages") ? facts.messages.filter((record) => matchesFact(record, params, ["messageId", "sessionId", "turnId", "traceId", "role", "status"])) : [], + parts: families.has("parts") ? facts.parts.filter((record) => matchesFact(record, params, ["partId", "messageId", "sessionId", "turnId", "traceId", "partType", "status"])) : [], + turns: families.has("turns") ? facts.turns.filter((record) => matchesFact(record, params, ["turnId", "sessionId", "traceId", "messageId", "status"])) : [], + traceEvents: families.has("traceEvents") ? facts.traceEvents + .filter((record) => matchesFact(record, params, ["id", "traceId", "sessionId", "turnId", "messageId", "eventType"])) + .filter((record) => !Number.isInteger(afterProjectedSeq) || afterProjectedSeq <= 0 || Number(record.projectedSeq) > afterProjectedSeq) + .sort((left, right) => Number(left.projectedSeq) - Number(right.projectedSeq)) : [], + checkpoints: families.has("checkpoints") ? facts.checkpoints.filter((record) => matchesFact(record, params, ["traceId", "sessionId", "turnId", "runId", "commandId", "projectionStatus", "projectionHealth"])) : [] + }; + if (workbenchTestFactOrder(params, "sessions") === "updated_desc") { + filtered.sessions.sort((left, right) => String(right.updatedAt ?? "").localeCompare(String(left.updatedAt ?? "")) || String(right.sessionId ?? "").localeCompare(String(left.sessionId ?? ""))); + } + const limit = Number.parseInt(params.limit ?? "", 10); + if (!Number.isInteger(limit) || limit <= 0) return filtered; + return Object.fromEntries(Object.entries(filtered).map(([key, value]) => [key, value.slice(0, limit)])); +} + +export function workbenchTestFactOrder(params = {}, family = "") { + const value = params[`${family}Order`] ?? params.order; + return value === "updated_desc" ? "updated_desc" : "updated_asc"; +} + +export function matchesFact(record, params, fields) { + return fields.every((field) => { + if (params[field] !== undefined && params[field] !== null && record[field] !== params[field]) return false; + const values = Array.isArray(params[`${field}s`]) ? params[`${field}s`].map((item) => String(item ?? "")).filter(Boolean) : []; + return values.length === 0 || values.includes(String(record[field] ?? "")); + }); +} + +export function matchesTraceFact(value, params) { + if (params.traceId && value !== params.traceId) return false; + const traceIds = Array.isArray(params.traceIds) ? params.traceIds.map((item) => String(item ?? "")).filter(Boolean) : []; + return traceIds.length === 0 || traceIds.includes(String(value ?? "")); +} + +export function testFactFamilySet(params = {}) { + const all = ["sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"]; + const raw = params.families ?? params.factFamilies; + if (!raw) return new Set(all); + const values = Array.isArray(raw) ? raw : String(raw).split(","); + const selected = values.map((item) => String(item ?? "").trim()).filter((item) => all.includes(item)); + return new Set(selected.length ? selected : all); +} + +export function mergeFacts(target, source) { + for (const key of Object.keys(target)) target[key].push(...source[key]); + return target; +} + +export function emptyFacts() { + return { sessions: [], messages: [], parts: [], turns: [], traceEvents: [], checkpoints: [] }; +} + +export function normalizeTestStatus(value) { + const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); + if (status === "active" || status === "busy" || status === "processing") return "running"; + if (status === "cancelled") return "canceled"; + return status || "unknown"; +} + +const testTerminalStatuses = new Set(["completed", "failed", "blocked", "timeout", "canceled", "idle"]); + +export async function getSseEvents(port, path, count) { + const controller = new AbortController(); + const response = await fetch(`http://127.0.0.1:${port}${path}`, { signal: controller.signal }); + assert.equal(response.status, 200); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + const events = []; + try { + while (events.length < count) { + const { value, done } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + for (;;) { + const index = buffer.indexOf("\n\n"); + if (index < 0) break; + const block = buffer.slice(0, index); + buffer = buffer.slice(index + 2); + events.push(parseSseBlock(block)); + if (events.length >= count) break; + } + } + } finally { + controller.abort(); + } + return events; +} + +export function parseSseBlock(block) { + const lines = block.split("\n"); + const event = lines.find((line) => line.startsWith("event:"))?.slice(6).trim() ?? "message"; + const id = lines.find((line) => line.startsWith("id:"))?.slice(3).trim() ?? null; + const data = lines.filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).join("\n"); + return { event, id, data: JSON.parse(data) }; +} + +export function createFakeKafkaFactory() { + let eachMessage = null; + let offset = 0; + let subscribedTopic = "hwlab.event.v1"; + const consumer = { + connect: async () => undefined, + subscribe: async (input = {}) => { subscribedTopic = String(input.topic ?? subscribedTopic); }, + run: async (input = {}) => { eachMessage = input.eachMessage; }, + stop: async () => undefined, + disconnect: async () => undefined + }; + return { + factory: () => ({ consumer: () => consumer }), + emit: async (value = {}) => { + await waitFor(() => typeof eachMessage === "function"); + await eachMessage({ + topic: subscribedTopic, + partition: 0, + message: { + offset: String(offset++), + key: Buffer.from(value.traceId ?? value.sessionId ?? "fake"), + timestamp: String(Date.now()), + value: Buffer.from(JSON.stringify(value)) + } + }); + } + }; +} + +export async function waitFor(predicate, timeoutMs = 1000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error("waitFor timed out"); +} diff --git a/internal/cloud/server-workbench-http.test.ts b/internal/cloud/server-workbench-http.test.ts index ecc79f6d..e1f6fff5 100644 --- a/internal/cloud/server-workbench-http.test.ts +++ b/internal/cloud/server-workbench-http.test.ts @@ -11,7 +11,31 @@ import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts"; import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts"; -const ACTOR = { id: "usr_workbench_reader", username: "reader", displayName: "Reader", role: "user", status: "active" }; +import { + ACTOR, + getJson, + waitForCondition, + createDurableFactsRuntimeStore, + createRuntimeStoreFromFacts, + buildDurableFactsForSession, + normalizeTestMessages, + testTimingProjection, + timestampIso, + elapsedMs, + normalizeTestEvents, + filterFacts, + workbenchTestFactOrder, + matchesFact, + matchesTraceFact, + testFactFamilySet, + mergeFacts, + emptyFacts, + normalizeTestStatus, + getSseEvents, + parseSseBlock, + createFakeKafkaFactory, + waitFor +} from "./server-workbench-http-test-helpers.ts"; test("workbench projection diagnostics exposes AgentRun refresh errors without terminal inference", () => { const traceId = "trc_projection_refresh_degraded"; @@ -37,7 +61,6 @@ test("workbench projection diagnostics exposes AgentRun refresh errors without t assert.equal(diagnostics.sourceRunId, "run_projection_refresh_degraded"); assert.equal(diagnostics.sourceCommandId, "cmd_projection_refresh_degraded"); }); - test("workbench projection diagnostics keeps projecting health distinct from caught-up", () => { const traceId = "trc_projection_health_projecting"; const result = { @@ -58,7 +81,6 @@ test("workbench projection diagnostics keeps projecting health distinct from cau assert.equal(diagnostics.projectionStatus, "projecting"); assert.equal(diagnostics.projectionHealth, "projecting"); }); - test("workbench runtime retryable dependency failures classify as 503 (#2020)", () => { const error = new Error("fetch failed"); error.name = "WorkbenchRuntimeDependencyError"; @@ -1526,1541 +1548,3 @@ test("workbench trace events reports metadata gap when turn projection is visibl await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); - -test("workbench trace events reports catch-up page when session projection is visible", async () => { - const traceStore = createCodeAgentTraceStore(); - const traceId = "trc_workbench_trace_events_gap"; - const session = { - id: "ses_workbench_trace_events_gap", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "completed", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_trace_events_gap", - threadId: "thread-workbench-trace-events-gap", - lastTraceId: traceId, - updatedAt: "2026-06-20T13:05:00.000Z", - session: { sessionStatus: "completed", lastTraceId: traceId, messages: [{ role: "user", text: "events gap", traceId }, { role: "agent", text: "OK", traceId }] } - }; - const accessController = { - store: {}, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ session, events: [], status: "completed", finalText: "OK", runId: "run_workbench_trace_events_gap", commandId: "cmd_workbench_trace_events_gap", lastProjectedSeq: 2 }] - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); - assert.equal(turn.status, 200); - assert.equal(turn.body.turn.status, "completed"); - - const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); - assert.equal(trace.status, 200); - assert.equal(trace.body.status, "projecting"); - assert.equal(trace.body.diagnostic.code, "workbench_trace_events_missing"); - assert.equal(trace.body.sessionId, session.id); - assert.equal(trace.body.projectionStatus, "projecting"); - assert.equal(trace.body.projectionHealth, "projecting"); - assert.equal(trace.body.lastProjectedSeq, 2); - assert.equal(trace.body.sourceRunId, "run_workbench_trace_events_gap"); - assert.equal(trace.body.sourceCommandId, "cmd_workbench_trace_events_gap"); - assert.equal(trace.body.blocker, null); - assert.deepEqual(trace.body.events, []); - assert.equal(trace.body.fullTraceLoaded, false); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model projects current turn running state from trace projection", async () => { - const traceStore = createCodeAgentTraceStore(); - const traceId = "trc_workbench_active_state"; - const session = { - id: "ses_workbench_active_state", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "active", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_active_state", - threadId: "thread-workbench-active-state", - lastTraceId: traceId, - updatedAt: "2026-06-17T00:01:00.000Z", - session: { sessionStatus: "active", lastTraceId: traceId, messages: [{ role: "user", text: "still running", traceId }] } - }; - traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false }); - const accessController = { - store: { - async listAgentSessionsForUser() { return [session]; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ - session, - events: [{ seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: session.updatedAt }], - status: "running", - lastProjectedSeq: 1 - }] - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const sessions = await getJson(port, "/v1/workbench/sessions"); - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].status, "running"); - assert.equal(sessions.body.sessions[0].running, true); - assert.equal(sessions.body.sessions[0].terminal, false); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model projects current turn running state for idle session summary", async () => { - const traceStore = createCodeAgentTraceStore(); - const traceId = "trc_workbench_idle_with_running_trace"; - const session = { - id: "ses_workbench_idle_with_running_trace", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "idle", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_idle_with_running_trace", - threadId: "thread-workbench-idle-with-running-trace", - lastTraceId: traceId, - updatedAt: "2026-06-17T00:02:00.000Z", - session: { sessionStatus: "idle", lastTraceId: traceId, messages: [{ role: "user", text: "still running despite idle row", traceId, status: "sent" }] } - }; - traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false }); - const accessController = { - store: { - async listAgentSessionsForUser() { return [session]; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ - session, - events: [{ seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: session.updatedAt }], - status: "running", - lastProjectedSeq: 1 - }] - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].status, "running"); - assert.equal(sessions.body.sessions[0].running, true); - assert.equal(sessions.body.sessions[0].terminal, false); - assert.equal(sessions.body.sessions[0].turnSummary.status, "running"); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model projects newest running turn over stale completed session lastTraceId", async () => { - const oldTraceId = "trc_workbench_stale_completed_old"; - const newTraceId = "trc_workbench_new_running_current"; - const session = { - id: "ses_workbench_stale_completed_new_running", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "completed", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_stale_completed_new_running", - threadId: "thread-workbench-stale-completed-new-running", - lastTraceId: oldTraceId, - updatedAt: "2026-07-01T00:00:02.000Z", - session: { - sessionStatus: "completed", - lastTraceId: oldTraceId, - messages: [ - { messageId: "msg_workbench_old_user", role: "user", text: "first", traceId: oldTraceId, status: "sent", projectedSeq: 10 }, - { messageId: "msg_workbench_old_agent", role: "agent", text: "first done", traceId: oldTraceId, status: "completed", projectedSeq: 20 } - ] - } - }; - const facts = emptyFacts(); - mergeFacts(facts, buildDurableFactsForSession({ session, status: "completed", finalText: "first done", lastProjectedSeq: 20 })); - facts.messages.push( - { messageId: "msg_workbench_new_user", sessionId: session.id, turnId: newTraceId, traceId: newTraceId, role: "user", status: "sent", text: "second", projectedSeq: 30, sourceSeq: 30, sourceEventId: "evt_new_user", terminal: false, sealed: false, updatedAt: "2026-07-01T00:00:03.000Z", valuesRedacted: true }, - { messageId: "msg_workbench_new_agent", sessionId: session.id, turnId: newTraceId, traceId: newTraceId, role: "agent", status: "running", text: "", projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_agent", terminal: false, sealed: false, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true } - ); - facts.turns.push({ turnId: newTraceId, sessionId: session.id, traceId: newTraceId, messageId: "msg_workbench_new_agent", status: "running", projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_turn", terminal: false, sealed: false, diagnostic: { projectionStatus: "projecting", projectionHealth: "healthy", valuesRedacted: true }, timing: { startedAt: "2026-07-01T00:00:03.000Z", lastEventAt: "2026-07-01T00:00:04.000Z", finishedAt: null, durationMs: null, valuesRedacted: true }, startedAt: "2026-07-01T00:00:03.000Z", lastEventAt: "2026-07-01T00:00:04.000Z", finishedAt: null, durationMs: null, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true }); - facts.checkpoints.push({ traceId: newTraceId, sessionId: session.id, turnId: newTraceId, projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_checkpoint", projectionStatus: "projecting", projectionHealth: "healthy", terminal: false, sealed: false, diagnostic: { projectionStatus: "projecting", projectionHealth: "healthy", valuesRedacted: true }, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true }); - const runtimeStore = createRuntimeStoreFromFacts(facts); - const accessController = { - store: { async listAgentSessionsForUser() { return [session]; } }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const server = createCloudApiServer({ accessController, traceStore: createCodeAgentTraceStore(), runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].lastTraceId, newTraceId); - assert.equal(sessions.body.sessions[0].status, "running"); - assert.equal(sessions.body.sessions[0].running, true); - assert.equal(sessions.body.sessions[0].terminal, false); - assert.equal(sessions.body.sessions[0].turnSummary.traceId, newTraceId); - assert.equal(sessions.body.sessions[0].turnSummary.status, "running"); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model projects completed current turn for idle session summary", async () => { - const traceStore = createCodeAgentTraceStore(); - const results = createCodeAgentChatResultStore(); - const traceId = "trc_workbench_idle_with_completed_trace"; - const finalText = "OK"; - const session = { - id: "ses_workbench_idle_with_completed_trace", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "idle", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_idle_with_completed_trace", - threadId: "thread-workbench-idle-with-completed-trace", - lastTraceId: traceId, - updatedAt: "2026-06-17T00:03:00.000Z", - session: { - sessionStatus: "idle", - lastTraceId: traceId, - messages: [ - { role: "user", text: "reply OK", traceId, status: "sent" }, - { role: "agent", text: "partial assistant text", traceId, status: "running" } - ] - } - }; - traceStore.append(traceId, { type: "request", status: "accepted", label: "request:accepted" }); - traceStore.append(traceId, { type: "result", status: "completed", label: "result:completed", terminal: true }); - results.set(traceId, { - status: "completed", - traceId, - ownerUserId: ACTOR.id, - conversationId: session.conversationId, - sessionId: session.id, - threadId: session.threadId, - finalResponse: finalText, - agentRun: { runId: "run_workbench_idle_completed", commandId: "cmd_workbench_idle_completed", status: "completed" } - }); - const accessController = { - store: { - async listAgentSessionsForUser() { return [session]; }, - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } }); - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ - session, - events: [ - { seq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-17T00:02:58.000Z" }, - { seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-17T00:03:00.000Z" } - ], - status: "completed", - finalText, - runId: "run_workbench_idle_completed", - commandId: "cmd_workbench_idle_completed", - lastProjectedSeq: 2 - }] - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results, backendPerformanceStore }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].status, "completed"); - assert.equal(sessions.body.sessions[0].running, false); - assert.equal(sessions.body.sessions[0].terminal, true); - assert.equal(sessions.body.sessions[0].turnSummary.status, "completed"); - - const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`); - assert.equal(messages.status, 200); - assert.equal(messages.body.messages[1].status, "completed"); - assert.equal(messages.body.messages[1].text, finalText); - - const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); - assert.equal(turn.status, 200); - assert.equal(turn.body.turn.status, "completed"); - assert.equal(turn.body.turn.running, false); - assert.equal(turn.body.turn.terminal, true); - assert.equal(turn.body.turn.assistantText, finalText); - assert.equal(turn.body.projectionStatus, "caught-up"); - assert.equal(turn.body.sourceRunId, "run_workbench_idle_completed"); - assert.equal(turn.body.sourceCommandId, "cmd_workbench_idle_completed"); - const metricsText = backendPerformanceStore.metricsText(); - const lagCountLine = metricsText.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_lag_seconds_count") && line.includes('projection_status="caught_up"')); - const turnGetCountLine = metricsText.split("\n").find((line) => line.startsWith("hwlab_workbench_turn_get_duration_seconds_count") && line.includes('route="/v1/workbench/turns/:traceId"')); - assert.ok(lagCountLine?.endsWith(" 1")); - assert.ok(turnGetCountLine?.endsWith(" 1")); - - const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); - assert.equal(trace.status, 200); - assert.equal(trace.body.projectionStatus, "caught-up"); - assert.equal(trace.body.sourceRunId, "run_workbench_idle_completed"); - assert.equal(trace.body.sourceCommandId, "cmd_workbench_idle_completed"); - assert.equal(trace.body.lastProjectedSeq, 2); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model lets terminal result override stale running session state", async () => { - const traceStore = createCodeAgentTraceStore(); - const results = createCodeAgentChatResultStore(); - const traceId = "trc_workbench_stale_running_result_completed"; - const finalText = "OK"; - const session = { - id: "ses_workbench_stale_running_result_completed", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_stale_running_result_completed", - threadId: "thread-workbench-stale-running-result-completed", - lastTraceId: traceId, - updatedAt: "2026-06-18T16:31:04.000Z", - session: { - sessionStatus: "running", - lastTraceId: traceId, - messages: [ - { role: "user", text: "reply OK", traceId, status: "sent" }, - { role: "agent", text: "", traceId, status: "running" } - ] - } - }; - traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false }); - results.set(traceId, { - status: "completed", - traceId, - ownerUserId: ACTOR.id, - conversationId: session.conversationId, - sessionId: session.id, - threadId: session.threadId, - finalResponse: finalText, - agentRun: { runId: "run_workbench_stale_running_completed", commandId: "cmd_workbench_stale_running_completed", status: "completed" } - }); - const accessController = { - store: { - async listAgentSessionsForUser() { return [session]; }, - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ - session, - events: [ - { seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T16:30:00.000Z" }, - { seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-18T16:31:04.000Z" } - ], - status: "completed", - finalText, - runId: "run_workbench_stale_running_completed", - commandId: "cmd_workbench_stale_running_completed", - lastProjectedSeq: 2 - }] - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].status, "completed"); - assert.equal(sessions.body.sessions[0].running, false); - assert.equal(sessions.body.sessions[0].terminal, true); - assert.equal(sessions.body.sessions[0].turnSummary.status, "completed"); - - const detail = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}`); - assert.equal(detail.status, 200); - assert.equal(detail.body.session.status, "completed"); - assert.equal(detail.body.session.running, false); - assert.equal(detail.body.session.terminal, true); - - const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`); - assert.equal(messages.status, 200); - assert.equal(messages.body.messages[1].status, "completed"); - assert.equal(messages.body.messages[1].text, finalText); - - const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); - assert.equal(turn.status, 200); - assert.equal(turn.body.status, "completed"); - assert.equal(turn.body.turn.status, "completed"); - assert.equal(turn.body.turn.running, false); - assert.equal(turn.body.turn.terminal, true); - assert.equal(turn.body.turn.assistantText, finalText); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench session list uses durable projection checkpoint without trace/result hydration", async () => { - const traceStore = createCodeAgentTraceStore(); - const traceId = "trc_workbench_compact_list_summary"; - const finalText = "compact list summary OK"; - const session = { - id: "ses_workbench_compact_list_summary", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_compact_list_summary", - threadId: "thread-workbench-compact-list-summary", - lastTraceId: traceId, - updatedAt: "2026-06-19T15:40:00.000Z", - session: { - sessionStatus: "running", - lastTraceId: traceId, - messages: [ - { role: "user", text: "compact summary", traceId, status: "sent" }, - { role: "agent", text: "", traceId, status: "running" } - ], - traceResults: { - [traceId]: { - traceId, - status: "completed", - finalResponse: { text: finalText, status: "completed", traceId, valuesPrinted: false }, - traceSummary: { - traceId, - source: "agent-session-compact-summary", - sourceEventCount: 42, - terminalStatus: "completed", - agentRun: { runId: "run_workbench_compact_list_summary", commandId: "cmd_workbench_compact_list_summary", lastSeq: 42, valuesPrinted: false }, - valuesPrinted: false - }, - agentRun: { runId: "run_workbench_compact_list_summary", commandId: "cmd_workbench_compact_list_summary", status: "completed", terminalStatus: "completed", lastSeq: 42, valuesPrinted: false }, - updatedAt: "2026-06-19T15:40:00.000Z", - valuesRedacted: true, - secretMaterialStored: false - } - }, - valuesRedacted: true, - secretMaterialStored: false - } - }; - const accessController = { - store: { - async listAgentSessionsForUser() { return [session]; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const factQueries = []; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ - session, - events: [ - { seq: 1, sourceSeq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-19T15:39:10.000Z" }, - { seq: 42, sourceSeq: 42, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-19T15:40:00.000Z" } - ], - status: "completed", - finalText, - runId: "run_workbench_compact_list_summary", - commandId: "cmd_workbench_compact_list_summary", - lastProjectedSeq: 42 - }], - queries: factQueries - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].status, "completed"); - assert.equal(sessions.body.sessions[0].running, false); - assert.equal(sessions.body.sessions[0].terminal, true); - assert.equal(sessions.body.sessions[0].turnSummary.status, "completed"); - assert.equal(sessions.body.sessions[0].turnSummary.eventCount, 42); - assert.equal(sessions.body.sessions[0].projectionStatus, "caught-up"); - assert.ok(factQueries.length >= 1); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench session list overlaps include lookup with the page query", async () => { - const makeSession = (id, traceId, updatedAt) => ({ - id, - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: id.replace(/^ses_/u, "cnv_"), - threadId: `thread-${id}`, - lastTraceId: traceId, - updatedAt, - session: { - sessionStatus: "running", - lastTraceId: traceId, - messages: [{ role: "user", text: id, traceId, status: "sent" }] - } - }); - const included = makeSession("ses_parallel_include", "trc_parallel_include", "2026-06-19T14:00:00.000Z"); - const firstPage = makeSession("ses_parallel_page_a", "trc_parallel_page_a", "2026-06-19T16:00:00.000Z"); - const secondPage = makeSession("ses_parallel_page_b", "trc_parallel_page_b", "2026-06-19T15:00:00.000Z"); - const facts = emptyFacts(); - for (const session of [included, firstPage, secondPage]) mergeFacts(facts, buildDurableFactsForSession({ session, status: "running" })); - const accessController = { - store: {}, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - let pageReleased = false; - let releasePage = () => {}; - const pageGate = new Promise((resolve) => { releasePage = resolve; }); - const queries = []; - const runtimeStore = { - async queryWorkbenchFacts(params = {}) { - queries.push({ ...params, pageReleased }); - if (params.sessionsOrder === "updated_desc") await pageGate; - const filtered = filterFacts(facts, params); - return { - facts: filtered, - count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), - persistence: { adapter: "test-durable-workbench-facts", durable: true } - }; - } - }; - const server = createCloudApiServer({ accessController, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const pending = getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(included.id)}&limit=1`); - await waitForCondition(() => queries.some((query) => query.sessionId === included.id) && queries.some((query) => query.sessionsOrder === "updated_desc")); - const includeIndex = queries.findIndex((query) => query.sessionId === included.id); - assert.ok(includeIndex >= 0); - assert.equal(queries[includeIndex].pageReleased, false); - pageReleased = true; - releasePage(); - const sessions = await pending; - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].sessionId, included.id); - assert.equal(sessions.body.hasMore, true); - assert.equal(queries.find((query) => query.sessionsOrder === "updated_desc")?.sessionProjection, "summary"); - assert.equal(queries.find((query) => query.sessionId === included.id)?.sessionProjection, undefined); - } finally { - pageReleased = true; - releasePage(); - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model reads terminal session status from durable facts", async () => { - const traceStore = createCodeAgentTraceStore(); - const traceId = "trc_workbench_durable_terminal_after_memory_running"; - const session = { - id: "ses_workbench_durable_terminal_after_memory_running", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_durable_terminal_after_memory_running", - threadId: "thread-workbench-durable-terminal-after-memory-running", - lastTraceId: traceId, - updatedAt: "2026-06-18T17:22:00.000Z", - session: { - sessionStatus: "running", - lastTraceId: traceId, - messages: [ - { role: "user", text: "reply OK", traceId, status: "sent" }, - { role: "agent", text: "", traceId, status: "running" } - ] - } - }; - traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false, seq: 1 }); - const durableEvents = [ - { traceId, seq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-18T17:21:40.000Z", valuesPrinted: false }, - { traceId, seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-18T17:22:00.000Z", valuesPrinted: false } - ]; - const accessController = { - store: { - async listAgentSessionsForUser() { return [session]; }, - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ session, events: durableEvents, status: "completed", finalText: "OK", lastProjectedSeq: 2 }] - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); - assert.equal(sessions.status, 200); - assert.equal(sessions.body.sessions[0].status, "completed"); - assert.equal(sessions.body.sessions[0].running, false); - assert.equal(sessions.body.sessions[0].terminal, true); - - const detail = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}`); - assert.equal(detail.status, 200); - assert.equal(detail.body.session.status, "completed"); - assert.equal(detail.body.session.running, false); - assert.equal(detail.body.session.terminal, true); - - const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`); - assert.equal(messages.status, 200); - assert.equal(messages.body.messages[1].status, "completed"); - - const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); - assert.equal(turn.status, 200); - assert.equal(turn.body.status, "completed"); - assert.equal(turn.body.turn.status, "completed"); - assert.equal(turn.body.turn.running, false); - assert.equal(turn.body.turn.terminal, true); - assert.equal(turn.body.turn.trace.eventCount, 2); - assert.equal(turn.body.projectionStatus, "caught-up"); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model keeps non-terminal durable tool completed events out of turn status", async () => { - const traceStore = createCodeAgentTraceStore(); - const results = createCodeAgentChatResultStore(); - const traceId = "trc_workbench_durable_tool_completed_nonterminal"; - const session = { - id: "ses_workbench_durable_tool_completed_nonterminal", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_durable_tool_completed_nonterminal", - threadId: "thread-workbench-durable-tool-completed-nonterminal", - lastTraceId: traceId, - updatedAt: "2026-06-18T19:35:51.787Z", - session: { - sessionStatus: "running", - lastTraceId: traceId, - messages: [ - { role: "user", text: "write benchmark", traceId, status: "sent" }, - { role: "agent", text: "", traceId, status: "running" } - ], - valuesRedacted: true, - secretMaterialStored: false - } - }; - traceStore.append(traceId, { seq: 1, type: "backend_status", status: "running", label: "runner:created", terminal: false }); - const durableEvents = [ - { traceId, seq: 1, type: "assistant_message", status: "running", label: "agentrun:assistant:message", terminal: false, message: "正在安装 Python。", createdAt: "2026-06-18T19:35:40.000Z", valuesPrinted: false }, - { traceId, seq: 2, type: "commandExecution", status: "completed", label: "item/commandExecution:completed", terminal: false, command: "ls -ld .", stdout: "OK\n", createdAt: "2026-06-18T19:35:51.787Z", valuesPrinted: false } - ]; - results.set(traceId, { - status: "running", - traceId, - ownerUserId: ACTOR.id, - conversationId: session.conversationId, - sessionId: session.id, - threadId: session.threadId, - finalResponse: null, - assistantText: null, - agentRun: { runId: "run_workbench_nonterminal_tool_completed", commandId: "cmd_workbench_nonterminal_tool_completed", status: "runner-job-created", runStatus: "pending", commandState: "pending" } - }); - const accessController = { - store: { - async listAgentSessionsForUser() { return [session]; }, - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ - session, - events: durableEvents, - status: "running", - assistantText: "partial assistant text", - runId: "run_workbench_nonterminal_tool_completed", - commandId: "cmd_workbench_nonterminal_tool_completed", - projectionStatus: "projecting", - lastProjectedSeq: 2 - }] - }); - const workbenchRuntime = { - async queryWorkbenchFacts(query) { return runtimeStore.queryWorkbenchFacts(query); } - }; - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, workbenchRuntime, codeAgentChatResults: results }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); - assert.equal(turn.status, 200); - assert.equal(turn.body.status, "running"); - assert.equal(turn.body.turn.status, "running"); - assert.equal(turn.body.turn.running, true); - assert.equal(turn.body.turn.terminal, false); - assert.equal(turn.body.turn.assistantText, null); - assert.equal(turn.body.turn.finalResponse, null); - assert.equal(turn.body.projectionStatus, "projecting"); - - const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); - assert.equal(trace.status, 200); - assert.equal(trace.body.events.at(-1).status, "completed"); - assert.equal(trace.body.events.at(-1).terminal, false); - assert.equal(trace.body.projectionStatus, "projecting"); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench trace events API blocks historical projectedSeq collisions instead of handing them to renderer", async () => { - const traceStore = createCodeAgentTraceStore(); - const traceId = "trc_workbench_projected_seq_collision"; - const session = { - id: "ses_workbench_projected_seq_collision", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_projected_seq_collision", - threadId: "thread-workbench-projected-seq-collision", - lastTraceId: traceId, - updatedAt: "2026-06-20T12:20:00.000Z", - session: { sessionStatus: "running", lastTraceId: traceId, messages: [{ role: "user", text: "collision", traceId, status: "sent" }] } - }; - const accessController = { - store: { - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = createDurableFactsRuntimeStore({ - sessions: [{ - session, - events: [ - { id: "wte_collision_a", projectedSeq: 1, sourceSeq: 1, sourceEventId: "source-a", type: "backend", status: "running", label: "projected:1:a", createdAt: "2026-06-20T12:19:58.000Z" }, - { id: "wte_collision_b", projectedSeq: 1, sourceSeq: 2, sourceEventId: "source-b", type: "assistant", status: "running", label: "projected:1:b", message: "later", createdAt: "2026-06-20T12:19:59.000Z" } - ], - status: "running", - projectionStatus: "projecting", - lastProjectedSeq: 1 - }] - }); - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); - assert.equal(trace.status, 200); - assert.equal(trace.body.status, "blocked"); - assert.equal(trace.body.projectionStatus, "blocked"); - assert.equal(trace.body.projectionHealth, "degraded"); - assert.equal(trace.body.blocker.code, "projected_seq_collision"); - assert.deepEqual(trace.body.events, []); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench realtime stream surfaces facts blocker instead of legacy trace fallback", async () => { - const traceStore = createCodeAgentTraceStore(); - const results = createCodeAgentChatResultStore(); - const traceId = "trc_workbench_realtime"; - const session = { - id: "ses_workbench_realtime", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_realtime", - threadId: "thread-workbench-realtime", - lastTraceId: traceId, - updatedAt: "2026-06-17T02:00:00.000Z", - session: { sessionStatus: "running", lastTraceId: traceId, messages: [{ role: "user", text: "stream", traceId }] } - }; - traceStore.append(traceId, { type: "assistant", status: "completed", label: "assistant:completed", terminal: true, message: "legacy trace answer" }); - results.set(traceId, { status: "completed", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId, finalResponse: { text: "legacy result answer" } }); - const accessController = { - store: { - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const server = createCloudApiServer({ accessController, traceStore, codeAgentChatResults: results, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 4); - const events = await eventsPromise; - assert.deepEqual(events.map((event) => event.event), [ - "workbench.connected", - "workbench.error", - "workbench.trace.snapshot", - "workbench.turn.snapshot" - ]); - assert.equal(events[0].data.filters.sessionId, session.id); - assert.equal(events[1].data.error.code, "workbench_facts_trace_missing"); - assert.equal(events[2].data.traceId, traceId); - assert.equal(events[2].data.snapshot.status, "unknown"); - assert.equal(events[2].data.snapshot.eventCount, 0); - assert.equal(events[3].data.turn.traceId, traceId); - assert.equal(events[3].data.turn.status, "unknown"); - assert.equal(events[3].data.turn.terminal, false); - assert.equal(events[3].data.turn.finalResponse, null); - assert.equal(JSON.stringify(events).includes("legacy result answer"), false); - assert.equal(JSON.stringify(events).includes("legacy trace answer"), false); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench realtime stream forwards HWLAB Kafka events after initial connection", async () => { - const fakeKafka = createFakeKafkaFactory(); - const traceId = "trc_workbench_realtime_after_seq"; - const session = { - id: "ses_workbench_realtime_after_seq", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_realtime_after_seq", - threadId: "thread-workbench-realtime-after-seq", - lastTraceId: traceId, - updatedAt: "2026-06-24T14:00:00.000Z", - session: { sessionStatus: "running", lastTraceId: traceId } - }; - const outboxQueries = []; - const workbenchRuntime = { - async readWorkbenchProjectionOutbox(params = {}) { - outboxQueries.push({ ...params }); - return [ - { outboxSeq: 11, projectedSeq: 7, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, createdAt: "2026-06-24T14:00:01.000Z" } - ]; - } - }; - const accessController = { - store: { - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const serverWithKafka = createCloudApiServer({ - accessController, - workbenchRuntime, - kafkaFactory: fakeKafka.factory, - env: { - HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" - } - }); - await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = serverWithKafka.address(); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_workbench_realtime_after_seq", - traceId, - context: { sourceSeq: 7, runId: "run_workbench_realtime_after_seq", commandId: "cmd_workbench_realtime_after_seq" }, - event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:test", message: "Kafka realtime event", sourceSeq: 7 } - }); }, 25); - const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2); - assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); - assert.equal(events[0].id, "10"); - assert.equal(events[1].id, "hwlab.event.v1:0:0"); - assert.equal(events[1].data.realtimeSource, "kafka"); - assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2"); - assert.equal(events[1].data.entity.family, "traceEvents"); - assert.equal(events[1].data.entity.version, 7); - assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0"); - assert.equal(events[1].data.event.message, "Kafka realtime event"); - assert.equal(events[1].data.kafka.topic, "hwlab.event.v1"); - assert.equal(events[1].data.cursor.traceSeq, 7); - assert.deepEqual(outboxQueries, []); - } finally { - await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench session realtime Kafka stream does not pin subscription to stale lastTraceId", async () => { - const fakeKafka = createFakeKafkaFactory(); - const staleTraceId = "trc_workbench_realtime_stale_trace"; - const liveTraceId = "trc_workbench_realtime_live_trace"; - const session = { - id: "ses_workbench_realtime_session_wide", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_realtime_session_wide", - threadId: "thread-workbench-realtime-session-wide", - lastTraceId: staleTraceId, - updatedAt: "2026-06-24T14:00:00.000Z", - session: { sessionStatus: "running", lastTraceId: staleTraceId } - }; - const accessController = { - store: { - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId() { return null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const serverWithKafka = createCloudApiServer({ - accessController, - workbenchRuntime: { - async queryWorkbenchFacts(params = {}) { - return { - facts: { - sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, threadId: session.threadId, lastTraceId: staleTraceId, status: "running", valuesRedacted: true }], - messages: [], - parts: [], - turns: [], - checkpoints: [] - }, - count: 1, - persistence: { adapter: "test-session-wide-kafka", durable: true }, - params - }; - } - }, - kafkaFactory: fakeKafka.factory, - env: { - HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" - } - }); - await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = serverWithKafka.address(); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_workbench_realtime_session_wide", - traceId: liveTraceId, - context: { runId: "run_workbench_realtime_session_wide", commandId: "cmd_workbench_realtime_session_wide" }, - event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:live", message: "Kafka live trace event" } - }); }, 100); - const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}`, 4); - assert.equal(events[0].event, "workbench.connected"); - const liveEvent = events.find((event) => event.event === "workbench.trace.event" && event.data?.traceId === liveTraceId); - assert.ok(liveEvent, JSON.stringify(events.map((event) => ({ event: event.event, traceId: event.data?.traceId, message: event.data?.event?.message })))); - assert.equal(liveEvent.data.event.message, "Kafka live trace event"); - assert.equal(liveEvent.data.realtimeAuthority, "workbench-realtime-authority-v2"); - } finally { - await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench read model exposes runtime trace projection query failures as projection blockers", async () => { - const traceStore = createCodeAgentTraceStore(); - const results = createCodeAgentChatResultStore(); - const traceId = "trc_workbench_projection_store_unavailable"; - const session = { - id: "ses_workbench_projection_store_unavailable", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_projection_store_unavailable", - threadId: "thread-workbench-projection-store-unavailable", - lastTraceId: traceId, - updatedAt: "2026-06-18T01:10:00.000Z", - session: { - sessionStatus: "running", - lastTraceId: traceId, - messages: [ - { role: "user", text: "projection store unavailable", traceId, status: "sent", createdAt: "2026-06-18T01:09:58.000Z" }, - { role: "agent", text: "", traceId, status: "running", createdAt: "2026-06-18T01:09:59.000Z" } - ], - valuesRedacted: true, - secretMaterialStored: false - } - }; - traceStore.append(traceId, { seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T01:10:00.000Z" }); - results.set(traceId, { status: "running", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId }); - const accessController = { - store: { - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const runtimeStore = { - async queryWorkbenchFacts() { - const error = new Error("runtime query failed"); - error.code = "TEST_RUNTIME_QUERY_FAILED"; - throw error; - } - }; - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); - assert.equal(turn.status, 503); - assert.equal(turn.body.error.code, "projection_store_unavailable"); - - const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); - assert.equal(trace.status, 503); - assert.equal(trace.body.error.code, "projection_store_unavailable"); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench trace events remain visible after session lastTraceId moves", async () => { - const oldTraceId = "trc_workbench_history_old_trace"; - const newTraceId = "trc_workbench_history_new_trace"; - const session = { - id: "ses_workbench_history_trace", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_history_trace", - threadId: "thread-workbench-history-trace", - lastTraceId: newTraceId, - updatedAt: "2026-06-20T10:10:00.000Z", - session: { - messages: [ - { role: "user", text: "old turn", traceId: oldTraceId, status: "sent", createdAt: "2026-06-20T10:00:00.000Z" }, - { role: "agent", text: "old final", traceId: oldTraceId, status: "completed", createdAt: "2026-06-20T10:00:01.000Z" } - ], - valuesRedacted: true - } - }; - const oldFacts = buildDurableFactsForSession({ - session: { ...session, status: "completed", lastTraceId: oldTraceId, updatedAt: "2026-06-20T10:01:00.000Z" }, - status: "completed", - finalText: "old final", - runId: "run_workbench_history_old_trace", - commandId: "cmd_workbench_history_old_trace", - events: [ - { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:00:00.000Z" }, - { projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:01:00.000Z" } - ], - lastProjectedSeq: 2 - }); - const newFacts = buildDurableFactsForSession({ - session: { ...session, session: { messages: [{ role: "user", text: "new turn", traceId: newTraceId, status: "sent" }] } }, - status: "running", - runId: "run_workbench_history_new_trace", - commandId: "cmd_workbench_history_new_trace", - events: [{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:10:00.000Z" }], - lastProjectedSeq: 1 - }); - const facts = emptyFacts(); - mergeFacts(facts, oldFacts); - mergeFacts(facts, newFacts); - facts.sessions = newFacts.sessions; - const queries = []; - const runtimeStore = { - async queryWorkbenchFacts(params = {}) { - queries.push(params); - const filtered = filterFacts(facts, params); - return { - facts: filtered, - count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), - persistence: { adapter: "test-durable-workbench-facts", durable: true } - }; - } - }; - const accessController = { - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const server = createCloudApiServer({ accessController, runtimeStore }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(oldTraceId)}/events?limit=10`); - assert.equal(response.status, 200); - assert.equal(response.body.status, "succeeded"); - assert.equal(response.body.sessionId, session.id); - assert.equal(response.body.traceId, oldTraceId); - assert.equal(response.body.events.length, 2); - assert.equal(response.body.events.at(-1).status, "completed"); - assert.equal(response.body.error, undefined); - assert.equal(queries.some((query) => query.sessionId === session.id && Array.isArray(query.families) && query.families.includes("sessions")), true); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench trace event page keeps per-trace terminal status after later session cancel", async () => { - const completedTraceId = "trc_workbench_completed_before_cancel"; - const canceledTraceId = "trc_workbench_later_cancel"; - const session = { - id: "ses_workbench_completed_before_cancel", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "canceled", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_completed_before_cancel", - threadId: "thread-workbench-completed-before-cancel", - lastTraceId: canceledTraceId, - updatedAt: "2026-06-20T10:20:30.000Z", - session: { - messages: [ - { role: "user", text: "completed first", traceId: completedTraceId, status: "sent", createdAt: "2026-06-20T10:20:00.000Z" }, - { role: "agent", text: "completed final", traceId: completedTraceId, status: "completed", createdAt: "2026-06-20T10:20:10.000Z" }, - { role: "user", text: "cancel later", traceId: canceledTraceId, status: "sent", createdAt: "2026-06-20T10:20:20.000Z" }, - { role: "agent", text: "hwlab-user-cancel", traceId: canceledTraceId, status: "canceled", createdAt: "2026-06-20T10:20:30.000Z" } - ], - valuesRedacted: true - } - }; - const facts = emptyFacts(); - mergeFacts(facts, buildDurableFactsForSession({ - session: { ...session, status: "completed", lastTraceId: completedTraceId, updatedAt: "2026-06-20T10:20:10.000Z" }, - status: "completed", - finalText: "completed final", - events: [ - { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:00.000Z" }, - { projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:20:10.000Z" } - ], - lastProjectedSeq: 2 - })); - mergeFacts(facts, buildDurableFactsForSession({ - session: { ...session, status: "canceled", lastTraceId: canceledTraceId, session: { messages: session.session.messages.slice(2), valuesRedacted: true } }, - status: "canceled", - finalText: "hwlab-user-cancel", - events: [ - { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:20.000Z" }, - { projectedSeq: 2, sourceSeq: 2, type: "cancel", status: "canceled", label: "runner:canceled", terminal: true, createdAt: "2026-06-20T10:20:30.000Z" } - ], - lastProjectedSeq: 2 - })); - facts.sessions = facts.sessions.filter((item) => item.lastTraceId === canceledTraceId); - const runtimeStore = { - async queryWorkbenchFacts(params = {}) { - const filtered = filterFacts(facts, params); - return { - facts: filtered, - count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), - persistence: { adapter: "test-durable-workbench-facts", durable: true } - }; - } - }; - const accessController = { - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(completedTraceId)}/events?limit=10`); - assert.equal(response.status, 200); - assert.equal(response.body.traceStatus, "completed"); - assert.equal(response.body.events.at(-1).status, "completed"); - } finally { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } -}); - -async function getJson(port, path) { - const response = await fetch(`http://127.0.0.1:${port}${path}`); - return { - status: response.status, - body: await response.json() - }; -} - -async function waitForCondition(predicate, timeoutMs = 500) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (predicate()) return; - await new Promise((resolve) => setTimeout(resolve, 5)); - } - assert.fail("condition was not met before timeout"); -} - -function createDurableFactsRuntimeStore({ sessions = [], queryError = null, queries = [] } = {}) { - const facts = emptyFacts(); - for (const item of sessions) mergeFacts(facts, buildDurableFactsForSession(item)); - return createRuntimeStoreFromFacts(facts, { queryError, queries }); -} - -function createRuntimeStoreFromFacts(facts, { queryError = null, queries = [] } = {}) { - return { - queries, - async queryWorkbenchFacts(params = {}) { - queries.push(params); - if (queryError) throw queryError; - const filtered = filterFacts(facts, params); - return { - facts: filtered, - count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), - persistence: { adapter: "test-durable-workbench-facts", durable: true } - }; - } - }; -} - -function buildDurableFactsForSession({ session, events = [], status = null, finalText = null, assistantText = null, runId = null, commandId = null, projectionStatus = null, projectionHealth = "healthy", lastProjectedSeq = null, omitSessions = false } = {}) { - const traceId = session.lastTraceId ?? session.session?.lastTraceId; - const projectedStatus = normalizeTestStatus(status ?? session.status); - const terminal = testTerminalStatuses.has(projectedStatus); - const normalizedEvents = normalizeTestEvents(events, session, { traceId }); - const timing = testTimingProjection({ session, events: normalizedEvents, status: projectedStatus, terminal }); - const normalizedMessages = normalizeTestMessages(session, { traceId, status: projectedStatus, finalText, terminal, timing }); - const explicitLastProjectedSeq = lastProjectedSeq !== null && lastProjectedSeq !== undefined ? Number(lastProjectedSeq) : NaN; - const projectedSeq = Number.isFinite(explicitLastProjectedSeq) ? explicitLastProjectedSeq : Math.max(normalizedEvents.length, normalizedMessages.length, 0); - const checkpointStatus = projectionStatus ?? (terminal ? "caught_up" : "projecting"); - return { - sessions: omitSessions ? [] : [{ - sessionId: session.id, - ownerUserId: session.ownerUserId, - ownerRole: session.ownerRole ?? null, - projectId: session.projectId ?? null, - conversationId: session.conversationId ?? null, - threadId: session.threadId ?? null, - agentId: session.agentId ?? "hwlab-code-agent", - status: projectedStatus, - lastTraceId: traceId, - providerProfile: session.session?.providerProfile ?? null, - projectedSeq, - sourceSeq: projectedSeq, - sourceEventId: traceId, - terminal, - sealed: terminal, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - createdAt: session.startedAt ?? session.createdAt ?? session.updatedAt ?? null, - updatedAt: session.updatedAt ?? session.endedAt ?? session.startedAt ?? null, - valuesRedacted: true - }], - messages: normalizedMessages, - parts: normalizedMessages.filter((message) => message.text).map((message, index) => { - const partType = message.role !== "user" && message.terminal ? "final_response" : "text"; - return { - partId: `prt_${session.id}_${index}`, - messageId: message.messageId, - sessionId: session.id, - turnId: message.turnId, - traceId: message.traceId, - partIndex: 0, - partType, - status: message.status, - text: message.text, - projectedSeq: message.projectedSeq, - sourceSeq: message.sourceSeq, - sourceEventId: message.sourceEventId, - terminal: message.terminal, - sealed: partType === "final_response" ? true : message.sealed, - updatedAt: message.updatedAt, - valuesRedacted: true - }; - }), - turns: traceId ? [{ - turnId: traceId, - sessionId: session.id, - traceId, - messageId: normalizedMessages.find((message) => message.role !== "user")?.messageId ?? null, - status: projectedStatus, - projectedSeq, - sourceSeq: projectedSeq, - sourceEventId: traceId, - terminal, - sealed: terminal, - assistantText: assistantText ?? null, - finalResponse: finalText ? { text: finalText, status: projectedStatus, traceId, valuesPrinted: false } : null, - diagnostic: { projectionStatus: checkpointStatus, projectionHealth, valuesRedacted: true }, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - createdAt: session.startedAt ?? session.updatedAt ?? null, - updatedAt: session.updatedAt ?? null, - valuesRedacted: true - }] : [], - traceEvents: normalizedEvents, - checkpoints: traceId ? [{ - traceId, - sessionId: session.id, - turnId: traceId, - runId, - commandId, - projectedSeq, - sourceSeq: projectedSeq, - sourceEventId: traceId, - projectionStatus: checkpointStatus, - projectionHealth, - terminal, - sealed: terminal, - diagnostic: { projectionStatus: checkpointStatus, projectionHealth, valuesRedacted: true }, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - createdAt: session.startedAt ?? session.updatedAt ?? null, - updatedAt: session.updatedAt ?? null, - valuesRedacted: true - }] : [] - }; -} - -function normalizeTestMessages(session, { traceId, status, finalText, terminal, timing }) { - const raw = Array.isArray(session.session?.messages) ? session.session.messages : []; - return raw.map((message, index) => { - const role = message.role ?? (index % 2 === 0 ? "user" : "agent"); - const isAssistant = role === "agent" || role === "assistant"; - const text = isAssistant && finalText ? finalText : String(message.text ?? message.content ?? ""); - const messageStatus = isAssistant && finalText ? status : normalizeTestStatus(message.status ?? (role === "user" ? "sent" : status)); - const messageTraceId = message.traceId ?? traceId; - const projectedSeq = Number.isFinite(Number(message.projectedSeq)) ? Number(message.projectedSeq) : index + 1; - const sourceSeq = Number.isFinite(Number(message.sourceSeq)) ? Number(message.sourceSeq) : projectedSeq; - return { - messageId: message.messageId ?? `msg_${session.id}_${index}`, - sessionId: session.id, - turnId: message.turnId ?? messageTraceId, - traceId: messageTraceId, - role, - status: messageStatus, - projectedSeq, - sourceSeq, - sourceEventId: `${session.id}:message:${index}`, - terminal: isAssistant && terminal, - sealed: isAssistant && terminal, - text, - ...(isAssistant ? { timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs } : {}), - createdAt: message.createdAt ?? session.startedAt ?? session.updatedAt ?? null, - updatedAt: message.updatedAt ?? session.updatedAt ?? null, - valuesRedacted: true - }; - }); -} - -function testTimingProjection({ session, events = [], status, terminal }) { - const startedAt = timestampIso(session.startedAt ?? session.createdAt ?? events[0]?.createdAt ?? events[0]?.occurredAt); - const lastEvent = events.at(-1) ?? null; - const lastEventAt = timestampIso(lastEvent?.updatedAt ?? lastEvent?.createdAt ?? lastEvent?.occurredAt ?? session.updatedAt); - const finishedAt = terminal ? timestampIso(session.endedAt ?? lastEventAt ?? session.updatedAt) : null; - const durationMs = terminal ? elapsedMs(startedAt, finishedAt ?? lastEventAt) : null; - return { startedAt, lastEventAt, finishedAt, durationMs, status, valuesRedacted: true }; -} - -function timestampIso(value) { - const ms = Date.parse(String(value ?? "")); - return Number.isFinite(ms) ? new Date(ms).toISOString() : null; -} - -function elapsedMs(startedAt, endedAt) { - const start = Date.parse(String(startedAt ?? "")); - const end = Date.parse(String(endedAt ?? "")); - if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null; - return end - start; -} - -function normalizeTestEvents(events, session, { traceId }) { - return events.map((event, index) => { - const projectedSeq = Number.isFinite(Number(event.projectedSeq)) ? Number(event.projectedSeq) : Number.isFinite(Number(event.seq)) ? Number(event.seq) : index + 1; - const sourceSeq = Number.isFinite(Number(event.sourceSeq)) ? Number(event.sourceSeq) : null; - return { - ...event, - id: event.id ?? `wte_${session.id}_${index}`, - traceId: event.traceId ?? traceId, - sessionId: event.sessionId ?? session.id, - turnId: event.turnId ?? traceId, - messageId: event.messageId ?? null, - sourceSeq, - sourceEventId: event.sourceEventId ?? `${traceId}:${sourceSeq ?? projectedSeq}:${index}`, - projectedSeq, - eventType: event.eventType ?? event.type ?? event.label ?? "event", - terminal: event.terminal === true, - sealed: event.terminal === true, - occurredAt: event.occurredAt ?? event.createdAt ?? session.updatedAt ?? null, - updatedAt: event.updatedAt ?? event.createdAt ?? session.updatedAt ?? null, - valuesRedacted: true - }; - }); -} - -function filterFacts(facts, params = {}) { - const families = testFactFamilySet(params); - const afterProjectedSeq = Number.parseInt(params.afterProjectedSeq ?? "", 10); - const filtered = { - sessions: families.has("sessions") ? facts.sessions.filter((record) => matchesFact(record, params, ["sessionId", "ownerUserId", "projectId", "conversationId", "threadId", "status"]) && matchesTraceFact(record.lastTraceId, params)) : [], - messages: families.has("messages") ? facts.messages.filter((record) => matchesFact(record, params, ["messageId", "sessionId", "turnId", "traceId", "role", "status"])) : [], - parts: families.has("parts") ? facts.parts.filter((record) => matchesFact(record, params, ["partId", "messageId", "sessionId", "turnId", "traceId", "partType", "status"])) : [], - turns: families.has("turns") ? facts.turns.filter((record) => matchesFact(record, params, ["turnId", "sessionId", "traceId", "messageId", "status"])) : [], - traceEvents: families.has("traceEvents") ? facts.traceEvents - .filter((record) => matchesFact(record, params, ["id", "traceId", "sessionId", "turnId", "messageId", "eventType"])) - .filter((record) => !Number.isInteger(afterProjectedSeq) || afterProjectedSeq <= 0 || Number(record.projectedSeq) > afterProjectedSeq) - .sort((left, right) => Number(left.projectedSeq) - Number(right.projectedSeq)) : [], - checkpoints: families.has("checkpoints") ? facts.checkpoints.filter((record) => matchesFact(record, params, ["traceId", "sessionId", "turnId", "runId", "commandId", "projectionStatus", "projectionHealth"])) : [] - }; - if (workbenchTestFactOrder(params, "sessions") === "updated_desc") { - filtered.sessions.sort((left, right) => String(right.updatedAt ?? "").localeCompare(String(left.updatedAt ?? "")) || String(right.sessionId ?? "").localeCompare(String(left.sessionId ?? ""))); - } - const limit = Number.parseInt(params.limit ?? "", 10); - if (!Number.isInteger(limit) || limit <= 0) return filtered; - return Object.fromEntries(Object.entries(filtered).map(([key, value]) => [key, value.slice(0, limit)])); -} - -function workbenchTestFactOrder(params = {}, family = "") { - const value = params[`${family}Order`] ?? params.order; - return value === "updated_desc" ? "updated_desc" : "updated_asc"; -} - -function matchesFact(record, params, fields) { - return fields.every((field) => { - if (params[field] !== undefined && params[field] !== null && record[field] !== params[field]) return false; - const values = Array.isArray(params[`${field}s`]) ? params[`${field}s`].map((item) => String(item ?? "")).filter(Boolean) : []; - return values.length === 0 || values.includes(String(record[field] ?? "")); - }); -} - -function matchesTraceFact(value, params) { - if (params.traceId && value !== params.traceId) return false; - const traceIds = Array.isArray(params.traceIds) ? params.traceIds.map((item) => String(item ?? "")).filter(Boolean) : []; - return traceIds.length === 0 || traceIds.includes(String(value ?? "")); -} - -function testFactFamilySet(params = {}) { - const all = ["sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"]; - const raw = params.families ?? params.factFamilies; - if (!raw) return new Set(all); - const values = Array.isArray(raw) ? raw : String(raw).split(","); - const selected = values.map((item) => String(item ?? "").trim()).filter((item) => all.includes(item)); - return new Set(selected.length ? selected : all); -} - -function mergeFacts(target, source) { - for (const key of Object.keys(target)) target[key].push(...source[key]); - return target; -} - -function emptyFacts() { - return { sessions: [], messages: [], parts: [], turns: [], traceEvents: [], checkpoints: [] }; -} - -function normalizeTestStatus(value) { - const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); - if (status === "active" || status === "busy" || status === "processing") return "running"; - if (status === "cancelled") return "canceled"; - return status || "unknown"; -} - -const testTerminalStatuses = new Set(["completed", "failed", "blocked", "timeout", "canceled", "idle"]); - -async function getSseEvents(port, path, count) { - const controller = new AbortController(); - const response = await fetch(`http://127.0.0.1:${port}${path}`, { signal: controller.signal }); - assert.equal(response.status, 200); - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - const events = []; - try { - while (events.length < count) { - const { value, done } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - for (;;) { - const index = buffer.indexOf("\n\n"); - if (index < 0) break; - const block = buffer.slice(0, index); - buffer = buffer.slice(index + 2); - events.push(parseSseBlock(block)); - if (events.length >= count) break; - } - } - } finally { - controller.abort(); - } - return events; -} - -function parseSseBlock(block) { - const lines = block.split("\n"); - const event = lines.find((line) => line.startsWith("event:"))?.slice(6).trim() ?? "message"; - const id = lines.find((line) => line.startsWith("id:"))?.slice(3).trim() ?? null; - const data = lines.filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).join("\n"); - return { event, id, data: JSON.parse(data) }; -} - -function createFakeKafkaFactory() { - let eachMessage = null; - let offset = 0; - let subscribedTopic = "hwlab.event.v1"; - const consumer = { - connect: async () => undefined, - subscribe: async (input = {}) => { subscribedTopic = String(input.topic ?? subscribedTopic); }, - run: async (input = {}) => { eachMessage = input.eachMessage; }, - stop: async () => undefined, - disconnect: async () => undefined - }; - return { - factory: () => ({ consumer: () => consumer }), - emit: async (value = {}) => { - await waitFor(() => typeof eachMessage === "function"); - await eachMessage({ - topic: subscribedTopic, - partition: 0, - message: { - offset: String(offset++), - key: Buffer.from(value.traceId ?? value.sessionId ?? "fake"), - timestamp: String(Date.now()), - value: Buffer.from(JSON.stringify(value)) - } - }); - } - }; -} - -async function waitFor(predicate, timeoutMs = 1000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (predicate()) return; - await new Promise((resolve) => setTimeout(resolve, 10)); - } - throw new Error("waitFor timed out"); -} diff --git a/internal/cloud/server-workbench-http.ts b/internal/cloud/server-workbench-http.ts index be23deaa..98a98eaf 100644 --- a/internal/cloud/server-workbench-http.ts +++ b/internal/cloud/server-workbench-http.ts @@ -1,3053 +1,12 @@ /* - * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-read-model-timeline-contract; draft-2026-06-28-p0-d518-session-timeline-consistency; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-api-contract; PJ2026-010401 Web工作台 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-model-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 - * 职责: Workbench REST read model。只读投影 session/message/turn/trace facts，不执行 AgentRun sync、billing finalize 或 workspace repair。 + * Workbench HTTP compatibility facade. + * Read routes, realtime transport, and fact projections are split by responsibility. */ -import { createHash } from "node:crypto"; - -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { - parsePositiveInteger, - safeConversationId, - safeOpaqueId, - safeSessionId, - safeTraceId, - sendJson -} from "./server-http-utils.ts"; -import { createWorkbenchReadModel } from "./workbench-read-model.ts"; -import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; -import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; -import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; -import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; -import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; - -const DEFAULT_PAGE_LIMIT = 50; -const DEFAULT_SESSION_LIST_LIMIT = 20; -const MAX_PAGE_LIMIT = 100; -const DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS = 15000; -const WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS = 2500; -const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; -const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]); -const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); -const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); -const WORKBENCH_TRACE_EVENT_METADATA_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); -const WORKBENCH_TRACE_EVENT_PAGE_FAMILIES = Object.freeze(["traceEvents"]); -const activeWorkbenchRealtimeConnections = new Set(); - -export async function drainWorkbenchRealtimeConnections(options = {}) { - const reason = textValue(options.reason) || "server_shutdown"; - const signal = textValue(options.signal) || null; - const timeoutMs = parsePositiveInteger(options.timeoutMs, WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS); - const connections = [...activeWorkbenchRealtimeConnections].filter((connection) => connection?.isActive?.()); - for (const connection of connections) { - try { - connection.close({ reason, signal }); - } catch { - // Best effort: one broken client must not block shutdown drain for the rest. - } - } - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const activeAfterClose = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length; - if (activeAfterClose === 0) break; - await sleepMs(Math.min(50, Math.max(1, deadline - Date.now()))); - } - const activeAfter = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length; - return { - ok: activeAfter === 0, - activeBefore: connections.length, - closed: Math.max(0, connections.length - activeAfter), - activeAfter, - timeout: activeAfter > 0, - reason, - signal - }; -} - -export async function handleWorkbenchReadModelHttp(request, response, url, options = {}) { - try { - const perf = options.backendPerformance; - const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); - if (!auth) return; - - if (url.pathname === "/v1/workbench/sync") { - await (perf ? perf.measure("workbench_sync", () => handleWorkbenchSyncHttp(request, response, url, options, auth.actor)) : handleWorkbenchSyncHttp(request, response, url, options, auth.actor)); - return; - } - - if (url.pathname === "/v1/workbench/sessions") { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - await (perf ? perf.measure("workbench_session_list", () => handleWorkbenchSessionList(request, response, url, options, auth.actor)) : handleWorkbenchSessionList(request, response, url, options, auth.actor)); - return; - } - - const sessionMessagesMatch = url.pathname.match(/^\/v1\/workbench\/sessions\/([^/]+)\/messages$/u); - if (sessionMessagesMatch) { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - await (perf ? perf.measure("workbench_message_page", () => handleWorkbenchMessagePage(request, response, url, options, auth.actor, decodeURIComponent(sessionMessagesMatch[1]))) : handleWorkbenchMessagePage(request, response, url, options, auth.actor, decodeURIComponent(sessionMessagesMatch[1]))); - return; - } - - const sessionMatch = url.pathname.match(/^\/v1\/workbench\/sessions\/([^/]+)$/u); - if (sessionMatch) { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - await (perf ? perf.measure("workbench_session_detail", () => handleWorkbenchSessionDetail(request, response, url, options, auth.actor, decodeURIComponent(sessionMatch[1]))) : handleWorkbenchSessionDetail(request, response, url, options, auth.actor, decodeURIComponent(sessionMatch[1]))); - return; - } - - const turnMatch = url.pathname.match(/^\/v1\/workbench\/turns\/([^/]+)$/u); - if (turnMatch) { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - await (perf ? perf.measure("workbench_turn_snapshot", () => handleWorkbenchTurnSnapshot(request, response, url, options, auth.actor, decodeURIComponent(turnMatch[1]))) : handleWorkbenchTurnSnapshot(request, response, url, options, auth.actor, decodeURIComponent(turnMatch[1]))); - return; - } - - const traceEventsMatch = url.pathname.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u); - if (traceEventsMatch) { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - await (perf ? perf.measure("workbench_trace_events", () => handleWorkbenchTraceEventPage(request, response, url, options, auth.actor, decodeURIComponent(traceEventsMatch[1]))) : handleWorkbenchTraceEventPage(request, response, url, options, auth.actor, decodeURIComponent(traceEventsMatch[1]))); - return; - } - - sendJson(response, 404, workbenchError("workbench_route_not_found", "Workbench read model route is not implemented.", { route: url.pathname })); - } catch (error) { - handleWorkbenchReadModelFailure(response, url, options, error); - } -} - -export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) { - try { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); - const requestedTraceId = safeTraceId(url.searchParams.get("traceId")); - const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS); - attachWorkbenchRealtimeOtelContext(request, { - sessionId: requestedSessionId, - traceId: requestedTraceId, - heartbeatMs, - realtimeSource: "kafka" - }); - const perf = options.backendPerformance; - const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); - if (!auth) return; - - if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) { - sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench read model is keyed by sessionId/threadId/traceId only.")); - return; - } - - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const readModel = createWorkbenchReadModel(options, auth.actor); - const requestedAfterSeq = workbenchRealtimeAfterSeq(request, url); - let closed = false; - let realtimeCloseReason = "client_close"; - let realtimeCloseSignal = null; - let realtimeSessionId = requestedSessionId ?? null; - let realtimeTraceId = requestedTraceId ?? null; - let realtimeThreadId = null; - const realtimeStartedAtMs = Date.now(); - const cleanup = []; - - response.writeHead(200, { - "content-type": "text/event-stream; charset=utf-8", - "cache-control": "no-store, no-transform", - connection: "keep-alive", - "x-accel-buffering": "no", - "x-content-type-options": "nosniff" - }); - if (typeof response.flushHeaders === "function") response.flushHeaders(); - - const writeEvent = (name, payload = {}) => { - if (closed || response.destroyed) return; - const startedAt = nowMs(); - const eventCreatedAt = realtimeEventCreatedAt(payload); - const traceSeq = realtimeTraceSeq(payload); - const eventId = realtimeEventId(payload); - response.write(`event: ${name}\n`); - if (eventId) response.write(`id: ${eventId}\n`); - response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`); - perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: "ok" }); - }; - - const realtimeConnection = { - close(fields = {}) { - if (closed || response.destroyed || response.writableEnded) return false; - const reason = textValue(fields.reason) || "server_shutdown"; - realtimeCloseReason = reason; - realtimeCloseSignal = textValue(fields.signal) || null; - try { - writeEvent("workbench.server_draining", { - type: "server.draining", - status: "closing", - reason, - signal: realtimeCloseSignal, - sessionId: realtimeSessionId, - threadId: realtimeThreadId, - traceId: realtimeTraceId, - observedAt: new Date().toISOString() - }); - response.end(); - return true; - } catch { - try { response.destroy(); } catch {} - return false; - } - }, - isActive() { - return !closed && !response.destroyed && !response.writableEnded; - } - }; - activeWorkbenchRealtimeConnections.add(realtimeConnection); - cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); - - writeEvent("workbench.connected", { - type: "connected", - status: "connected", - realtimeSource: "kafka", - snapshotSource: "read-model", - heartbeatMs, - cursor: { outboxSeq: requestedAfterSeq }, - filters: { - sessionId: requestedSessionId, - traceId: requestedTraceId - } - }); - - const resolveInitialSession = requestedSessionId && requestedAfterSeq <= 0; - const initialContext = resolveInitialSession - ? await realtimeInitialSessionContext(readModel, auth.actor, requestedSessionId) - : { facts: emptyFactSet(), session: null, blocker: null }; - if (initialContext.blocker) { - writeEvent("workbench.error", { - type: "error", - reason: "initial-session", - sessionId: requestedSessionId, - traceId: requestedTraceId, - error: initialContext.blocker - }); - } - const initialFactSession = initialContext.session; - const streamSessionId = factSessionId(initialFactSession) ?? requestedSessionId ?? null; - const streamThreadId = safeOpaqueId(initialFactSession?.threadId) ?? (textValue(initialFactSession?.threadId) || null); - const activeTraceId = requestedTraceId - ?? factLastTraceId(initialFactSession) - ?? factLatestTraceIdForSession(initialContext.facts, streamSessionId) - ?? null; - realtimeSessionId = streamSessionId ?? requestedSessionId ?? null; - realtimeTraceId = activeTraceId ?? requestedTraceId ?? null; - realtimeThreadId = streamThreadId ?? null; - attachWorkbenchRealtimeOtelContext(request, { - sessionId: streamSessionId ?? requestedSessionId, - traceId: activeTraceId, - threadId: streamThreadId, - heartbeatMs, - realtimeSource: "kafka" - }); - emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); - if (activeTraceId && requestedAfterSeq <= 0) { - await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })); - } - const kafkaFilters = resolveWorkbenchRealtimeKafkaFilters({ traceId: requestedTraceId, sessionId: streamSessionId, traceStore }); - try { - const kafkaStream = await openKafkaEventStream({ - env: options.env ?? process.env, - stream: "hwlab", - fromBeginning: false, - ...kafkaFilters, - kafkaFactory: options.kafkaFactory, - onEvent: async (record) => { - if (closed || response.destroyed) return; - const realtime = workbenchRealtimeEventFromKafka(record, { sessionId: streamSessionId, threadId: streamThreadId, traceId: activeTraceId }); - if (!realtime) return; - writeEvent("workbench.trace.event", realtime.traceEvent); - if (realtime.turnSnapshot) writeEvent("workbench.turn.snapshot", realtime.turnSnapshot); - }, - onError: (error) => { - writeEvent("workbench.error", { - type: "error", - realtimeSource: "kafka", - sessionId: streamSessionId, - threadId: streamThreadId, - traceId: activeTraceId, - error: { code: "workbench_kafka_stream_failed", message: error?.message ?? "Workbench Kafka realtime stream failed." } - }); - } - }); - cleanup.push(() => { void kafkaStream.stop?.(); }); - } catch (kafkaError) { - writeEvent("workbench.error", { - type: "error", - realtimeSource: "kafka", - sessionId: streamSessionId, - threadId: streamThreadId, - traceId: activeTraceId, - error: { code: "workbench_kafka_stream_unavailable", message: kafkaError?.message ?? "Workbench Kafka realtime stream is unavailable." } - }); - } - - const heartbeat = setInterval(async () => { - try { - writeEvent("workbench.heartbeat", { - type: "heartbeat", - sessionId: requestedSessionId ?? null, - traceId: activeTraceId ?? null, - observedAt: new Date().toISOString() - }); - } catch (error) { - writeEvent("workbench.error", { - type: "error", - error: { code: "workbench_realtime_heartbeat_failed", message: error?.message ?? "Workbench realtime heartbeat failed." } - }); - } - }, Math.max(1000, heartbeatMs)); - cleanup.push(() => clearInterval(heartbeat)); - - response.on("close", () => { - if (closed) return; - closed = true; - emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { - reason: realtimeCloseReason, - signal: realtimeCloseSignal, - startedAtMs: realtimeStartedAtMs, - sessionId: realtimeSessionId, - threadId: realtimeThreadId, - traceId: realtimeTraceId, - activeConnectionCount: activeWorkbenchRealtimeConnections.size - }); - for (const item of cleanup.splice(0)) item(); - }); - } catch (error) { - handleWorkbenchRealtimeFailure(request, response, url, options, error); - } -} - -function attachWorkbenchRealtimeOtelContext(request, fields = {}) { - const context = request?.hwlabHttpRequestContext; - if (!context) return; - context.route = "/v1/workbench/events"; - context.otelAttributes = { - ...(context.otelAttributes ?? {}), - "workbench.route": "events", - "workbench.session_id": fields.sessionId ?? null, - traceId: fields.traceId ?? null, - "workbench.trace_id": fields.traceId ?? null, - "workbench.thread_id": fields.threadId ?? null, - "workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null, - "workbench.sse.realtime_source": fields.realtimeSource ?? null, - "workbench.sse.outbox_mode": false - }; -} - -function resolveWorkbenchRealtimeKafkaFilters({ traceId = null, sessionId = null, traceStore = null } = {}) { - const resolved = traceId && typeof traceStore?.snapshot === "function" ? collectTraceLinkedIds(traceStore.snapshot(traceId)) : {}; - const hasAgentRunKey = Boolean(resolved.runId || resolved.commandId); - const fallbackSessionId = resolved.sessionId || agentRunScopedSessionIdFromWorkbenchSession(sessionId) || sessionId; - return compactObject({ - traceId: hasAgentRunKey ? null : traceId, - sessionId: hasAgentRunKey ? null : fallbackSessionId, - runId: resolved.runId, - commandId: resolved.commandId - }); -} - -function agentRunScopedSessionIdFromWorkbenchSession(sessionId) { - const safeId = safeSessionId(sessionId); - if (!safeId || isAgentRunAliasSessionId(safeId)) return safeId; - const base = safeId.replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session"; - return safeSessionId(`ses_agentrun_${base}`); -} - -function collectTraceLinkedIds(snapshot) { - const out = { sessionId: null, runId: null, commandId: null }; - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - for (const event of events) collectTraceLinkedIdsFromRecord(event, out); - collectTraceLinkedIdsFromRecord(snapshot?.lastEvent, out); - return out; -} - -function collectTraceLinkedIdsFromRecord(record, out) { - const value = record && typeof record === "object" && !Array.isArray(record) ? record : {}; - const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {}; - const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {}; - out.sessionId ||= safeSessionId(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId); - out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId); - out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId); -} - -function workbenchRealtimeEventFromKafka(record, context = {}) { - const value = record?.value && typeof record.value === "object" && !Array.isArray(record.value) ? record.value : null; - if (!value) return null; - const hwlabEvent = value.event && typeof value.event === "object" && !Array.isArray(value.event) ? value.event : {}; - const sourceContext = value.context && typeof value.context === "object" && !Array.isArray(value.context) ? value.context : {}; - const traceId = safeTraceId(value.traceId ?? hwlabEvent.traceId ?? context.traceId) ?? textValue(value.traceId ?? hwlabEvent.traceId ?? context.traceId); - const sessionId = safeSessionId(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId) ?? textValue(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId); - const threadId = safeOpaqueId(context.threadId ?? sourceContext.threadId) ?? textValue(context.threadId ?? sourceContext.threadId); - if (!traceId && !sessionId) return null; - const sourceSeq = integerValue(hwlabEvent.sourceSeq ?? sourceContext.sourceSeq); - const kafka = { - topic: textValue(record.topic), - partition: integerValue(record.partition), - offset: textValue(record.offset), - key: textValue(record.key), - timestamp: textValue(record.timestamp), - valuesRedacted: true - }; - const event = { - ...hwlabEvent, - traceId: traceId ?? hwlabEvent.traceId ?? null, - sessionId: sessionId ?? hwlabEvent.sessionId ?? null, - threadId: threadId ?? sourceContext.threadId ?? null, - runId: textValue(hwlabEvent.runId ?? sourceContext.runId), - commandId: textValue(hwlabEvent.commandId ?? sourceContext.commandId), - source: textValue(hwlabEvent.source) || "hwlab.kafka", - sourceSeq, - kafka, - valuesPrinted: false - }; - const status = normalizeStatus(event.status); - const terminal = event.terminal === true || TERMINAL_STATUSES.has(status); - const cursor = { - traceSeq: sourceSeq ?? integerValue(record.offset), - kafkaOffset: textValue(record.offset), - kafkaPartition: integerValue(record.partition) - }; - const entityVersion = sourceSeq ?? integerValue(record.offset) ?? 0; - const projectionRevision = ["kafka", kafka.topic, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"); - const traceEntity = { - family: "traceEvents", - id: [traceId ?? "trace", event.runId, event.commandId, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"), - version: entityVersion, - traceSeq: cursor.traceSeq, - projectionRevision, - authority: WORKBENCH_REALTIME_AUTHORITY_VERSION - }; - const traceEvent = { - id: kafka.topic && kafka.partition !== null && kafka.offset ? `${kafka.topic}:${kafka.partition}:${kafka.offset}` : null, - type: "trace.event", - contractVersion: "workbench-sync-v1", - realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, - realtimeSource: "kafka", - sessionId, - threadId, - traceId, - event, - snapshot: { traceId, sessionId, threadId, status: status || event.status || null, events: [event], eventCount: 1 }, - cursor, - entity: traceEntity, - projectionRevision, - kafka, - valuesPrinted: false - }; - const turnSnapshot = terminal ? { - type: "turn.snapshot", - contractVersion: "workbench-sync-v1", - realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, - realtimeSource: "kafka", - sessionId, - threadId, - traceId, - reason: "kafka-terminal", - cursor, - turn: { - traceId, - sessionId, - threadId, - status: status || event.status || "completed", - running: false, - terminal: true, - finalResponse: terminalFinalResponse(status || event.status || "completed", { - traceId, - status: status || event.status || "completed", - finalResponse: event.text ? { text: event.text, status: status || event.status || "completed", traceId, source: "kafka-terminal-event", valuesPrinted: false } : null, - finalText: event.text ?? event.message ?? null, - error: event.errorCode ? { message: event.message ?? event.errorCode } : null - }), - agentRun: compactObject({ runId: event.runId, commandId: event.commandId }), - valuesPrinted: false - }, - entity: { - family: "turns", - id: traceId ?? event.runId ?? event.commandId ?? traceEntity.id, - version: entityVersion, - traceSeq: cursor.traceSeq, - projectionRevision, - authority: WORKBENCH_REALTIME_AUTHORITY_VERSION - }, - projectionRevision, - kafka, - valuesPrinted: false - } : null; - return { traceEvent, turnSnapshot }; -} - -function compactObject(value) { - return Object.fromEntries(Object.entries(value || {}).filter(([, entry]) => textValue(entry))); -} - -function integerValue(value) { - const number = Number(value); - return Number.isFinite(number) ? Math.trunc(number) : null; -} - -function attachWorkbenchReadModelDiagnosticOtel(request, fields = {}) { - const context = request?.hwlabHttpRequestContext; - if (!context) return; - const code = textValue(fields.code) || "workbench_read_model_not_found"; - const route = textValue(fields.route) || context.route || "/v1/workbench"; - context.route = route; - context.lastHttpError = { - code, - category: "workbench_read_model", - layer: "workbench.read_model" - }; - context.otelAttributes = { - ...(context.otelAttributes ?? {}), - "workbench.route": route, - "workbench.read_model.route": route, - "workbench.read_model.result": code, - "workbench.read_model.count": Number.isFinite(Number(fields.count)) ? Math.trunc(Number(fields.count)) : null, - "workbench.session_id": fields.sessionId ?? null, - "workbench.trace_id": fields.traceId ?? null, - "workbench.turn_id": fields.turnId ?? null, - traceId: fields.traceId ?? null, - "error.code": code, - "error.category": "workbench_read_model", - "error.layer": "workbench.read_model" - }; -} - -function emitWorkbenchRealtimeAcceptedOtelSpan(request, env = process.env) { - const context = request?.hwlabHttpRequestContext; - if (!context?.traceId) return; - const endedAtMs = Date.now(); - void emitHttpServerRequestSpan(context, env, { - startTimeMs: context.startedAtMs, - endTimeMs: endedAtMs, - statusCode: 200, - method: "GET", - route: "/v1/workbench/events", - attributes: { - ...(context.otelAttributes ?? {}), - "http.closed_by": "sse-accepted", - "workbench.sse.lifecycle": "accepted", - "workbench.sse.accept_latency_ms": Math.max(0, endedAtMs - Number(context.startedAtMs ?? endedAtMs)) - } - }).catch(() => undefined); -} - -function emitWorkbenchRealtimeClosedOtelSpan(request, env = process.env, fields = {}) { - const context = request?.hwlabHttpRequestContext; - if (!context?.traceId) return; - const endedAtMs = Date.now(); - const startedAtMs = Number(fields.startedAtMs ?? context.startedAtMs ?? endedAtMs); - const reason = textValue(fields.reason) || "client_close"; - void emitHttpServerRequestSpan(context, env, { - startTimeMs: Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs, - endTimeMs: endedAtMs, - statusCode: 200, - method: "GET", - route: "/v1/workbench/events", - attributes: { - ...(context.otelAttributes ?? {}), - "http.closed_by": reason, - "workbench.sse.lifecycle": "closed", - "workbench.sse.close_reason": reason, - "workbench.sse.close_signal": fields.signal ?? null, - "workbench.sse.duration_ms": Math.max(0, endedAtMs - (Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs)), - "workbench.sse.session_id": fields.sessionId ?? null, - "workbench.sse.thread_id": fields.threadId ?? null, - "workbench.sse.trace_id": fields.traceId ?? null, - "workbench.sse.active_connection_count": Number.isFinite(Number(fields.activeConnectionCount)) ? Number(fields.activeConnectionCount) : null - } - }).catch(() => undefined); -} - -function handleWorkbenchRealtimeFailure(request, response, url, options = {}, error) { - const classified = classifyWorkbenchReadModelFailure(error); - const context = request?.hwlabHttpRequestContext; - if (context) { - context.route = "/v1/workbench/events"; - context.otelAttributes = { - ...(context.otelAttributes ?? {}), - "workbench.route": "events", - "workbench.sse.lifecycle": "failed", - "error.code": classified.code, - "error.layer": "workbench.realtime" - }; - } - logWorkbenchReadModelFailure(options.logger ?? console, { - event: "workbench_realtime_route_failed", - ok: false, - route: "/v1/workbench/events", - statusCode: classified.statusCode, - errorCode: classified.code, - errorName: error?.name ?? "Error", - message: error instanceof Error ? error.message : String(error ?? "unknown"), - valuesRedacted: true - }); - if (response.headersSent && !response.destroyed) { - try { - response.write(`event: workbench.error\n`); - response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), type: "error", error: { code: classified.code, message: classified.message, valuesRedacted: true } })}\n\n`); - response.end(); - } catch { - try { response.destroy(); } catch {} - } - return; - } - if (!response.destroyed) sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { route: "/v1/workbench/events", errorName: error?.name ?? "Error" })); -} - -async function writeMessageRealtimeSnapshotSafe({ writeEvent, readModel, actor, sessionId, threadId = null, traceId = null, reason = "message", cursor = null } = {}) { - const safeSession = safeSessionId(sessionId); - if (!safeSession || typeof readModel?.queryFacts !== "function") return; - try { - const result = await queryFactsByRouteId(readModel, safeSession, { families: WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES }); - if (result.error) return; - const session = visibleFactSessions(result.facts, actor)[0] ?? null; - if (!session) return; - const messages = factMessagesForSession(session, result.facts); - const message = [...messages].reverse().find((item) => { - if (!isAssistantLikeRole(item?.role)) return false; - return !traceId || item.traceId === traceId; - }) ?? null; - if (!message) return; - writeEvent("workbench.message.snapshot", { - type: "message.snapshot", - sessionId: factSessionId(session), - threadId: threadId ?? session.threadId ?? null, - traceId: message.traceId ?? traceId ?? null, - reason, - cursor, - message - }); - } catch (error) { - writeEvent("workbench.error", { - type: "error", - error: { code: "workbench_message_snapshot_failed", message: error?.message ?? "Workbench realtime message snapshot failed." } - }); - } -} - -function realtimeEventCreatedAt(payload) { - const candidates = [payload?.event?.createdAt, payload?.snapshot?.lastEvent?.createdAt, payload?.turn?.updatedAt]; - for (const value of candidates) { - const text = textValue(value); - if (text) return text; - } - return null; -} - -function realtimeTraceSeq(payload) { - const seq = Number(payload?.cursor?.traceSeq ?? payload?.event?.seq ?? payload?.snapshot?.lastEvent?.seq); - return Number.isFinite(seq) && seq >= 0 ? seq : null; -} - -function realtimeEventId(payload) { - const raw = payload?.cursor?.outboxSeq ?? payload?.outboxSeq ?? payload?.id; - const text = textValue(raw); - if (!text || /[\r\n]/u.test(text)) return null; - return text.slice(0, 128); -} - -function workbenchRealtimeAfterSeq(request, url) { - for (const value of [url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq"), request?.headers?.["last-event-id"]]) { - const parsed = nonNegativeInteger(value, 0); - if (parsed > 0) return parsed; - } - return 0; -} - -function nowMs() { - if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); - return Date.now(); -} - -function sleepMs(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function recordWorkbenchTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) { - const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; - if (!performanceStore?.recordWorkbenchTurnRead) return; - performanceStore.recordWorkbenchTurnRead({ - route: url?.pathname ?? "/v1/workbench/turns/:id", - status: body?.status ?? statusClassLabel(statusCode), - degradedReason: workbenchTurnDegradedReason(body), - durationMs: nowMs() - startedAt, - responseBytes: responseBodyBytes(body), - timedOut: statusCode === 504 || body?.error?.code === "timeout" - }); -} - -function recordWorkbenchCacheMetric(options = {}, url, body = {}, startedAt = nowMs()) { - const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; - if (!performanceStore?.recordWorkbenchCache) return; - const cache = body?.cache && typeof body.cache === "object" ? body.cache : null; - if (!cache) return; - performanceStore.recordWorkbenchCache({ - route: url?.pathname ?? "/v1/workbench", - cacheKeyClass: cache.class, - cacheStatus: cache.status, - operation: "read", - durationMs: nowMs() - startedAt, - payloadBytes: cache.payloadBytes, - cacheAgeMs: cache.cacheAgeMs, - dbQueryAvoided: cache.dbQueryAvoided, - freshnessSloMs: cache.freshnessSloMs, - projectionSeq: cache.projectionSeq - }); -} - -function recordWorkbenchProjectionMetric(options = {}, { result = null, trace = null, projection = null, diagnostic = null } = {}) { - const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; - if (!performanceStore?.recordWorkbenchProjection || !projection) return; - const sourceLatestSeq = workbenchProjectionSourceLatestSeq(result, trace, projection); - const lastProjectedSeq = Number(diagnostic?.lastProjectedSeq ?? projection?.lastProjectedSeq ?? 0); - if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return; - performanceStore.recordWorkbenchProjection({ - sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq, - lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0, - sourceLatestEventAt: workbenchProjectionSourceLatestAt(result, trace, projection), - projectedLatestEventAt: diagnostic?.updatedAt ?? projection?.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null, - terminalSourceAt: workbenchTerminalSourceAt(result, projection), - terminalProjectedAt: diagnostic?.projectionStatus === "caught-up" ? diagnostic?.updatedAt ?? projection?.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null, - status: projection?.status ?? "unknown", - reason: workbenchProjectionReason(diagnostic), - projectionStatus: diagnostic?.projectionStatus ?? "unknown", - source: "workbench_read_model" - }); -} - -function workbenchProjectionSourceLatestSeq(result = null, trace = null, projection = null) { - const candidates = [ - result?.agentRun?.lastSeq, - result?.traceSummary?.agentRun?.lastSeq, - result?.traceSummary?.lastSeq, - result?.providerTrace?.lastSeq, - result?.runnerTrace?.lastEvent?.sourceSeq, - result?.runnerTrace?.lastEvent?.seq, - result?.runnerTrace?.eventCount, - trace?.lastSeq, - trace?.lastEvent?.sourceSeq, - trace?.lastEvent?.seq, - trace?.eventCount, - projection?.lastProjectedSeq, - projection?.eventCount - ]; - for (const value of candidates) { - const seq = Number(value); - if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq); - } - return NaN; -} - -function workbenchProjectionSourceLatestAt(result = null, trace = null, projection = null) { - return result?.traceSummary?.updatedAt - ?? result?.agentRun?.updatedAt - ?? result?.providerTrace?.updatedAt - ?? result?.runnerTrace?.updatedAt - ?? trace?.updatedAt - ?? result?.updatedAt - ?? projection?.updatedAt - ?? null; -} - -function workbenchTerminalSourceAt(result = null, projection = null) { - if (projection?.terminal !== true) return null; - return result?.traceSummary?.updatedAt - ?? result?.agentRun?.updatedAt - ?? result?.updatedAt - ?? projection?.updatedAt - ?? null; -} - -function workbenchProjectionReason(diagnostic = null) { - if (diagnostic?.blocker) return diagnostic.blocker.code ?? "projection_blocked"; - if (diagnostic?.projectionStatus === "projecting") return "projecting"; - if (diagnostic?.projectionStatus === "caught-up") return "none"; - return diagnostic?.projectionStatus ?? "unknown"; -} - -function workbenchTurnDegradedReason(body = {}) { - if (body?.projection?.blocker?.code) return body.projection.blocker.code; - if (body?.blocker?.code) return body.blocker.code; - if (body?.error?.code) return body.error.code; - if (body?.projectionStatus === "projecting") return "projecting"; - if (body?.projectionStatus === "blocked") return "projection_blocked"; - return "none"; -} - -function responseBodyBytes(body = {}) { - try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; } -} - -function statusClassLabel(value) { - const status = Number(value); - if (!Number.isFinite(status) || status <= 0) return "unknown"; - return `${Math.floor(status / 100)}xx`; -} - -async function authenticateWorkbenchRead(request, response, options) { - const access = options.accessController; - if (!access?.authenticate) { - sendJson(response, 503, workbenchError("workbench_access_controller_missing", "Workbench read model requires access controller.")); - return null; - } - await access.ensureBootstrap?.(); - const auth = await access.authenticate(request, { required: true }); - if (!auth?.ok) { - sendJson(response, auth?.status ?? 401, auth ?? workbenchError("auth_required", "Authentication is required.")); - return null; - } - return auth; -} - -async function handleWorkbenchSessionList(request, response, url, options, actor) { - const startedAt = nowMs(); - if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) return sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench session list is keyed by sessionId only.")); - const limit = boundedSessionListLimit(url.searchParams.get("limit")); - const offset = cursorOffset(url.searchParams.get("cursor") ?? url.searchParams.get("after")); - const includeSessionId = safeSessionId(url.searchParams.get("includeSessionId")); - const includeRouteId = includeSessionId ?? safeConversationId(url.searchParams.get("includeSessionId")); - const runtime = workbenchRuntimeReader(request, options); - if (typeof runtime?.listWorkbenchSessions !== "function") { - if (typeof runtime?.queryWorkbenchFacts !== "function") { - throw runtimeDependencyError("workbench_runtime_unconfigured", "Workbench runtime service is required for session list.", false); - } - const readModel = workbenchRuntimeFactsReadModel(request, options, actor); - const payload = await handleWorkbenchSessionListFromFacts(readModel, url, options, actor, { limit, offset, includeRouteId }); - recordWorkbenchCacheMetric(options, url, payload, startedAt); - emitWorkbenchSessionListReadOtel(request, options, { - startTimeMs: startedAt, - statusCode: 200, - source: "facts", - limit, - offset, - includeSessionId: includeRouteId, - payload - }); - return sendJson(response, 200, payload); - } - const payload = await runtime.listWorkbenchSessions({ - limit, - offset, - cursor: url.searchParams.get("cursor") ?? url.searchParams.get("after") ?? null, - includeSessionId: includeRouteId ?? null, - actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } - }); - recordWorkbenchCacheMetric(options, url, payload, startedAt); - emitWorkbenchSessionListReadOtel(request, options, { - startTimeMs: startedAt, - statusCode: 200, - source: "runtime", - limit, - offset, - includeSessionId: includeRouteId, - payload - }); - return sendJson(response, 200, payload); -} - -async function handleWorkbenchSessionListFromFacts(readModel, url, options, actor, { limit, offset, includeRouteId }) { - const perf = options.backendPerformance; - const pageQuery = { ownerUserId: actor.role === "admin" ? undefined : actor.id, limit: limit + offset + 1, families: WORKBENCH_SESSION_LIST_PAGE_FAMILIES, sessionsOrder: "updated_desc", sessionProjection: "summary" }; - const includeQueryPromise = includeRouteId ? queryWorkbenchSessionInclude(readModel, includeRouteId, perf) : null; - const pageResult = await (perf ? perf.measure("workbench_session_page_query", () => readModel.queryFacts(pageQuery)) : readModel.queryFacts(pageQuery)); - if (pageResult.error) throw pageResult.error; - let pageFacts = pageResult.facts; - let naturalPage = visibleFactSessions(pageFacts, actor).slice(offset, offset + limit + 1); - if (includeRouteId && !naturalPage.some((session) => factSessionMatchesRouteId(session, includeRouteId))) { - const includeResult = await includeQueryPromise; - if (includeResult.error) throw includeResult.error; - const included = visibleFactSessions(includeResult.facts, actor)[0] ?? null; - if (included) { - pageFacts = mergeFactSets(pageFacts, includeResult.facts); - naturalPage = [included, ...naturalPage]; - } - } - const pageSessions = naturalPage.slice(0, limit); - const summaryResult = await (perf ? perf.measure("workbench_session_summary_query", () => queryFactsForSessionSummaries(readModel, pageSessions)) : queryFactsForSessionSummaries(readModel, pageSessions)); - if (summaryResult.error) throw summaryResult.error; - pageFacts = mergeFactSets(pageFacts, summaryResult.facts); - const summaries = pageSessions.map((session) => factSessionSummary(session, pageFacts)).filter(Boolean); - const hasMore = naturalPage.length > limit; - return { - ok: true, - status: "succeeded", - contractVersion: "workbench-sessions-v1", - sessions: summaries, - count: summaries.length, - cursor: offset > 0 ? cursorFromOffset(offset) : null, - hasMore, - nextCursor: hasMore ? cursorFromOffset(offset + limit) : null, - cache: pageResult.cache ?? summaryResult.cache ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function runtimeDependencyError(code, message, retryable = true) { - const error = new Error(message || code); - error.name = "WorkbenchRuntimeDependencyError"; - error.code = code; - error.data = { serviceId: "hwlab-workbench-runtime", retryable, transient: retryable, valuesRedacted: true }; - return error; -} - -function workbenchRuntimeReader(request, options) { - return options.workbenchRuntime - ?? options.runtimeStore - ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger, traceparent: request?.hwlabHttpRequestContext?.traceparent }); -} - -function workbenchRuntimeFactsReadModel(request, options, actor) { - const runtime = workbenchRuntimeReader(request, options); - if (typeof runtime?.queryWorkbenchFacts !== "function") { - throw runtimeDependencyError("workbench_runtime_unconfigured", "Workbench runtime service is required for Workbench facts reads.", false); - } - return { - async queryFacts(query = {}) { - try { - return await runtime.queryWorkbenchFacts({ - ...query, - actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } - }); - } catch (error) { - return { facts: emptyFactSet(), count: 0, durable: true, error: workbenchRuntimeFactsQueryError(error), persistence: null }; - } - } - }; -} - -function workbenchRuntimeFactsQueryError(error) { - const normalized = error instanceof Error ? error : new Error(String(error ?? "Workbench facts query failed.")); - const data = objectValue(normalized.data); - normalized.data = { - ...data, - retryable: data.retryable !== false, - transient: data.transient !== false, - retryAfterMs: data.retryAfterMs ?? 5000, - valuesRedacted: true - }; - return normalized; -} - -function queryWorkbenchSessionInclude(readModel, includeRouteId, perf = null) { - const run = () => queryFactsByRouteId(readModel, includeRouteId, { families: WORKBENCH_SESSION_LIST_PAGE_FAMILIES }); - const promise = perf ? perf.measure("workbench_session_include_query", run) : run(); - return promise.catch((error) => ({ error })); -} - -async function queryFactsByRouteId(readModel, routeId, query = {}) { - const baseQuery = { ...query, limit: query.limit ?? MAX_PAGE_LIMIT }; - const sessionId = safeSessionId(routeId); - if (sessionId) return readModel.queryFacts({ ...baseQuery, sessionId }); - const conversationId = safeConversationId(routeId); - if (conversationId) { - const sessionResult = await readModel.queryFacts({ ...baseQuery, families: WORKBENCH_SESSION_LIST_PAGE_FAMILIES, conversationId }); - if (sessionResult.error || queryRequestsOnlySessionFacts(baseQuery)) return sessionResult; - const resolvedSessionId = factSessionId(factArray(sessionResult.facts?.sessions)[0] ?? null); - if (!resolvedSessionId) return sessionResult; - const relatedResult = await readModel.queryFacts({ ...baseQuery, sessionId: resolvedSessionId }); - if (relatedResult.error) return relatedResult; - const facts = mergeFactSets(sessionResult.facts, relatedResult.facts); - return { facts, count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0), durable: relatedResult.durable ?? sessionResult.durable, persistence: relatedResult.persistence ?? sessionResult.persistence ?? null }; - } - return { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; -} - -function queryRequestsOnlySessionFacts(query = {}) { - const families = Array.isArray(query.families) ? query.families : []; - return families.length === 1 && families[0] === "sessions"; -} - -async function queryFactsForSessionSummaries(readModel, sessions = []) { - const sessionIds = uniqueText(sessions.map(factSessionId)); - if (sessionIds.length === 0 && uniqueText(sessions.map(factLastTraceId)).length === 0) return { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; - // Session list summary is hot and is also sampled by the two-page web-probe - // observer. Avoid parallel fan-out here so one list request cannot consume - // multiple durable-read connections while another page is refreshing. - const bySession = sessionIds.length > 0 - ? await readModel.queryFacts({ sessionIds, families: ["messages", "parts", "turns"] }) - : { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; - if (bySession.error) return bySession; - const traceIds = summaryTraceIds(sessions, bySession.facts); - const byTrace = traceIds.length > 0 - ? await readModel.queryFacts({ traceIds, families: ["checkpoints"] }) - : { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; - if (byTrace.error) return byTrace; - const facts = mergeFactSets(bySession.facts, byTrace.facts); - return { facts, count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0), durable: bySession.durable ?? byTrace.durable, persistence: bySession.persistence ?? byTrace.persistence ?? null }; -} - -function summaryTraceIds(sessions = [], facts = {}) { - return uniqueText([ - ...sessions.map(factLastTraceId), - ...sessions.map((session) => factCurrentTraceForSession(facts, factSessionId(session), session)?.traceId) - ]); -} - -function visibleFactSessions(facts, actor) { - return hideShadowedAgentRunAliasSessions(factArray(facts?.sessions) - .filter((session) => canActorReadFactSession(session, actor)) - .sort(compareFactSessionsDesc)); -} - -function hideShadowedAgentRunAliasSessions(sessions = []) { - const canonicalTraceIds = new Set(sessions - .filter((session) => !isAgentRunAliasSessionId(factSessionId(session))) - .map(factLastTraceId) - .filter(Boolean)); - if (canonicalTraceIds.size === 0) return sessions; - return sessions.filter((session) => !isAgentRunAliasSessionId(factSessionId(session)) || !canonicalTraceIds.has(factLastTraceId(session))); -} - -function isAgentRunAliasSessionId(sessionId) { - return /^ses_agentrun_/u.test(textValue(sessionId)); -} - -function canActorReadFactSession(session, actor) { - if (!session || !actor) return false; - if (normalizeStatus(session.status) === "archived") return false; - if (actor.role === "admin") return true; - return session.ownerUserId === actor.id; -} - -async function visibleFactSessionForTrace(readModel, facts, actor, traceId) { - const visible = visibleFactSessions(facts, actor); - const byLastTrace = visible.find((item) => item.lastTraceId === traceId) ?? null; - if (byLastTrace) return { session: byLastTrace, facts }; - const relatedSessionIds = uniqueText([ - factTurnForTrace(facts, traceId)?.sessionId, - factArray(facts.messages).find((message) => message.traceId === traceId)?.sessionId, - factCheckpointForTrace(facts, traceId)?.sessionId - ]); - const alreadyLoaded = visible.find((item) => relatedSessionIds.includes(factSessionId(item))) ?? null; - if (alreadyLoaded) return { session: alreadyLoaded, facts }; - for (const sessionId of relatedSessionIds) { - const result = await queryFactsByRouteId(readModel, sessionId, { families: ["sessions"], limit: 1 }); - if (result.error) return { error: result.error }; - const session = visibleFactSessions(result.facts, actor)[0] ?? null; - if (session) return { session, facts: mergeFactSets(facts, result.facts) }; - } - return { session: null, facts }; -} - -function factSessionMatchesRouteId(session, routeId) { - const sessionId = safeSessionId(routeId); - if (sessionId && factSessionId(session) === sessionId) return true; - const conversationId = safeConversationId(routeId); - return Boolean(conversationId && session?.conversationId === conversationId); -} - -function factSessionSummary(session, facts = {}) { - const sessionId = factSessionId(session); - if (!sessionId) return null; - const currentTrace = factCurrentTraceForSession(facts, sessionId, session); - const traceId = currentTrace?.traceId ?? factLastTraceId(session) ?? factLatestTraceIdForSession(facts, sessionId); - const projection = traceId ? factProjectionForTrace(facts, traceId) : null; - const turn = traceId ? factTurnForTrace(facts, traceId) : null; - const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null; - const trace = traceId ? factTraceSnapshot(facts, traceId) : null; - const messages = factMessagesForSession(session, facts); - const terminalMessage = traceId ? factTerminalMessageForTrace(messages, traceId) : null; - const traceStatus = normalizeTerminalStatus(trace?.status); - const checkpointStatus = normalizeTerminalStatus(checkpoint?.status); - const messageTerminalStatus = factTerminalStatusFromMessageProjection(terminalMessage); - const turnStatus = normalizeStatus(turn?.status); - const launchContext = compactLaunchContext(session?.sessionJson?.launchContext); - const status = normalizeStatus(checkpointStatus ?? traceStatus ?? messageTerminalStatus ?? nonUnknownStatus(turnStatus) ?? nonUnknownStatus(currentTrace?.status) ?? session?.status); - const timing = traceId - ? factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? terminalMessage : null, turn, trace) - : factTimingProjection(session, status); - const title = sessionTitleFromMessages(messages); - const preview = sessionPreviewFromMessages(messages); - return { - sessionId, - threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), - agentId: session?.agentId ?? "hwlab-code-agent", - title, - preview, - titleSource: title ? "message-projection" : null, - previewSource: preview ? "message-projection" : null, - status, - running: RUNNING_STATUSES.has(status), - terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status), - lastTraceId: traceId, - projection, - projectionStatus: projection?.projectionStatus ?? null, - projectionHealth: projection?.projectionHealth ?? null, - staleMs: projection?.staleMs ?? null, - blocker: projection?.blocker ?? null, - providerProfile: textValue(session?.providerProfile ?? session?.sessionJson?.providerProfile) || null, - launchContext, - messageCount: messages.length, - firstUserMessagePreview: firstUserPreview(messages), - updatedAt: factUpdatedAt(session), - turnSummary: turn ? { - turnId: factTurnId(turn, traceId), - traceId, - status, - running: RUNNING_STATUSES.has(status), - terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status), - eventCount: projection?.lastProjectedSeq ?? trace?.eventCount, - projection, - projectionStatus: projection?.projectionStatus ?? null, - projectionHealth: projection?.projectionHealth ?? null, - staleMs: projection?.staleMs ?? null, - blocker: projection?.blocker ?? null, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - updatedAt: factUpdatedAt(turn) - } : null, - valuesRedacted: true - }; -} - -function factSessionDetail(session, facts = {}, options = {}) { - const sessionId = factSessionId(session); - return buildWorkbenchSessionDetail({ - summary: factSessionSummary(session, facts), - session, - sessionId, - includeMessages: options.includeMessages === true, - messages: factMessagesForSession(session, facts) - }); -} - -function factMessagesForSession(session, facts = {}) { - const sessionId = factSessionId(session); - if (!sessionId) return []; - const partsByMessageId = new Map(); - for (const part of factArray(facts.parts)) { - if (part.sessionId !== sessionId) continue; - const messageId = textValue(part.messageId); - if (!messageId) continue; - const existing = partsByMessageId.get(messageId) ?? []; - existing.push(part); - partsByMessageId.set(messageId, existing); - } - const messageGroups = canonicalFactMessageGroupsForSession( - factArray(facts.messages) - .filter((message) => message.sessionId === sessionId) - .sort((left, right) => compareFactMessagesForSessionAsc(left, right, facts)), - partsByMessageId, - facts - ); - return messageGroups - .sort((left, right) => compareFactMessagesForSessionAsc(left.message, right.message, facts)) - .map((group) => factMessageDto(group.message, group.parts, facts)) - .filter(Boolean); -} - -function canonicalFactMessageGroupsForSession(messages = [], partsByMessageId = new Map(), facts = {}) { - const groups = new Map(); - for (const message of messages) { - const key = factMessageCanonicalGroupKey(message); - const existing = groups.get(key) ?? []; - existing.push(message); - groups.set(key, existing); - } - return [...groups.values()].map((groupMessages) => { - const message = selectCanonicalFactMessage(groupMessages, partsByMessageId, facts); - const aliasIds = uniqueText(groupMessages.map((item) => item?.messageId)); - const parts = aliasIds.flatMap((messageId) => partsByMessageId.get(messageId) ?? []); - return { message, parts }; - }); -} - -function factMessageCanonicalGroupKey(message) { - const messageId = textValue(message?.messageId) || textValue(message?.id); - const role = textValue(message?.role) || "agent"; - const traceId = safeTraceId(message?.traceId); - const sessionId = factSessionId(message) ?? textValue(message?.sessionId) ?? "session"; - if (traceId && isAssistantLikeRole(role)) return `assistant:${sessionId}:${traceId}`; - return `message:${messageId || sessionId}:${role}:${traceId || "none"}`; -} - -function selectCanonicalFactMessage(messages = [], partsByMessageId = new Map(), facts = {}) { - if (messages.length <= 1) return messages[0] ?? null; - const traceId = safeTraceId(messages[0]?.traceId); - const preferredMessageId = traceId ? workbenchLifecycleMessageId(traceId, "agent") : null; - const turnMessageId = traceId ? textValue(factTurnForTrace(facts, traceId)?.messageId) || null : null; - return [...messages].sort((left, right) => { - const score = factMessageCanonicalScore(right, partsByMessageId, preferredMessageId, turnMessageId) - factMessageCanonicalScore(left, partsByMessageId, preferredMessageId, turnMessageId); - return score || compareFactRecordsDesc(left, right); - })[0] ?? messages[0] ?? null; -} - -function factMessageCanonicalScore(message, partsByMessageId = new Map(), preferredMessageId = null, turnMessageId = null) { - const messageId = textValue(message?.messageId) || textValue(message?.id); - let score = 0; - if (preferredMessageId && messageId === preferredMessageId) score += 10000; - if (turnMessageId && messageId === turnMessageId) score += 1000; - if (factMessageHasFinalResponsePart(message, partsByMessageId)) score += 100; - if (message?.sealed === true || message?.terminal === true) score += 50; - if (isTerminalProjectionStatus(normalizeStatus(message?.status))) score += 25; - score += Math.min(10, factSeq(message) ?? 0); - return score; -} - -function factMessageHasFinalResponsePart(message, partsByMessageId = new Map()) { - const messageId = textValue(message?.messageId) || textValue(message?.id); - if (!messageId) return false; - return (partsByMessageId.get(messageId) ?? []).some((part) => (textValue(part?.partType ?? part?.type) === "final_response") && Boolean(projectionText(part?.text, part?.content, part?.message))); -} - -function workbenchLifecycleMessageId(traceId, role = "agent") { - const traceSuffix = (safeTraceId(traceId) || String(traceId || "trace")) - .replace(/^trc_/u, "") - .replace(/[^A-Za-z0-9_.:-]/gu, "_") - .slice(0, 48) || "trace"; - return `msg_${traceSuffix}_${role === "user" ? "user" : "agent"}`; -} - -function factMessageDto(message, parts = [], facts = {}) { - const messageId = safeMessageId(message?.messageId) || textValue(message?.messageId); - if (!messageId) return null; - const traceId = safeTraceId(message?.traceId) ?? null; - const role = textValue(message?.role) || "agent"; - const turn = traceId ? factTurnForTrace(facts, traceId) : null; - const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null; - const messageStatus = normalizeStatus(message?.status); - const assistantLike = isAssistantLikeRole(role); - const legacyText = projectionText(message?.text, message?.content, message?.message, message?.finalResponse); - const baseParts = parts.length > 0 - ? [...parts].sort(compareFactPartsAsc).map((part) => factPartDto(part, messageId, traceId)).filter(Boolean) - : !assistantLike && legacyText ? [partFact({ type: "text", text: legacyText, status: message?.status }, 0, messageId, traceId)] : []; - const checkpointStatus = assistantLike ? normalizeTerminalStatus(checkpoint?.status) : null; - const preliminaryStatus = normalizeStatus(checkpointStatus ?? (assistantLike ? turn?.status : null) ?? messageStatus); - const syntheticTerminalFinalResponse = assistantLike && !factPartsHaveFinalResponse(baseParts) - ? factSyntheticTerminalFinalResponse(preliminaryStatus, traceId, message, checkpoint, turn) - : null; - const normalizedParts = syntheticTerminalFinalResponse - ? [...baseParts, partFact({ type: "final_response", text: syntheticTerminalFinalResponse.text, status: syntheticTerminalFinalResponse.status }, baseParts.length, messageId, traceId)] - : baseParts; - const messageTerminalStatus = assistantLike ? factTerminalStatusFromMessageProjection(message, normalizedParts) : null; - const status = normalizeStatus(checkpointStatus ?? messageTerminalStatus ?? (assistantLike ? turn?.status : null) ?? messageStatus); - const timing = assistantLike - ? factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? { ...message, parts: normalizedParts } : null, turn, message) - : factTimingProjection(message, status); - const text = factMessageAuthorityText({ role, status, parts: normalizedParts, legacyText }); - return { - messageId, - role, - sessionId: (safeSessionId(message?.sessionId) ?? textValue(message?.sessionId)) || null, - traceId, - turnId: safeTurnId(message?.turnId) || traceId, - status, - parts: normalizedParts, - text: text || "", - textPreview: text ? text.slice(0, 240) : null, - createdAt: textValue(message?.createdAt) || null, - updatedAt: factUpdatedAt(message), - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - projectionStatus: null, - projectionHealth: null, - valuesRedacted: message?.valuesRedacted !== false - }; -} - -function factMessageAuthorityText({ role, status, parts = [], legacyText = null } = {}) { - if (!isAssistantLikeRole(role)) return firstFactPartText(parts) ?? legacyText ?? ""; - if (!isTerminalProjectionStatus(status)) return ""; - return firstFactPartText(parts, "final_response") ?? ""; -} - -function factPartsHaveFinalResponse(parts = []) { - return parts.some((part) => part?.type === "final_response" && Boolean(projectionText(part?.text))); -} - -function factSyntheticTerminalFinalResponse(status, traceId = null, ...records) { - const terminalStatus = normalizeTerminalStatus(status); - if (!terminalStatus || terminalStatus === "completed") return null; - return terminalFinalResponse(terminalStatus, { traceId, status: terminalStatus, records }, { evidence: records }); -} - -function isTerminalProjectionStatus(status) { - return TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status); -} - -function firstFactPartText(parts = [], type = null) { - for (const part of parts) { - if (type && part?.type !== type) continue; - const text = projectionText(part?.text); - if (text) return text; - } - return null; -} - -function factMessageFinalResponseText(message) { - if (!message || !isAssistantLikeRole(message.role) || !isTerminalProjectionStatus(message.status)) return null; - return firstFactPartText(message.parts, "final_response"); -} - -function factTerminalMessageForTrace(messages = [], traceId = null) { - const safeTrace = safeTraceId(traceId); - return [...factArray(messages)].reverse().find((message) => { - if (!message || !isAssistantLikeRole(message.role)) return false; - if (safeTrace && message.traceId !== safeTrace) return false; - return Boolean(factTerminalStatusFromMessageProjection(message)); - }) ?? null; -} - -function factTerminalStatusFromMessageProjection(message = null, parts = null) { - if (!message || !isAssistantLikeRole(message.role)) return null; - const sourceParts = Array.isArray(parts) ? parts : factArray(message.parts); - if (!firstFactPartText(sourceParts, "final_response")) return null; - const messageStatus = normalizeTerminalStatus(message.status); - if (messageStatus) return messageStatus; - for (const part of sourceParts) { - if (textValue(part?.partType ?? part?.type) !== "final_response") continue; - if (!projectionText(part?.text, part?.content, part?.message)) continue; - const partStatus = normalizeTerminalStatus(part?.status); - if (partStatus) return partStatus; - if (part?.terminal === true || part?.sealed === true) return "completed"; - } - return message.terminal === true || message.sealed === true ? "completed" : null; -} - -function factPartDto(part, messageId, traceId) { - const partId = safePartId(part?.partId) || textValue(part?.partId) || `prt_${hash(`${messageId}:${part?.partIndex ?? 0}:${part?.partType ?? part?.type ?? "text"}`).slice(0, 24)}`; - const text = projectionText(part?.text, part?.content, part?.message); - return { - partId, - messageId, - traceId: safeTraceId(part?.traceId) ?? traceId, - type: textValue(part?.partType ?? part?.type) || "text", - text: text || null, - status: normalizeStatus(part?.status), - toolName: textValue(part?.toolName ?? part?.name) || null, - createdAt: textValue(part?.createdAt ?? part?.occurredAt) || null, - valuesRedacted: part?.valuesRedacted !== false - }; -} - -function factTurnForTrace(facts = {}, traceId, turnId = null) { - const safeTrace = safeTraceId(traceId); - if (!safeTrace) return null; - const requestedTurn = safeTurnId(turnId); - const matches = factArray(facts.turns) - .filter((turn) => turn.traceId === safeTrace && (!requestedTurn || turn.turnId === requestedTurn || turn.turnId === safeTrace)) - .sort(compareFactRecordsDesc); - return matches[0] ?? null; -} - -function factTurnSnapshot({ turn = null, session = null, facts = {}, traceId, turnId = null } = {}) { - const safeTrace = safeTraceId(traceId); - const resolvedTurnId = factTurnId(turn, safeTrace) ?? safeTurnId(turnId) ?? safeTrace; - const messages = session ? factMessagesForSession(session, facts) : []; - const userMessage = messages.find((message) => message.role === "user") ?? null; - const assistantMessage = [...messages].reverse().find((message) => isAssistantLikeRole(message.role)) ?? null; - const terminalMessage = safeTrace ? factTerminalMessageForTrace(messages, safeTrace) : null; - const checkpoint = safeTrace ? factCheckpointForTrace(facts, safeTrace) : null; - const trace = factTraceSnapshot(facts, safeTrace); - const checkpointStatus = normalizeTerminalStatus(checkpoint?.status); - const traceStatus = normalizeTerminalStatus(trace.status); - const messageTerminalStatus = factTerminalStatusFromMessageProjection(terminalMessage); - const status = normalizeStatus(checkpointStatus ?? traceStatus ?? messageTerminalStatus ?? turn?.status ?? session?.status); - const terminal = isTerminalProjectionStatus(status); - const assistantText = terminal ? factMessageFinalResponseText(assistantMessage) : null; - const timing = factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? terminalMessage : null, turn, trace); - return { - turnId: resolvedTurnId, - traceId: safeTrace, - status, - running: RUNNING_STATUSES.has(status), - terminal, - sessionId: factSessionId(session) ?? turn?.sessionId ?? checkpoint?.sessionId ?? null, - threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), - userMessageId: userMessage?.messageId ?? null, - assistantMessageId: assistantMessage?.messageId ?? turn?.messageId ?? null, - assistantText: assistantText ?? null, - finalResponse: assistantText ? { text: assistantText, sealed: true, source: "message-part" } : null, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - agentRun: checkpoint ? { - runId: textValue(checkpoint.runId) || null, - commandId: textValue(checkpoint.commandId) || null, - status, - lastSeq: factSeq(checkpoint), - valuesRedacted: true - } : null, - trace: { - traceId: safeTrace, - status: trace.status, - eventCount: trace.eventCount, - timing: trace.timing, - startedAt: trace.startedAt, - lastEventAt: trace.lastEventAt, - finishedAt: trace.finishedAt, - durationMs: trace.durationMs, - updatedAt: trace.updatedAt - }, - urls: { - self: resolvedTurnId ? `/v1/workbench/turns/${encodeURIComponent(resolvedTurnId)}` : null, - traceEvents: safeTrace ? `/v1/workbench/traces/${encodeURIComponent(safeTrace)}/events` : null - } - }; -} - -function factTraceSnapshot(facts = {}, traceId) { - const safeTrace = safeTraceId(traceId); - const events = factArray(facts.traceEvents) - .filter((event) => !safeTrace || event.traceId === safeTrace) - .sort(compareFactTraceEventsAsc) - .map(factTraceEventDto) - .filter(Boolean); - const status = durableTraceStatus(events); - const lastEvent = events.at(-1) ?? null; - const checkpoint = factCheckpointForTrace(facts, safeTrace); - const timing = factTraceTimingProjection(events, checkpoint, status); - return { - traceId: safeTrace, - status, - eventCount: events.length, - events, - lastEvent, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - updatedAt: lastEvent?.updatedAt ?? lastEvent?.createdAt ?? checkpoint?.updatedAt ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function factTraceTimingProjection(events = [], checkpoint = null, status = null) { - const checkpointTiming = factTimingProjection(checkpoint, status); - const normalizedStatus = normalizeStatus(status ?? checkpoint?.status); - const terminal = TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus); - const observedAt = new Date().toISOString(); - const eventStartTimes = factArray(events).flatMap((event) => [event?.createdAt, event?.occurredAt]); - const eventActivityTimes = factArray(events).flatMap((event) => [event?.createdAt, event?.occurredAt, event?.updatedAt]); - const terminalEventTimes = factArray(events) - .filter(factTraceEventIsTerminalAuthority) - .flatMap(factTraceEventTerminalTimes); - const startedAt = firstTimestampIso(checkpointTiming.startedAt, ...eventStartTimes); - const finishedAt = terminal ? latestTimestampIso(checkpointTiming.finishedAt, ...terminalEventTimes) : null; - const lastEventAt = terminal && finishedAt ? finishedAt : latestTimestampIso(checkpointTiming.lastEventAt, ...eventActivityTimes); - const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt); - const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt); - return { - ...checkpointTiming, - startedAt, - lastEventAt, - finishedAt, - durationMs, - observedAt: terminal ? null : observedAt, - lastEventAgeMs, - valuesRedacted: true - }; -} - -function factTraceEventIsTerminalAuthority(event = null) { - const eventType = textValue(event?.eventType ?? event?.type); - return event?.terminal === true || event?.sealed === true || eventType === "terminal"; -} - -function factTraceEventTerminalTimes(event = null) { - const source = objectValue(event?.timing); - return [source?.finishedAt, event?.finishedAt, source?.lastEventAt, event?.lastEventAt, event?.occurredAt, event?.createdAt]; -} - -function factTraceEventDto(event, index) { - const seq = factProjectedSeq(event); - if (!seq) return null; - const sourceSeq = Number.isFinite(Number(event?.sourceSeq)) && Number(event.sourceSeq) >= 0 ? Math.trunc(Number(event.sourceSeq)) : null; - return { - ...event, - id: textValue(event?.id) || textValue(event?.sourceEventId) || null, - seq, - projectedSeq: seq, - sourceSeq, - type: textValue(event?.type ?? event?.eventType) || "event", - label: textValue(event?.label ?? event?.eventType ?? event?.type) || null, - status: normalizeStatus(event?.status), - createdAt: textValue(event?.createdAt ?? event?.occurredAt) || null, - updatedAt: factUpdatedAt(event), - valuesRedacted: event?.valuesRedacted !== false - }; -} - -function factProjectionForTrace(facts = {}, traceId) { - const checkpoint = factCheckpointForTrace(facts, traceId); - if (!checkpoint) { - return { - projectionStatus: "unknown", - projectionHealth: "unknown", - lastProjectedSeq: null, - sourceRunId: null, - sourceCommandId: null, - staleMs: null, - blocker: null, - updatedAt: null, - valuesRedacted: true - }; - } - const projectionStatus = normalizeProjectionStatus(checkpoint.projectionStatus); - const projectionHealth = normalizeProjectionHealth(checkpoint.projectionHealth, projectionStatus); - const diagnostic = objectValue(checkpoint.diagnostic); - const timing = factTimingProjection(checkpoint, normalizeStatus(checkpoint.status)); - return { - projectionStatus, - projectionHealth, - lastProjectedSeq: factSeq(checkpoint), - sourceRunId: textValue(checkpoint.runId ?? checkpoint.sourceRunId) || null, - sourceCommandId: textValue(checkpoint.commandId ?? checkpoint.sourceCommandId) || null, - staleMs: null, - blocker: diagnostic.blocker ?? checkpoint.blocker ?? null, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - updatedAt: factUpdatedAt(checkpoint), - valuesRedacted: true - }; -} - -function factTimingProjection(record = null, status = null) { - const source = objectValue(record?.timing); - const observedAt = new Date().toISOString(); - const startedAt = timestampIso(source?.startedAt ?? record?.startedAt ?? record?.createdAt); - const lastEventAt = timestampIso(source?.lastEventAt ?? record?.lastEventAt ?? record?.updatedAt ?? record?.createdAt); - const normalizedStatus = normalizeStatus(status ?? record?.status); - const terminal = record?.terminal === true || TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus); - const finishedAt = terminal ? timestampIso(source?.finishedAt ?? record?.finishedAt ?? record?.completedAt) : null; - const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt); - const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt); - return { startedAt, lastEventAt, finishedAt, durationMs, observedAt: terminal ? null : observedAt, lastEventAgeMs, valuesRedacted: source?.valuesRedacted !== false }; -} - -function factCombinedTimingProjection(status = null, ...records) { - const normalizedStatus = normalizeStatus(status ?? records.find((record) => record?.status)?.status); - const terminal = TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus); - const observedAt = new Date().toISOString(); - const timings = records.map((record) => factTimingSource(record)).filter(Boolean); - const startedAt = firstTimestampIso(...timings.map((timing) => timing.startedAt)); - const finishedAt = terminal ? latestTimestampIso(...timings.map((timing) => timing.finishedAt)) : null; - const lastEventAt = terminal && finishedAt ? finishedAt : latestTimestampIso(...timings.map((timing) => timing.lastEventAt)); - const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt); - const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt); - return { startedAt, lastEventAt, finishedAt, durationMs, observedAt: terminal ? null : observedAt, lastEventAgeMs, valuesRedacted: true }; -} - -function factTimingSource(record = null) { - if (!record) return null; - const source = objectValue(record?.timing); - return { - startedAt: timestampIso(source?.startedAt ?? record?.startedAt ?? record?.createdAt), - lastEventAt: timestampIso(source?.lastEventAt ?? record?.lastEventAt ?? record?.updatedAt ?? record?.createdAt), - finishedAt: timestampIso(source?.finishedAt ?? record?.finishedAt ?? record?.completedAt) - }; -} - -function firstTimestampIso(...values) { - for (const value of values) { - const timestamp = timestampIso(value); - if (timestamp) return timestamp; - } - return null; -} - -function latestTimestampIso(...values) { - let latest = null; - let latestMs = Number.NEGATIVE_INFINITY; - for (const value of values) { - const timestamp = timestampIso(value); - if (!timestamp) continue; - const ms = Date.parse(timestamp); - if (!Number.isFinite(ms) || ms < latestMs) continue; - latest = timestamp; - latestMs = ms; - } - return latest; -} - -function elapsedFactMs(startedAt, endedAt) { - const start = Date.parse(String(startedAt ?? "")); - const end = Date.parse(String(endedAt ?? "")); - if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null; - return Math.trunc(end - start); -} - -function timestampIso(value) { - const ms = Date.parse(String(value ?? "")); - return Number.isFinite(ms) ? new Date(ms).toISOString() : null; -} - -function factCheckpointForTrace(facts = {}, traceId) { - const safeTrace = safeTraceId(traceId); - if (!safeTrace) return null; - return factArray(facts.checkpoints) - .filter((checkpoint) => checkpoint.traceId === safeTrace) - .sort(compareFactRecordsDesc)[0] ?? null; -} - -function workbenchProjectionStoreError(error) { - const data = objectValue(error?.data) ?? {}; - const retryAfterNumber = Number(data.retryAfterMs); - const retryAfterMs = Number.isFinite(retryAfterNumber) && retryAfterNumber >= 0 ? Math.trunc(retryAfterNumber) : null; - const retryAttempt = nonNegativeIntegerOrNull(data.retryAttempt ?? data.retryAttempts); - const retryMax = nonNegativeIntegerOrNull(data.retryMax); - const retryDelaysMs = Array.isArray(data.retryDelaysMs) - ? data.retryDelaysMs.map(nonNegativeIntegerOrNull).filter((value) => value !== null) - : []; - return workbenchError("projection_store_unavailable", "Workbench durable projection store is unavailable.", { - causeCode: error?.code ?? null, - queryResult: textValue(data.queryResult) || null, - blockedLayer: textValue(data.blockedLayer) || null, - retryable: data.retryable !== false, - transient: data.transient === true || data.retryable === true, - retryAfterMs: retryAfterMs ?? null, - retryAttempt, - retryMax, - retryAttempts: retryAttempt, - retryExhausted: data.retryExhausted === true, - retryDelaysMs: retryDelaysMs.length > 0 ? retryDelaysMs : null - }); -} - -function nonNegativeIntegerOrNull(value) { - const parsed = Number(value); - return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null; -} - -function factSessionId(session) { - return safeSessionId(session?.sessionId ?? session?.id) ?? null; -} - -function factLastTraceId(session) { - return safeTraceId(session?.lastTraceId ?? session?.traceId) ?? null; -} - -function factLatestTraceIdForSession(facts = {}, sessionId) { - const turn = factArray(facts.turns).filter((item) => item.sessionId === sessionId).sort(compareFactRecordsDesc)[0] ?? null; - return safeTraceId(turn?.traceId) ?? null; -} - -function factCurrentTraceForSession(facts = {}, sessionId, session = null) { - const candidates = []; - const sessionTraceId = factLastTraceId(session); - if (sessionTraceId) candidates.push(factTraceCandidate(session, sessionTraceId, normalizeStatus(session?.status), 0)); - for (const message of factArray(facts.messages)) { - if (message?.sessionId !== sessionId || !safeTraceId(message?.traceId)) continue; - candidates.push(factMessageTraceCandidate(message)); - } - for (const turn of factArray(facts.turns)) { - if (turn?.sessionId !== sessionId || !safeTraceId(turn?.traceId)) continue; - candidates.push(factTraceCandidate(turn, safeTraceId(turn.traceId), normalizeStatus(turn?.status), 3)); - } - return candidates.sort(compareTraceCandidatesDesc)[0] ?? null; -} - -function factMessageTraceCandidate(message) { - const role = textValue(message?.role); - let status = normalizeStatus(message?.status); - let priority = 1; - if (isAssistantLikeRole(role)) { - priority = 2; - if (status === "unknown") status = "running"; - } else if (safeTraceId(message?.traceId)) { - status = "running"; - } - return factTraceCandidate(message, safeTraceId(message?.traceId), status, priority); -} - -function factTraceCandidate(record, traceId, status, priority) { - return { traceId: safeTraceId(traceId) ?? null, status: normalizeStatus(status), seq: factSeq(record), updatedAt: factUpdatedAt(record), priority }; -} - -function compareTraceCandidatesDesc(left, right) { - return compareNumberDesc(left?.seq, right?.seq) || compareTimestampDesc(left?.updatedAt, right?.updatedAt) || compareNumberDesc(left?.priority, right?.priority); -} - -function nonUnknownStatus(status) { - return normalizeStatus(status) === "unknown" ? null : status; -} - -function factTurnId(turn, fallbackTraceId = null) { - return safeTurnId(turn?.turnId) ?? safeTraceId(turn?.turnId) ?? safeTraceId(fallbackTraceId) ?? null; -} - -function factUpdatedAt(record) { - return textValue(record?.updatedAt ?? record?.occurredAt ?? record?.createdAt) || null; -} - -function factSeq(record) { - for (const value of [record?.projectedSeq, record?.sourceSeq, record?.seq]) { - const parsed = Number(value); - if (Number.isFinite(parsed) && parsed >= 0) return Math.trunc(parsed); - } - return null; -} - -function factProjectedSeq(record) { - const parsed = Number(record?.projectedSeq); - return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : null; -} - -function normalizeProjectionStatus(value) { - const status = normalizeStatus(value); - if (status === "caught-up") return "caught-up"; - if (status === "caughtup") return "caught-up"; - return ["projecting", "blocked", "stalled", "unknown"].includes(status) ? status : "unknown"; -} - -function normalizeProjectionHealth(value, projectionStatus = "unknown") { - const health = normalizeStatus(value); - if (health === "healthy" && projectionStatus === "caught-up") return "caught-up"; - if (health === "healthy" && projectionStatus === "projecting") return "projecting"; - if (["caught-up", "projecting", "degraded", "stalled", "unavailable", "unknown"].includes(health)) return health; - return projectionStatus === "caught-up" || projectionStatus === "projecting" ? projectionStatus : "unknown"; -} - -function compareFactSessionsDesc(left, right) { - return compareTimestampDesc(factUpdatedAt(left), factUpdatedAt(right)) || compareText(factSessionId(left), factSessionId(right)); -} - -function compareFactMessagesAsc(left, right) { - return compareNumberAsc(factSeq(left), factSeq(right)) || compareTimestampAsc(left?.createdAt ?? left?.updatedAt, right?.createdAt ?? right?.updatedAt) || compareText(left?.messageId, right?.messageId); -} - -function compareFactMessagesForSessionAsc(left, right, facts = {}) { - const leftKey = factMessageTimelineKey(left, facts); - const rightKey = factMessageTimelineKey(right, facts); - return compareOptionalTimestampAsc(leftKey.timelineAnchorAt, rightKey.timelineAnchorAt) - || compareOptionalNumberAsc(leftKey.turnSeq, rightKey.turnSeq) - || compareTimestampAsc(leftKey.turnStartedAt, rightKey.turnStartedAt) - || compareText(leftKey.traceId, rightKey.traceId) - || compareOptionalNumberAsc(leftKey.roleRank, rightKey.roleRank) - || compareTimestampAsc(leftKey.messageCreatedAt, rightKey.messageCreatedAt) - || compareOptionalNumberAsc(leftKey.messageSeq, rightKey.messageSeq) - || compareText(left?.messageId, right?.messageId); -} - -function factMessageTimelineKey(message, facts = {}) { - const traceId = safeTraceId(message?.traceId) ?? null; - const turn = traceId ? factTurnForTrace(facts, traceId) : null; - const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null; - const anchorMessage = traceId ? factTimelineAnchorMessageForTrace(facts, traceId) : null; - return { - traceId: traceId ?? "", - timelineAnchorAt: firstTimestampIso(anchorMessage?.createdAt, anchorMessage?.updatedAt), - turnSeq: firstFiniteNumber(factSeq(turn), factSeq(checkpoint)), - turnStartedAt: firstTimestampIso(turn?.startedAt, checkpoint?.startedAt, message?.createdAt, message?.updatedAt), - roleRank: factMessageTimelineRoleRank(message?.role), - messageCreatedAt: timestampIso(message?.createdAt ?? message?.updatedAt), - messageSeq: factSeq(message) - }; -} - -function factTimelineAnchorMessageForTrace(facts = {}, traceId) { - const safeTrace = safeTraceId(traceId); - if (!safeTrace) return null; - const messages = factArray(facts.messages).filter((message) => message.traceId === safeTrace); - const userMessages = messages.filter((message) => normalizeRole(message?.role) === "user"); - const candidates = userMessages.length > 0 ? userMessages : messages; - return [...candidates].sort((left, right) => compareOptionalTimestampAsc(left?.createdAt ?? left?.updatedAt, right?.createdAt ?? right?.updatedAt) || compareOptionalNumberAsc(factSeq(left), factSeq(right)) || compareText(left?.messageId, right?.messageId))[0] ?? null; -} - -function factMessageTimelineRoleRank(role) { - const normalized = normalizeRole(role); - if (normalized === "user") return 0; - if (isAssistantLikeRole(normalized)) return 1; - return 2; -} - -function firstFiniteNumber(...values) { - for (const value of values) { - const parsed = Number(value); - if (Number.isFinite(parsed)) return Math.trunc(parsed); - } - return null; -} - -function compareOptionalNumberAsc(left, right) { - const leftNumber = Number(left); - const rightNumber = Number(right); - const leftComparable = Number.isFinite(leftNumber) ? leftNumber : Number.MAX_SAFE_INTEGER; - const rightComparable = Number.isFinite(rightNumber) ? rightNumber : Number.MAX_SAFE_INTEGER; - return leftComparable - rightComparable; -} - -function compareFactPartsAsc(left, right) { - return compareNumberAsc(left?.partIndex, right?.partIndex) || compareNumberAsc(factSeq(left), factSeq(right)) || compareText(left?.partId, right?.partId); -} - -function compareFactTraceEventsAsc(left, right) { - return compareNumberAsc(factProjectedSeq(left), factProjectedSeq(right)); -} - -function compareFactRecordsDesc(left, right) { - return compareNumberDesc(factSeq(left), factSeq(right)) || compareTimestampDesc(factUpdatedAt(left), factUpdatedAt(right)); -} - -function compareTimestampAsc(left, right) { - return timestampMs(left) - timestampMs(right); -} - -function compareTimestampDesc(left, right) { - return timestampMs(right) - timestampMs(left); -} - -function compareOptionalTimestampAsc(left, right) { - const leftMs = optionalTimestampMs(left); - const rightMs = optionalTimestampMs(right); - return (leftMs ?? Number.MAX_SAFE_INTEGER) - (rightMs ?? Number.MAX_SAFE_INTEGER); -} - -function compareNumberAsc(left, right) { - return numericValue(left, Number.MAX_SAFE_INTEGER) - numericValue(right, Number.MAX_SAFE_INTEGER); -} - -function compareNumberDesc(left, right) { - return numericValue(right, -1) - numericValue(left, -1); -} - -function compareText(left, right) { - return textValue(left).localeCompare(textValue(right)); -} - -function timestampMs(value) { - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? parsed : 0; -} - -function optionalTimestampMs(value) { - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? parsed : null; -} - -function numericValue(value, fallback) { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : fallback; -} - -function mergeFactSets(...sets) { - const merged = emptyFactSet(); - const keys = { - sessions: "sessionId", - messages: "messageId", - parts: "partId", - turns: "turnId", - traceEvents: "id", - checkpoints: "traceId" - }; - for (const set of sets) { - for (const [key, idKey] of Object.entries(keys)) { - const seen = new Set(merged[key].map((item) => textValue(item?.[idKey]))); - for (const item of factArray(set?.[key])) { - const id = textValue(item?.[idKey]); - if (id && seen.has(id)) continue; - merged[key].push(item); - if (id) seen.add(id); - } - } - } - return merged; -} - -function emptyFactSet() { - return { - sessions: [], - messages: [], - parts: [], - turns: [], - traceEvents: [], - checkpoints: [] - }; -} - -function factArray(value) { - return Array.isArray(value) ? value : []; -} - -function uniqueText(values = []) { - return [...new Set(values.map((value) => textValue(value)).filter(Boolean))]; -} - -async function handleWorkbenchSessionDetail(request, response, url, options, actor, sessionId) { - const readModel = workbenchRuntimeFactsReadModel(request, options, actor); - const includeMessages = includeMessagesForSessionDetail(url.searchParams); - const result = await queryFactsByRouteId(readModel, sessionId, { families: WORKBENCH_SESSION_DETAIL_FAMILIES }); - if (result.error) return sendJson(response, 503, workbenchProjectionStoreError(result.error)); - const session = visibleFactSessions(result.facts, actor)[0] ?? null; - if (!session) { - const body = workbenchError("workbench_session_not_found", "Workbench session is not visible to the current actor.", { sessionId }); - attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/sessions/:id", code: "workbench_session_not_found", sessionId, count: result.count }); - return sendJson(response, 404, body); - } - sendJson(response, 200, { - ok: true, - status: "found", - contractVersion: "workbench-session-detail-v1", - session: factSessionDetail(session, result.facts, { includeMessages }), - detailMode: includeMessages ? "full" : "metadata-only", - messagesIncluded: includeMessages, - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function handleWorkbenchMessagePage(request, response, url, options, actor, sessionId) { - const startedAt = nowMs(); - const readModel = workbenchRuntimeFactsReadModel(request, options, actor); - const limit = boundedLimit(url.searchParams.get("limit")); - const cursorParam = url.searchParams.get("cursor") ?? url.searchParams.get("after"); - const hasCursor = Boolean(textValue(cursorParam)); - const result = await queryFactsByRouteId(readModel, sessionId, { - families: WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES, - ...(hasCursor ? {} : { limit: limit + 1, messagesOrder: "updated_desc", partsOrder: "updated_desc", turnsOrder: "updated_desc", checkpointsOrder: "updated_desc" }) - }); - if (result.error) return sendJson(response, 503, workbenchProjectionStoreError(result.error)); - const session = visibleFactSessions(result.facts, actor)[0] ?? null; - if (!session) { - const body = workbenchError("workbench_session_not_found", "Workbench session is not visible to the current actor.", { sessionId }); - attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/sessions/:id/messages", code: "workbench_session_not_found", sessionId, count: result.count }); - return sendJson(response, 404, body); - } - const messages = factMessagesForSession(session, result.facts); - const offset = hasCursor ? cursorOffset(cursorParam) : 0; - const page = messages.slice(offset, offset + limit); - const nextOffset = offset + page.length; - const resolvedSessionId = factSessionId(session); - const body = { - ok: true, - status: "succeeded", - contractVersion: "workbench-message-page-v1", - sessionId: resolvedSessionId, - messages: page, - count: page.length, - total: messages.length, - cursor: offset > 0 ? cursorFromOffset(offset) : null, - nextCursor: nextOffset < messages.length ? cursorFromOffset(nextOffset) : null, - hasMore: nextOffset < messages.length, - timelineDigest: workbenchTimelineDigest(page), - roleSequencePrefix: messagesRoleSequence(page).slice(0, 32), - adjacentSameRoleCount: adjacentSameRoleCount(page), - traceIds: uniqueText(page.map((message) => message?.traceId)).slice(0, 16), - valuesRedacted: true, - secretMaterialStored: false - }; - emitWorkbenchSessionMessagesReadOtel(request, options, { - startTimeMs: startedAt, - statusCode: 200, - sessionId: resolvedSessionId, - limit, - offset, - messages: page, - total: messages.length, - hasMore: body.hasMore - }); - sendJson(response, 200, body); -} - -async function handleWorkbenchTurnSnapshot(request, response, url, options, actor, rawTurnId) { - const startedAt = nowMs(); - const queryTraceId = safeTraceId(url.searchParams.get("traceId")); - const traceId = safeTraceId(rawTurnId) ?? queryTraceId; - const turnId = safeTurnId(rawTurnId) || traceId || rawTurnId; - if (!traceId) { - const body = workbenchError("invalid_turn_id", "turnId must currently be a trace id or include traceId query.", { turnId }); - recordWorkbenchTurnReadMetric(options, url, 400, body, startedAt); - return sendJson(response, 400, body); - } - const readModel = workbenchRuntimeFactsReadModel(request, options, actor); - const result = await readModel.queryFacts({ traceId, limit: MAX_PAGE_LIMIT, families: [...WORKBENCH_TRACE_EVENT_METADATA_FAMILIES, ...WORKBENCH_TRACE_EVENT_PAGE_FAMILIES] }); - if (result.error) { - const body = workbenchProjectionStoreError(result.error); - emitWorkbenchTurnStatusReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 503, - phase: "query_failed", - turnId, - errorCode: result.error?.code ?? "workbench_turn_query_failed" - }, result.error); - recordWorkbenchTurnReadMetric(options, url, 503, body, startedAt); - return sendJson(response, 503, body); - } - const session = visibleFactSessions(result.facts, actor).find((item) => item.lastTraceId === traceId) ?? visibleFactSessions(result.facts, actor)[0] ?? null; - const turn = factTurnForTrace(result.facts, traceId, turnId); - const found = Boolean(session || turn); - if (!found) { - const body = workbenchError("workbench_turn_not_found", "Workbench turn is not visible to the current actor.", { turnId, traceId }); - attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/turns/:id", code: "workbench_turn_not_found", traceId, turnId, count: result.count }); - emitWorkbenchTurnStatusReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 404, - phase: "not_found", - turnId, - readCount: result.count, - errorCode: "workbench_turn_not_found" - }); - recordWorkbenchTurnReadMetric(options, url, 404, body, startedAt); - return sendJson(response, 404, body); - } - const projection = factProjectionForTrace(result.facts, traceId); - const snapshot = factTurnSnapshot({ turn, session, facts: result.facts, traceId, turnId }); - const status = snapshot.status; - const body = { - ok: true, - status, - contractVersion: "workbench-turn-snapshot-v1", - turn: snapshot, - projection, - cache: result.cache ?? null, - projectionStatus: projection.projectionStatus, - projectionHealth: projection.projectionHealth, - lastProjectedSeq: projection.lastProjectedSeq, - sourceRunId: projection.sourceRunId, - sourceCommandId: projection.sourceCommandId, - staleMs: projection.staleMs, - blocker: projection.blocker, - valuesRedacted: true, - secretMaterialStored: false - }; - recordWorkbenchProjectionMetric(options, { - projection: { ...projection, status, eventCount: snapshot.trace?.eventCount ?? projection.lastProjectedSeq ?? 0 }, - diagnostic: projection - }); - emitWorkbenchTurnStatusReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 200, - phase: "succeeded", - status, - terminal: snapshot.terminal, - sessionId: snapshot.sessionId, - turnId: snapshot.turnId, - traceEventCount: snapshot.trace?.eventCount ?? null, - latestProjectedSeq: projection.lastProjectedSeq, - projectionStatus: projection.projectionStatus, - projectionHealth: projection.projectionHealth, - sourceRunId: projection.sourceRunId, - sourceCommandId: projection.sourceCommandId, - readCount: result.count - }); - recordWorkbenchTurnReadMetric(options, url, 200, body, startedAt); - recordWorkbenchCacheMetric(options, url, body, startedAt); - sendJson(response, 200, body); -} - -async function handleWorkbenchTraceEventPage(request, response, url, options, actor, rawTraceId) { - const startedAt = nowMs(); - const traceId = safeTraceId(rawTraceId); - if (!traceId) return sendJson(response, 400, workbenchError("invalid_trace_id", "traceId must start with trc_.", { traceId: rawTraceId })); - const readModel = workbenchRuntimeFactsReadModel(request, options, actor); - const pageOptions = tracePageOptions(url); - const metadata = await readModel.queryFacts({ traceId, families: WORKBENCH_TRACE_EVENT_METADATA_FAMILIES, limit: 1 }); - if (metadata.error) { - emitWorkbenchTraceEventsReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 503, - phase: "metadata_query_failed", - pageOptions, - errorCode: metadata.error?.code ?? "workbench_trace_metadata_query_failed" - }, metadata.error); - return sendJson(response, 503, workbenchProjectionStoreError(metadata.error)); - } - const resolved = await visibleFactSessionForTrace(readModel, metadata.facts, actor, traceId); - if (resolved.error) { - emitWorkbenchTraceEventsReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 503, - phase: "session_lookup_failed", - pageOptions, - metadataCount: metadata.count, - errorCode: resolved.error?.code ?? "workbench_trace_session_lookup_failed" - }, resolved.error); - return sendJson(response, 503, workbenchProjectionStoreError(resolved.error)); - } - const session = resolved.session; - const metadataFacts = resolved.facts ?? metadata.facts; - if (!session) { - const context = await readModel.queryFacts({ traceId, families: ["sessions", "turns", "checkpoints"], limit: MAX_PAGE_LIMIT }); - if (context.error) { - emitWorkbenchTraceEventsReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 503, - phase: "context_query_failed", - pageOptions, - metadataCount: metadata.count, - errorCode: context.error?.code ?? "workbench_trace_context_query_failed" - }, context.error); - return sendJson(response, 503, workbenchProjectionStoreError(context.error)); - } - const contextSessions = visibleFactSessions(context.facts, actor); - const contextSession = contextSessions.find((item) => item.lastTraceId === traceId) ?? contextSessions[0] ?? null; - const contextTurn = factTurnForTrace(context.facts, traceId, traceId); - if (contextSession || contextTurn) { - const projection = factProjectionForTrace(context.facts, traceId); - const blocker = traceEventsReadModelBlocker("workbench_trace_metadata_missing", "Workbench trace metadata is missing from the trace events read model.", { traceId, projection, session: contextSession, turn: contextTurn, route: url.pathname }); - attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/traces/:id/events", code: "workbench_trace_metadata_missing", sessionId: factSessionId(contextSession), traceId, count: context.count }); - emitWorkbenchTraceEventsReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 404, - phase: "metadata_missing", - pageOptions, - projection, - sessionId: factSessionId(contextSession), - turnId: factTurnId(contextTurn, traceId), - readCount: context.count, - metadataCount: metadata.count, - errorCode: "workbench_trace_metadata_missing" - }); - return sendJson(response, 404, workbenchTraceEventsReadModelError(blocker, { traceId, projection, session: contextSession, turn: contextTurn })); - } - attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/traces/:id/events", code: "workbench_trace_not_found", traceId, count: context.count }); - emitWorkbenchTraceEventsReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 404, - phase: "not_found", - pageOptions, - readCount: context.count, - metadataCount: metadata.count, - errorCode: "workbench_trace_not_found" - }); - return sendJson(response, 404, workbenchError("workbench_trace_not_found", "Workbench trace is not visible to the current actor.", { traceId })); - } - const projection = factProjectionForTrace(metadataFacts, traceId); - const pageResult = await readModel.queryFacts({ traceId, families: WORKBENCH_TRACE_EVENT_PAGE_FAMILIES, afterProjectedSeq: pageOptions.afterProjectedSeq, limit: pageOptions.limit + 1 }); - if (pageResult.error) { - emitWorkbenchTraceEventsReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 503, - phase: "events_query_failed", - pageOptions, - projection, - sessionId: factSessionId(session), - metadataCount: metadata.count, - errorCode: pageResult.error?.code ?? "workbench_trace_events_query_failed" - }, pageResult.error); - return sendJson(response, 503, workbenchProjectionStoreError(pageResult.error)); - } - const traceTurn = factTurnForTrace(metadataFacts, traceId, traceId); - const turnTraceStatus = normalizeStatus(traceTurn?.status); - const traceStatus = turnTraceStatus !== "unknown" ? turnTraceStatus : durableTraceStatus(pageResult.facts.traceEvents); - const page = traceEventPageFromFacts(pageResult.facts.traceEvents, pageOptions, { total: projection.lastProjectedSeq, traceLastSeq: projection.lastProjectedSeq, traceStatus }); - const missingTraceEvents = traceEventPageMissing(page, projection, pageOptions); - const missingTraceEventsDiagnostic = missingTraceEvents - ? traceEventsReadModelBlocker("workbench_trace_events_missing", "Workbench trace events are still catching up in the durable read model.", { traceId, projection, session, turn: traceTurn, route: url.pathname }) - : null; - if (missingTraceEvents) { - attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/traces/:id/events", code: "workbench_trace_events_missing", sessionId: factSessionId(session), traceId, count: pageResult.count }); - } - const responseProjection = page.blocker ? blockedTraceEventProjection(projection, page.blocker) : missingTraceEvents ? catchingUpTraceEventProjection(projection, missingTraceEventsDiagnostic) : projection; - const body = { - ok: true, - status: page.blocker ? "blocked" : missingTraceEvents ? "projecting" : "succeeded", - contractVersion: "workbench-trace-events-v1", - traceId, - sessionId: factSessionId(session), - threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), - ...page, - events: factArray(page.events).map((event) => ({ ...event, detailProjection: true, authority: "trace-detail-only" })), - projection: responseProjection, - detailProjection: true, - authority: "trace-detail-only", - realtimeAuthority: "workbench-realtime-authority-v2", - cache: pageResult.cache ?? metadata.cache ?? null, - projectionStatus: responseProjection.projectionStatus, - projectionHealth: responseProjection.projectionHealth, - lastProjectedSeq: responseProjection.lastProjectedSeq, - sourceRunId: responseProjection.sourceRunId, - sourceCommandId: responseProjection.sourceCommandId, - staleMs: responseProjection.staleMs, - blocker: responseProjection.blocker, - timing: responseProjection.timing, - startedAt: responseProjection.startedAt, - lastEventAt: responseProjection.lastEventAt, - finishedAt: responseProjection.finishedAt, - durationMs: responseProjection.durationMs, - diagnostic: responseProjection.diagnostic ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; - recordWorkbenchCacheMetric(options, url, body, startedAt); - emitWorkbenchTraceEventsReadOtel(traceId, options, { - startTimeMs: startedAt, - statusCode: 200, - phase: page.blocker ? "blocked" : missingTraceEvents ? "events_catching_up" : "succeeded", - pageOptions, - page, - projection: responseProjection, - sessionId: body.sessionId, - turnId: factTurnId(traceTurn, traceId), - readCount: pageResult.count, - rawEventCount: factArray(pageResult.facts?.traceEvents).length, - metadataCount: metadata.count, - traceStatus - }); - sendJson(response, 200, body); -} - -function emitWorkbenchTurnStatusReadOtel(traceId, options = {}, fields = {}, error = null) { - const safeId = safeTraceId(traceId); - if (!safeId) return; - const statusCode = Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : 0; - void emitCodeAgentOtelSpan("turn_status_read", safeId, options.env ?? process.env, { - startTimeMs: fields.startTimeMs, - status: statusCode >= 500 || error ? "error" : "ok", - error, - attributes: { - "http.method": "GET", - "http.route": "/v1/workbench/turns/:id", - "http.status_code": statusCode || null, - phase: fields.phase ?? null, - status: fields.status ?? null, - terminal: typeof fields.terminal === "boolean" ? fields.terminal : null, - sessionId: fields.sessionId ?? null, - turnId: fields.turnId ?? safeId, - readCount: nonNegativeInteger(fields.readCount), - traceEventCount: nonNegativeInteger(fields.traceEventCount), - latestProjectedSeq: nonNegativeInteger(fields.latestProjectedSeq), - projectionStatus: fields.projectionStatus ?? null, - projectionHealth: fields.projectionHealth ?? null, - sourceRunId: fields.sourceRunId ?? null, - sourceCommandId: fields.sourceCommandId ?? null, - errorCode: fields.errorCode ?? null, - valuesRedacted: true - } - }); -} - -function emitWorkbenchTraceEventsReadOtel(traceId, options = {}, fields = {}, error = null) { - const safeId = safeTraceId(traceId); - if (!safeId) return; - const page = fields.page ?? {}; - const range = page.range ?? {}; - const projection = fields.projection ?? {}; - const pageOptions = fields.pageOptions ?? {}; - const statusCode = Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : 0; - const returnedEvents = Array.isArray(page.events) ? page.events.length : nonNegativeInteger(fields.returnedEvents); - const afterProjectedSeq = nonNegativeInteger(pageOptions.afterProjectedSeq ?? range.afterProjectedSeq); - const limit = nonNegativeInteger(pageOptions.limit ?? range.limit); - const hasMore = typeof page.hasMore === "boolean" ? page.hasMore : null; - void emitCodeAgentOtelSpan("trace_events_read", safeId, options.env ?? process.env, { - startTimeMs: fields.startTimeMs, - status: statusCode >= 500 || error ? "error" : "ok", - error, - attributes: { - "http.method": "GET", - "http.route": "/v1/workbench/traces/:id/events", - "http.status_code": statusCode || null, - phase: fields.phase ?? null, - sessionId: fields.sessionId ?? null, - turnId: fields.turnId ?? safeId, - returnedEvents, - rawEventCount: nonNegativeInteger(fields.rawEventCount), - readCount: nonNegativeInteger(fields.readCount), - metadataCount: nonNegativeInteger(fields.metadataCount), - sinceSeq: afterProjectedSeq, - afterProjectedSeq, - limit, - fromSeq: nonNegativeInteger(range.fromProjectedSeq), - toSeq: nonNegativeInteger(range.toProjectedSeq), - totalEvents: nonNegativeInteger(range.total ?? page.eventCount), - hasMore, - fullTraceLoaded: typeof page.fullTraceLoaded === "boolean" ? page.fullTraceLoaded : hasMore === null ? null : !hasMore && !page.blocker, - traceLastSeq: nonNegativeInteger(page.traceLastSeq ?? projection.lastProjectedSeq), - projectionStatus: projection.projectionStatus ?? null, - projectionHealth: projection.projectionHealth ?? null, - sourceRunId: projection.sourceRunId ?? null, - sourceCommandId: projection.sourceCommandId ?? null, - traceStatus: fields.traceStatus ?? page.traceStatus ?? null, - errorCode: fields.errorCode ?? null, - valuesRedacted: true - } - }); -} - -function emitWorkbenchSessionListReadOtel(request, options = {}, fields = {}, error = null) { - const payload = fields.payload && typeof fields.payload === "object" ? fields.payload : {}; - const sessions = Array.isArray(payload.sessions) ? payload.sessions : []; - const fallbackTitleCount = sessions.filter(sessionSummaryNeedsTitleFallback).length; - const traceIds = uniqueText(sessions.map((session) => session?.lastTraceId)).slice(0, 8); - const businessTraceId = (safeTraceId(traceIds[0]) ?? textValue(fields.includeSessionId)) || "workbench_session_list"; - void emitCodeAgentOtelSpan("session_list_read", businessTraceId, options.env ?? process.env, { - startTimeMs: fields.startTimeMs, - status: error ? "error" : "ok", - error, - attributes: { - "http.method": "GET", - "http.route": "/v1/workbench/sessions", - "http.status_code": Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : null, - source: fields.source ?? null, - limit: nonNegativeInteger(fields.limit), - offset: nonNegativeInteger(fields.offset), - includeSessionId: fields.includeSessionId ?? null, - sessionCount: sessions.length, - fallbackTitleCount, - fallbackTitleRatio: sessions.length > 0 ? Number((fallbackTitleCount / sessions.length).toFixed(4)) : 0, - emptyPreviewCount: sessions.filter((session) => !sessionPreviewText(session)).length, - sessionIds: uniqueText(sessions.map((session) => session?.sessionId)).slice(0, 8).join(","), - traceIds: traceIds.join(","), - hasMore: payload.hasMore === true, - valuesRedacted: true - } - }); -} - -function emitWorkbenchSessionMessagesReadOtel(request, options = {}, fields = {}, error = null) { - const messages = Array.isArray(fields.messages) ? fields.messages : []; - const traceIds = uniqueText(messages.map((message) => message?.traceId)).slice(0, 8); - const businessTraceId = (safeTraceId(traceIds[0]) ?? textValue(fields.sessionId)) || "workbench_session_messages"; - const roleSequence = messagesRoleSequence(messages); - void emitCodeAgentOtelSpan("session_messages_read", businessTraceId, options.env ?? process.env, { - startTimeMs: fields.startTimeMs, - status: error ? "error" : "ok", - error, - attributes: { - "http.method": "GET", - "http.route": "/v1/workbench/sessions/:id/messages", - "http.status_code": Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : null, - sessionId: fields.sessionId ?? null, - limit: nonNegativeInteger(fields.limit), - offset: nonNegativeInteger(fields.offset), - returnedMessages: messages.length, - totalMessages: nonNegativeInteger(fields.total), - hasMore: fields.hasMore === true, - roleSequencePrefix: roleSequence.slice(0, 32), - consecutiveUserPrefix: consecutiveRolePrefix(messages, "user"), - adjacentSameRoleCount: adjacentSameRoleCount(messages), - timelineDigest: workbenchTimelineDigest(messages), - userCount: messages.filter((message) => normalizeRole(message?.role) === "user").length, - agentCount: messages.filter((message) => isAssistantLikeRole(message?.role)).length, - traceIds: traceIds.join(","), - valuesRedacted: true - } - }); -} - -function messagesRoleSequence(messages = []) { - return messages.map((message) => roleSequenceSymbol(message?.role)).join(""); -} - -function workbenchTimelineDigest(messages = []) { - const payload = messages.map((message) => { - const terminal = isTerminalProjectionStatus(normalizeStatus(message?.status)) || Boolean(message?.finishedAt); - return { - messageId: message?.messageId ?? null, - role: normalizeRole(message?.role), - sessionId: message?.sessionId ?? null, - traceId: message?.traceId ?? null, - turnId: message?.turnId ?? null, - status: normalizeStatus(message?.status), - partIds: factArray(message?.parts).map((part) => part?.partId ?? null), - textHash: textValue(message?.text || message?.textPreview) ? hash(textValue(message?.text || message?.textPreview)).slice(0, 16) : null, - startedAt: message?.startedAt ?? null, - finishedAt: terminal ? message?.finishedAt ?? null : null, - durationMs: terminal && Number.isFinite(Number(message?.durationMs)) ? Math.trunc(Number(message.durationMs)) : null - }; - }); - return `sha256:${hash(JSON.stringify(payload))}`; -} - -function sessionSummaryNeedsTitleFallback(session) { - if (!session || typeof session !== "object") return true; - return !sessionPreviewText(session); -} - -function sessionPreviewText(session) { - return textValue( - session?.firstUserMessagePreview - ?? session?.snapshot?.firstUserMessagePreview - ?? session?.userPreview - ?? session?.title - ?? session?.name - ); -} - -function roleSequenceSymbol(role) { - const normalized = normalizeRole(role); - if (normalized === "user") return "U"; - if (normalized === "agent" || normalized === "assistant") return "A"; - if (normalized === "system") return "S"; - return "?"; -} - -function normalizeRole(role) { - return textValue(role).toLowerCase(); -} - -function consecutiveRolePrefix(messages = [], role) { - let count = 0; - for (const message of messages) { - if (normalizeRole(message?.role) !== role) break; - count += 1; - } - return count; -} - -function adjacentSameRoleCount(messages = []) { - let count = 0; - for (let index = 1; index < messages.length; index += 1) { - const previous = normalizeRole(messages[index - 1]?.role); - if (previous && previous === normalizeRole(messages[index]?.role)) count += 1; - } - return count; -} - -function traceEventPageFromFacts(sourceEvents, options, metadata = {}) { - const rows = factArray(sourceEvents).filter((event) => factProjectedSeq(event)).sort(compareFactTraceEventsAsc); - const collision = projectedSeqCollision(rows); - if (collision) return blockedTraceEventPage(options, metadata, collision); - const pageRows = rows.slice(0, options.limit); - const events = pageRows.map(factTraceEventDto).filter(Boolean); - const toProjectedSeq = events.length ? events.at(-1).projectedSeq : null; - const hasMore = rows.length > options.limit; - const total = Number.isFinite(Number(metadata.total)) ? Math.trunc(Number(metadata.total)) : Math.max(options.afterProjectedSeq, toProjectedSeq ?? options.afterProjectedSeq); - const traceLastSeq = Number.isFinite(Number(metadata.traceLastSeq)) ? Math.trunc(Number(metadata.traceLastSeq)) : total; - return { - events, - eventCount: total ?? Math.max(options.afterProjectedSeq, toProjectedSeq ?? options.afterProjectedSeq), - range: { - afterProjectedSeq: options.afterProjectedSeq, - fromProjectedSeq: events.length ? events[0].projectedSeq : null, - toProjectedSeq, - limit: options.limit, - returned: events.length, - total - }, - hasMore, - fullTraceLoaded: !hasMore && (toProjectedSeq ?? options.afterProjectedSeq) >= traceLastSeq, - traceLastSeq, - nextProjectedSeq: events.length ? toProjectedSeq : options.afterProjectedSeq, - nextCursor: hasMore && toProjectedSeq ? `projected:${toProjectedSeq}` : null, - traceStatus: metadata.traceStatus ?? "unknown", - updatedAt: events.at(-1)?.updatedAt ?? events.at(-1)?.createdAt ?? null - }; -} - -function projectedSeqCollision(rows = []) { - const seen = new Map(); - for (const row of rows) { - const projectedSeq = factProjectedSeq(row); - if (!projectedSeq) continue; - const id = textValue(row?.id ?? row?.sourceEventId) || null; - const existing = seen.get(projectedSeq); - if (existing && existing.id !== id) { - return { - code: "projected_seq_collision", - category: "workbench-projection", - layer: "workbench-trace-events", - message: "Trace projection contains duplicate projectedSeq values; canonical event order is blocked until projection is rebuilt.", - projectedSeq, - eventIds: [existing.id, id].filter(Boolean), - retryable: false, - valuesPrinted: false - }; - } - if (!existing) seen.set(projectedSeq, { id }); - } - return null; -} - -function blockedTraceEventPage(options, metadata = {}, blocker) { - const total = Number.isFinite(Number(metadata.total)) ? Math.trunc(Number(metadata.total)) : null; - return { - events: [], - eventCount: total ?? options.afterProjectedSeq, - range: { - afterProjectedSeq: options.afterProjectedSeq, - fromProjectedSeq: null, - toProjectedSeq: null, - limit: options.limit, - returned: 0, - total - }, - hasMore: false, - nextProjectedSeq: options.afterProjectedSeq, - nextCursor: null, - traceStatus: metadata.traceStatus ?? "unknown", - updatedAt: null, - blocker - }; -} - -function blockedTraceEventProjection(projection = {}, blocker) { - return { - ...projection, - projectionStatus: "blocked", - projectionHealth: "degraded", - blocker, - valuesRedacted: true - }; -} - -function catchingUpTraceEventProjection(projection = {}, diagnostic = null) { - const status = projection.projectionStatus === "caught-up" || !projection.projectionStatus ? "projecting" : projection.projectionStatus; - const health = projection.projectionHealth === "caught-up" || projection.projectionHealth === "healthy" || !projection.projectionHealth ? "projecting" : projection.projectionHealth; - return { - ...projection, - projectionStatus: status, - projectionHealth: health, - blocker: null, - diagnostic: diagnostic ? { ...diagnostic, projectionStatus: status, projectionHealth: health } : null, - valuesRedacted: true - }; -} - -function traceEventPageMissing(page = {}, projection = {}, options = {}) { - if (page.blocker) return false; - const expectedSeq = Number(projection?.lastProjectedSeq); - if (!Number.isFinite(expectedSeq) || expectedSeq <= 0) return false; - return factArray(page.events).length === 0 && expectedSeq > Number(options.afterProjectedSeq ?? 0); -} - -function traceEventsReadModelBlocker(code, message, { traceId, projection = {}, session = null, turn = null, route = null } = {}) { - return { - code, - message, - userMessage: message, - layer: "workbench-read-model", - category: "trace-events", - retryable: true, - route, - traceId, - sessionId: factSessionId(session) ?? turn?.sessionId ?? null, - turnId: factTurnId(turn, traceId), - projectionStatus: projection.projectionStatus ?? null, - projectionHealth: "degraded", - lastProjectedSeq: projection.lastProjectedSeq ?? null, - sourceRunId: projection.sourceRunId ?? null, - sourceCommandId: projection.sourceCommandId ?? null, - retryAfterMs: 5000, - valuesPrinted: false - }; -} - -function workbenchTraceEventsReadModelError(blocker, { traceId, projection = {}, session = null, turn = null } = {}) { - const responseProjection = blockedTraceEventProjection(projection, blocker); - return workbenchError(blocker.code, blocker.message, { - traceId, - sessionId: blocker.sessionId ?? factSessionId(session) ?? turn?.sessionId ?? null, - turnId: blocker.turnId ?? factTurnId(turn, traceId), - route: blocker.route ?? null, - layer: blocker.layer, - category: blocker.category, - retryable: blocker.retryable, - projection: responseProjection, - projectionStatus: responseProjection.projectionStatus, - projectionHealth: responseProjection.projectionHealth, - lastProjectedSeq: responseProjection.lastProjectedSeq, - sourceRunId: responseProjection.sourceRunId, - sourceCommandId: responseProjection.sourceCommandId, - blocker, - diagnostic: { - contractVersion: "hwlab-error-diagnostic-v1", - route: blocker.route ?? null, - layer: blocker.layer, - category: blocker.category, - code: blocker.code, - httpStatus: 404, - source: "server", - retryable: blocker.retryable, - valuesPrinted: false - } - }); -} - -function projectionText(...values) { - for (const value of values) { - if (value && typeof value === "object") { - const nested = messageAuthorityTextValue(value.text ?? value.content ?? value.message ?? value.summary ?? value.preview ?? value.title); - if (nested) return nested; - continue; - } - const text = messageAuthorityTextValue(value); - if (text) return text; - } - return null; -} - -function partFact(part, index, messageId, traceId) { - const type = textValue(part?.type) || "text"; - const text = messageAuthorityTextValue(part?.text ?? part?.content ?? part?.message); - return { - partId: safePartId(part?.partId ?? part?.id) || `prt_${hash(`${messageId}:${index}:${type}`).slice(0, 24)}`, - messageId, - traceId, - type, - text: text || null, - status: normalizeStatus(part?.status ?? "completed"), - toolName: textValue(part?.toolName ?? part?.name) || null, - createdAt: textValue(part?.createdAt ?? part?.timestamp) || null, - valuesRedacted: part?.valuesRedacted !== false - }; -} - -function tracePageOptions(url) { - const cursor = textValue(url.searchParams.get("cursor")); - const cursorProjectedSeq = cursor.startsWith("projected:") ? Number.parseInt(cursor.slice("projected:".length), 10) : NaN; - const afterProjectedSeq = Number.isInteger(cursorProjectedSeq) && cursorProjectedSeq >= 0 - ? cursorProjectedSeq - : nonNegativeInteger(url.searchParams.get("afterProjectedSeq"), 0); - return { afterProjectedSeq, limit: boundedLimit(url.searchParams.get("limit")) }; -} - -async function realtimeInitialSessionContext(readModel, actor, sessionId) { - const safeId = safeSessionId(sessionId); - if (!safeId || typeof readModel?.queryFacts !== "function") { - const blocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench realtime initial session must be read from durable facts; the facts query interface is unavailable.", { session: { sessionId: safeId }, route: "/v1/workbench/events" }); - return { facts: emptyFactSet(), session: null, blocker }; - } - try { - const result = await readModel.queryFacts({ sessionId: safeId, families: WORKBENCH_SESSION_DETAIL_FAMILIES, limit: MAX_PAGE_LIMIT }); - if (result.error || result.durable === false) { - const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" }); - blocker.causeCode = result.error?.code ?? null; - blocker.causeName = result.error?.name ?? null; - blocker.message = result.error?.message ? `${blocker.message} Cause: ${result.error.message}` : result.durable === false ? `${blocker.message} Cause: durable facts reader unavailable.` : blocker.message; - return { facts: emptyFactSet(), session: null, blocker }; - } - const facts = result.facts ?? emptyFactSet(); - const session = visibleFactSessions(facts, actor).find((item) => factSessionId(item) === safeId) ?? null; - return { facts, session, blocker: null }; - } catch (error) { - const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" }); - blocker.causeCode = error?.code ?? null; - blocker.causeName = error?.name ?? null; - blocker.message = error?.message ? `${blocker.message} Cause: ${error.message}` : blocker.message; - return { facts: emptyFactSet(), session: null, blocker }; - } -} - -async function writeTraceRealtimeSnapshot({ writeEvent, options, actor, traceId, reason }) { - const context = await visibleTraceContext(options, actor, traceId); - if (!context.visible) { - writeEvent("workbench.trace.unavailable", { - type: "trace.unavailable", - reason, - traceId, - error: { code: "workbench_trace_not_found", message: "Workbench trace is not visible to the current actor." } - }); - return; - } - if (context.factsReadBlocker) { - writeEvent("workbench.error", { - type: "error", - traceId, - reason, - error: context.factsReadBlocker - }); - } - const sessionId = factSessionId(context.factSession) ?? safeSessionId(context.session?.id) ?? null; - const threadId = safeOpaqueId(context.factSession?.threadId ?? context.session?.threadId) ?? (textValue(context.factSession?.threadId ?? context.session?.threadId) || null); - const realtimeTraceSnapshot = context.factsReadBlocker ? blockedRealtimeTraceSnapshot(context) : traceSnapshotForRealtime(context.trace); - const traceSeq = context.factsReadBlocker ? 0 : traceSnapshotLastSeq(context.trace); - writeEvent("workbench.trace.snapshot", { - type: "trace.snapshot", - reason, - sessionId, - threadId, - traceId, - snapshot: { ...realtimeTraceSnapshot, sessionId, threadId }, - cursor: { traceSeq } - }); - writeEvent("workbench.turn.snapshot", { - type: "turn.snapshot", - reason, - sessionId, - threadId, - traceId, - turn: realtimeTurnSnapshot(context), - cursor: { traceSeq } - }); -} - -function realtimeTurnSnapshot(context) { - if (context?.factsReadBlocker) return blockedRealtimeTurnSnapshot(context, context.factsReadBlocker); - const facts = context?.facts; - const traceId = safeTraceId(context?.traceId); - if (traceId && facts && context?.factSession) { - const factTurn = factTurnForTrace(facts, traceId); - const projected = factTurnSnapshot({ turn: factTurn, session: context.factSession, facts, traceId, turnId: context.turnId }); - if (projected) return projected; - } - const blocker = traceEventsReadModelBlocker("workbench_facts_turn_missing", "Workbench durable facts did not contain a turn snapshot; realtime must surface the read-model gap instead of falling back to legacy trace projection.", { traceId, session: context?.factSession, route: "/v1/workbench/events" }); - return blockedRealtimeTurnSnapshot(context, blocker); -} - -function blockedRealtimeTraceSnapshot(context) { - const traceId = safeTraceId(context?.traceId ?? context?.factsReadBlocker?.traceId) ?? null; - const blocker = context?.factsReadBlocker ?? traceEventsReadModelBlocker("workbench_facts_unavailable", "Workbench durable facts are unavailable for realtime trace snapshot.", { traceId, route: "/v1/workbench/events" }); - const projection = blockedTraceEventProjection({}, blocker); - return { - traceId, - status: "unknown", - traceStatus: "unknown", - events: [], - eventCount: 0, - fullTraceLoaded: true, - hasMore: false, - nextProjectedSeq: 0, - updatedAt: null, - projection, - projectionStatus: projection.projectionStatus, - projectionHealth: projection.projectionHealth, - staleMs: projection.staleMs ?? null, - blocker, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function blockedRealtimeTurnSnapshot(context, blocker) { - const traceId = safeTraceId(context?.traceId ?? blocker?.traceId) ?? null; - const turnId = safeTurnId(context?.turnId ?? blocker?.turnId) || traceId; - const sessionId = safeSessionId(blocker?.sessionId ?? factSessionId(context?.factSession) ?? context?.session?.id) ?? null; - const threadId = safeOpaqueId(context?.factSession?.threadId ?? context?.session?.threadId) ?? (textValue(context?.session?.threadId) || null); - const projection = blockedTraceEventProjection({}, blocker); - return { - turnId, - traceId, - status: "unknown", - running: false, - terminal: false, - sessionId, - threadId, - userMessageId: null, - assistantMessageId: null, - assistantText: null, - finalResponse: null, - timing: null, - startedAt: null, - lastEventAt: null, - finishedAt: null, - durationMs: null, - agentRun: null, - trace: { - traceId, - status: "unknown", - eventCount: 0, - updatedAt: null - }, - urls: { - self: turnId ? `/v1/workbench/turns/${encodeURIComponent(turnId)}` : null, - traceEvents: traceId ? `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` : null - }, - projectionStatus: projection.projectionStatus, - projectionHealth: projection.projectionHealth, - staleMs: projection.staleMs ?? null, - blocker, - projection, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -async function writeTraceRealtimeSnapshotSafe(input) { - try { - await writeTraceRealtimeSnapshot(input); - } catch (error) { - input.writeEvent("workbench.error", { - type: "error", - traceId: input.traceId, - reason: input.reason, - error: { - code: "workbench_trace_snapshot_failed", - causeCode: error?.code ?? null, - retryable: error?.data?.retryable === true, - message: error?.message ?? "Workbench trace snapshot failed.", - valuesRedacted: true - } - }); - } -} - -async function visibleTraceContext(options, actor, traceId) { - const readModel = createWorkbenchReadModel(options, actor); - if (typeof readModel.queryFacts !== "function") { - const projection = factProjectionForTrace(emptyFactSet(), traceId); - const factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench durable facts query is unavailable; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); - return { visible: true, turnId: traceId, traceId, status: "unknown", projection, result: null, session: null, trace: factTraceSnapshot(emptyFactSet(), traceId), facts: emptyFactSet(), factSession: null, factsReadBlocker }; - } - - let facts = emptyFactSet(); - let factsReadBlocker = null; - try { - const factResult = await readModel.queryFacts({ traceId, families: [...WORKBENCH_SESSION_DETAIL_FAMILIES, ...WORKBENCH_TRACE_EVENT_PAGE_FAMILIES], limit: MAX_PAGE_LIMIT }); - if (factResult.error || factResult.durable === false) { - const projection = factProjectionForTrace(facts, traceId); - factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); - factsReadBlocker.causeCode = factResult.error?.code ?? null; - factsReadBlocker.causeName = factResult.error?.name ?? null; - factsReadBlocker.message = factResult.error?.message ? `${factsReadBlocker.message} Cause: ${factResult.error.message}` : factResult.durable === false ? `${factsReadBlocker.message} Cause: durable facts reader unavailable.` : factsReadBlocker.message; - } else { - facts = factResult?.facts ?? facts; - } - } catch (error) { - const projection = factProjectionForTrace(facts, traceId); - factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); - factsReadBlocker.causeCode = error?.code ?? null; - factsReadBlocker.causeName = error?.name ?? null; - factsReadBlocker.message = error?.message ? `${factsReadBlocker.message} Cause: ${error.message}` : factsReadBlocker.message; - } - - const resolved = factsReadBlocker ? { session: null, facts } : await visibleFactSessionForTrace(readModel, facts, actor, traceId); - if (resolved.error) { - const projection = factProjectionForTrace(facts, traceId); - factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts session lookup failed; realtime must surface the read-model error instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); - factsReadBlocker.causeCode = resolved.error?.code ?? null; - factsReadBlocker.causeName = resolved.error?.name ?? null; - factsReadBlocker.message = resolved.error?.message ? `${factsReadBlocker.message} Cause: ${resolved.error.message}` : factsReadBlocker.message; - } else if (resolved.facts) { - facts = resolved.facts; - } - - const factSession = resolved.session ?? null; - const trace = factTraceSnapshot(facts, traceId); - const turn = factTurnForTrace(facts, traceId, traceId); - let projection = factsReadBlocker ? blockedTraceEventProjection(factProjectionForTrace(facts, traceId), factsReadBlocker) : factProjectionForTrace(facts, traceId); - const visible = Boolean(factsReadBlocker || factSession || turn || factCheckpointForTrace(facts, traceId) || trace.eventCount > 0); - if (!visible) { - factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_trace_missing", "Workbench realtime could not find durable facts for the requested trace; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); - projection = blockedTraceEventProjection(projection, factsReadBlocker); - } - if (!factsReadBlocker && !factSession) { - factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_session_missing", "Workbench durable facts did not contain the trace session; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, turn, route: "/v1/workbench/events" }); - projection = blockedTraceEventProjection(projection, factsReadBlocker); - } - const snapshot = factTurnSnapshot({ turn, session: factSession, facts, traceId, turnId: traceId }); - return { visible: true, turnId: snapshot.turnId ?? traceId, traceId, status: snapshot.status, projection, result: null, session: null, trace, facts, factSession, factsReadBlocker }; -} - -function traceSnapshotForRealtime(snapshot) { - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - return { - ...snapshot, - events, - eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length, - lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function traceSnapshotSummary(snapshot) { - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - return { - traceId: snapshot?.traceId ?? null, - status: snapshot?.status ?? "missing", - eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length, - lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null, - updatedAt: snapshot?.updatedAt ?? null, - valuesRedacted: true - }; -} - -function traceSnapshotLastSeq(snapshot) { - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - return events.reduce((max, event, index) => Math.max(max, eventSeq(event, index)), 0); -} - -function handleWorkbenchReadModelFailure(response, url, options = {}, error) { - const classified = classifyWorkbenchReadModelFailure(error); - const route = workbenchReadRouteTemplate(url?.pathname ?? "/v1/workbench"); - logWorkbenchReadModelFailure(options.logger ?? console, { - event: "workbench_read_model_route_failed", - ok: false, - route, - statusCode: classified.statusCode, - errorCode: classified.code, - errorName: error?.name ?? "Error", - message: error instanceof Error ? error.message : String(error ?? "unknown"), - valuesRedacted: true - }); - if (response.headersSent || response.destroyed) return; - sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { - route, - errorName: error?.name ?? "Error", - retryable: classified.retryable === true, - transient: classified.transient === true, - serviceId: classified.serviceId ?? null, - runtimeStatus: classified.runtimeStatus ?? null - })); -} - -export function classifyWorkbenchReadModelFailure(error) { - const message = String(error?.message ?? error ?? ""); - const code = String(error?.code ?? "").toLowerCase(); - const runtimeStatus = Number(error?.data?.status ?? 0); - const runtimePayloadError = error?.data?.payload?.error ?? null; - if (error?.name === "WorkbenchRuntimeDependencyError" || code.startsWith("workbench_runtime_")) { - const retryable = error?.data?.retryable === true || error?.data?.transient === true || runtimePayloadError?.retryable === true; - return { - statusCode: retryable || (Number.isFinite(runtimeStatus) && runtimeStatus >= 500) ? 503 : 500, - code: String(runtimePayloadError?.code ?? error?.code ?? "workbench_runtime_dependency_failed"), - message: String(runtimePayloadError?.message ?? error?.message ?? "Workbench runtime dependency failed."), - retryable, - transient: retryable, - serviceId: String(error?.data?.serviceId ?? "hwlab-workbench-runtime"), - runtimeStatus: Number.isFinite(runtimeStatus) && runtimeStatus > 0 ? runtimeStatus : null - }; - } - if (/timeout|connection terminated|terminating connection|econnreset|econnrefused|etimedout/iu.test(message) || /timeout|econnreset|econnrefused|etimedout/u.test(code)) { - return { - statusCode: 503, - code: "workbench_read_model_store_unavailable", - message: "Workbench read model store is temporarily unavailable." - }; - } - return { - statusCode: 500, - code: "workbench_read_model_failed", - message: "Workbench read model failed." - }; -} - -function workbenchReadRouteTemplate(pathname) { - const path = String(pathname ?? ""); - if (path === "/v1/workbench/sessions") return path; - if (/^\/v1\/workbench\/sessions\/[^/]+\/messages$/u.test(path)) return "/v1/workbench/sessions/:id/messages"; - if (/^\/v1\/workbench\/sessions\/[^/]+$/u.test(path)) return "/v1/workbench/sessions/:id"; - if (/^\/v1\/workbench\/turns\/[^/]+$/u.test(path)) return "/v1/workbench/turns/:id"; - if (/^\/v1\/workbench\/traces\/[^/]+\/events$/u.test(path)) return "/v1/workbench/traces/:id/events"; - return path || "/v1/workbench"; -} - -function logWorkbenchReadModelFailure(logger, payload) { - try { - const line = JSON.stringify(payload); - if (typeof logger?.error === "function") logger.error(line); - else if (typeof logger?.warn === "function") logger.warn(line); - } catch { - // Read-model failure logging must not affect response generation. - } -} - -function methodNotAllowed(response, allowed) { - sendJson(response, 405, workbenchError("method_not_allowed", `Use ${allowed} for this Workbench read model route.`)); -} - -function workbenchError(code, message, extra = {}) { - return { - ok: false, - status: "failed", - error: { code, message, ...extra }, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function boundedLimit(value) { - return Math.min(MAX_PAGE_LIMIT, parsePositiveInteger(value, DEFAULT_PAGE_LIMIT)); -} - -function boundedSessionListLimit(value) { - return Math.min(MAX_PAGE_LIMIT, parsePositiveInteger(value, DEFAULT_SESSION_LIST_LIMIT)); -} - -function cursorOffset(value) { - const text = textValue(value); - if (!text) return 0; - if (text.startsWith("idx:")) return nonNegativeInteger(text.slice(4), 0); - return nonNegativeInteger(text, 0); -} - -function cursorFromOffset(offset) { - return `idx:${Math.max(0, Math.trunc(Number(offset) || 0))}`; -} - -function nonNegativeInteger(value, fallback) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; -} - -function eventSeq(event, index) { - const seq = Number(event?.seq); - return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; -} - -function normalizeStatus(value) { - const text = textValue(value).toLowerCase().replace(/_/gu, "-"); - if (text === "cancelled") return "canceled"; - return text || "unknown"; -} - -function normalizeTerminalStatus(value) { - const status = normalizeStatus(value); - return TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status) ? status : null; -} - -function firstUserPreview(messages) { - return messages.find((message) => message.role === "user")?.textPreview ?? null; -} - -function latestUserPreview(messages) { - return [...messages].reverse().find((message) => message.role === "user")?.textPreview ?? null; -} - -function latestValidMessagePreview(messages) { - return [...messages].reverse().find((message) => message.role === "user" || isAssistantLikeRole(message.role))?.textPreview ?? null; -} - -function latestAssistantPreview(messages) { - return [...messages].reverse().find((message) => isAssistantLikeRole(message.role))?.textPreview ?? null; -} - -function sessionTitleFromMessages(messages = []) { - return boundedPreviewText(latestUserPreview(messages) ?? firstUserPreview(messages) ?? latestAssistantPreview(messages)); -} - -function sessionPreviewFromMessages(messages = []) { - return boundedPreviewText(latestValidMessagePreview(messages) ?? firstUserPreview(messages) ?? latestAssistantPreview(messages)); -} - -function boundedPreviewText(value, maxLength = 160) { - const text = textValue(value).replace(/\s+/gu, " ").trim(); - if (!text) return null; - return text.length > maxLength ? `${text.slice(0, maxLength - 3)}...` : text; -} - -function isAssistantLikeRole(role) { - return role === "assistant" || role === "agent"; -} - -function objectValue(value) { - return value && typeof value === "object" && !Array.isArray(value) ? value : {}; -} - -function textValue(value) { - return String(value ?? "").trim(); -} - -function messageAuthorityTextValue(value) { - const text = String(value ?? "").replace(/\r\n?/gu, "\n"); - if (!text.trim() || text.trim() === "[object Object]") return ""; - return text; -} - -function safeTurnId(value) { - const text = textValue(value); - return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; -} - -function safeMessageId(value) { - const text = textValue(value); - return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; -} - -function safePartId(value) { - const text = textValue(value); - return /^prt_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; -} - -function hash(value) { - return createHash("sha256").update(String(value)).digest("hex"); -} +export { + classifyWorkbenchReadModelFailure, + handleWorkbenchReadModelHttp +} from "./server-workbench-read-http.ts"; +export { + drainWorkbenchRealtimeConnections, + handleWorkbenchRealtimeHttp +} from "./server-workbench-realtime-http.ts"; diff --git a/internal/cloud/server-workbench-read-http.ts b/internal/cloud/server-workbench-read-http.ts new file mode 100644 index 00000000..5a4a1f4c --- /dev/null +++ b/internal/cloud/server-workbench-read-http.ts @@ -0,0 +1,1417 @@ +/* + * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-read-model-timeline-contract; draft-2026-06-28-p0-d518-session-timeline-consistency; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-api-contract; PJ2026-010401 Web工作台 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-model-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 + * 职责: Workbench REST read model。只读投影 session/message/turn/trace facts，不执行 AgentRun sync、billing finalize 或 workspace repair。 + */ +import { createHash } from "node:crypto"; + +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { + parsePositiveInteger, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson +} from "./server-http-utils.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; +import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; +import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; +import { openKafkaEventStream } from "./kafka-event-bridge.ts"; +import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; +import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; +import * as workbenchFacts from "./server-workbench-facts.ts"; + +const DEFAULT_PAGE_LIMIT = 50; +const DEFAULT_SESSION_LIST_LIMIT = 20; +const MAX_PAGE_LIMIT = 100; +const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]); +const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); +const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); +const WORKBENCH_TRACE_EVENT_METADATA_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); +const WORKBENCH_TRACE_EVENT_PAGE_FAMILIES = Object.freeze(["traceEvents"]); + +function attachWorkbenchReadModelDiagnosticOtel(request, fields = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context) return; + const code = textValue(fields.code) || "workbench_read_model_not_found"; + const route = textValue(fields.route) || context.route || "/v1/workbench"; + context.route = route; + context.lastHttpError = { code, category: "workbench_read_model", layer: "workbench.read_model" }; + context.otelAttributes = { + ...(context.otelAttributes ?? {}), + "workbench.route": route, + "workbench.read_model.route": route, + "workbench.read_model.result": code, + "workbench.read_model.count": Number.isFinite(Number(fields.count)) ? Math.trunc(Number(fields.count)) : null, + "workbench.session_id": fields.sessionId ?? null, + "workbench.trace_id": fields.traceId ?? null, + "workbench.turn_id": fields.turnId ?? null, + traceId: fields.traceId ?? null, + "error.code": code, + "error.category": "workbench_read_model", + "error.layer": "workbench.read_model" + }; +} + +const { + factSessionSummary, + factSessionDetail, + factMessagesForSession, + canonicalFactMessageGroupsForSession, + factMessageCanonicalGroupKey, + selectCanonicalFactMessage, + factMessageCanonicalScore, + factMessageHasFinalResponsePart, + workbenchLifecycleMessageId, + factMessageDto, + factMessageAuthorityText, + factPartsHaveFinalResponse, + factSyntheticTerminalFinalResponse, + isTerminalProjectionStatus, + firstFactPartText, + factMessageFinalResponseText, + factTerminalMessageForTrace, + factTerminalStatusFromMessageProjection, + factPartDto, + factTurnForTrace, + factTurnSnapshot, + factTraceSnapshot, + factTraceTimingProjection, + factTraceEventIsTerminalAuthority, + factTraceEventTerminalTimes, + factTraceEventDto, + factProjectionForTrace, + factTimingProjection, + factCombinedTimingProjection, + factTimingSource, + firstTimestampIso, + latestTimestampIso, + elapsedFactMs, + timestampIso, + factCheckpointForTrace, + workbenchProjectionStoreError, + nonNegativeIntegerOrNull, + factSessionId, + factLastTraceId, + factLatestTraceIdForSession, + factCurrentTraceForSession, + factMessageTraceCandidate, + factTraceCandidate, + compareTraceCandidatesDesc, + nonUnknownStatus, + factTurnId, + factUpdatedAt, + factSeq, + factProjectedSeq, + normalizeProjectionStatus, + normalizeProjectionHealth, + compareFactSessionsDesc, + compareFactMessagesAsc, + compareFactMessagesForSessionAsc, + factMessageTimelineKey, + factTimelineAnchorMessageForTrace, + factMessageTimelineRoleRank, + firstFiniteNumber, + compareOptionalNumberAsc, + compareFactPartsAsc, + compareFactTraceEventsAsc, + compareFactRecordsDesc, + compareTimestampAsc, + compareTimestampDesc, + compareOptionalTimestampAsc, + compareNumberAsc, + compareNumberDesc, + compareText, + timestampMs, + optionalTimestampMs, + numericValue, + mergeFactSets, + emptyFactSet, + factArray, + uniqueText, + eventSeq, + normalizeStatus, + normalizeTerminalStatus, + firstUserPreview, + latestUserPreview, + latestValidMessagePreview, + latestAssistantPreview, + sessionTitleFromMessages, + sessionPreviewFromMessages, + boundedPreviewText, + isAssistantLikeRole, + objectValue, + textValue, + messageAuthorityTextValue, + safeTurnId, + safeMessageId, + safePartId, + hash +} = workbenchFacts; + +export async function handleWorkbenchReadModelHttp(request, response, url, options = {}) { + try { + const perf = options.backendPerformance; + const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); + if (!auth) return; + + if (url.pathname === "/v1/workbench/sync") { + await (perf ? perf.measure("workbench_sync", () => handleWorkbenchSyncHttp(request, response, url, options, auth.actor)) : handleWorkbenchSyncHttp(request, response, url, options, auth.actor)); + return; + } + + if (url.pathname === "/v1/workbench/sessions") { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + await (perf ? perf.measure("workbench_session_list", () => handleWorkbenchSessionList(request, response, url, options, auth.actor)) : handleWorkbenchSessionList(request, response, url, options, auth.actor)); + return; + } + + const sessionMessagesMatch = url.pathname.match(/^\/v1\/workbench\/sessions\/([^/]+)\/messages$/u); + if (sessionMessagesMatch) { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + await (perf ? perf.measure("workbench_message_page", () => handleWorkbenchMessagePage(request, response, url, options, auth.actor, decodeURIComponent(sessionMessagesMatch[1]))) : handleWorkbenchMessagePage(request, response, url, options, auth.actor, decodeURIComponent(sessionMessagesMatch[1]))); + return; + } + + const sessionMatch = url.pathname.match(/^\/v1\/workbench\/sessions\/([^/]+)$/u); + if (sessionMatch) { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + await (perf ? perf.measure("workbench_session_detail", () => handleWorkbenchSessionDetail(request, response, url, options, auth.actor, decodeURIComponent(sessionMatch[1]))) : handleWorkbenchSessionDetail(request, response, url, options, auth.actor, decodeURIComponent(sessionMatch[1]))); + return; + } + + const turnMatch = url.pathname.match(/^\/v1\/workbench\/turns\/([^/]+)$/u); + if (turnMatch) { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + await (perf ? perf.measure("workbench_turn_snapshot", () => handleWorkbenchTurnSnapshot(request, response, url, options, auth.actor, decodeURIComponent(turnMatch[1]))) : handleWorkbenchTurnSnapshot(request, response, url, options, auth.actor, decodeURIComponent(turnMatch[1]))); + return; + } + + const traceEventsMatch = url.pathname.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u); + if (traceEventsMatch) { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + await (perf ? perf.measure("workbench_trace_events", () => handleWorkbenchTraceEventPage(request, response, url, options, auth.actor, decodeURIComponent(traceEventsMatch[1]))) : handleWorkbenchTraceEventPage(request, response, url, options, auth.actor, decodeURIComponent(traceEventsMatch[1]))); + return; + } + + sendJson(response, 404, workbenchError("workbench_route_not_found", "Workbench read model route is not implemented.", { route: url.pathname })); + } catch (error) { + handleWorkbenchReadModelFailure(response, url, options, error); + } +} + + +export function recordWorkbenchTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchTurnRead) return; + performanceStore.recordWorkbenchTurnRead({ + route: url?.pathname ?? "/v1/workbench/turns/:id", + status: body?.status ?? statusClassLabel(statusCode), + degradedReason: workbenchTurnDegradedReason(body), + durationMs: nowMs() - startedAt, + responseBytes: responseBodyBytes(body), + timedOut: statusCode === 504 || body?.error?.code === "timeout" + }); +} + +export function recordWorkbenchCacheMetric(options = {}, url, body = {}, startedAt = nowMs()) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchCache) return; + const cache = body?.cache && typeof body.cache === "object" ? body.cache : null; + if (!cache) return; + performanceStore.recordWorkbenchCache({ + route: url?.pathname ?? "/v1/workbench", + cacheKeyClass: cache.class, + cacheStatus: cache.status, + operation: "read", + durationMs: nowMs() - startedAt, + payloadBytes: cache.payloadBytes, + cacheAgeMs: cache.cacheAgeMs, + dbQueryAvoided: cache.dbQueryAvoided, + freshnessSloMs: cache.freshnessSloMs, + projectionSeq: cache.projectionSeq + }); +} + +export function recordWorkbenchProjectionMetric(options = {}, { result = null, trace = null, projection = null, diagnostic = null } = {}) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchProjection || !projection) return; + const sourceLatestSeq = workbenchProjectionSourceLatestSeq(result, trace, projection); + const lastProjectedSeq = Number(diagnostic?.lastProjectedSeq ?? projection?.lastProjectedSeq ?? 0); + if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return; + performanceStore.recordWorkbenchProjection({ + sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq, + lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0, + sourceLatestEventAt: workbenchProjectionSourceLatestAt(result, trace, projection), + projectedLatestEventAt: diagnostic?.updatedAt ?? projection?.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null, + terminalSourceAt: workbenchTerminalSourceAt(result, projection), + terminalProjectedAt: diagnostic?.projectionStatus === "caught-up" ? diagnostic?.updatedAt ?? projection?.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null, + status: projection?.status ?? "unknown", + reason: workbenchProjectionReason(diagnostic), + projectionStatus: diagnostic?.projectionStatus ?? "unknown", + source: "workbench_read_model" + }); +} + +export function workbenchProjectionSourceLatestSeq(result = null, trace = null, projection = null) { + const candidates = [ + result?.agentRun?.lastSeq, + result?.traceSummary?.agentRun?.lastSeq, + result?.traceSummary?.lastSeq, + result?.providerTrace?.lastSeq, + result?.runnerTrace?.lastEvent?.sourceSeq, + result?.runnerTrace?.lastEvent?.seq, + result?.runnerTrace?.eventCount, + trace?.lastSeq, + trace?.lastEvent?.sourceSeq, + trace?.lastEvent?.seq, + trace?.eventCount, + projection?.lastProjectedSeq, + projection?.eventCount + ]; + for (const value of candidates) { + const seq = Number(value); + if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq); + } + return NaN; +} + +export function workbenchProjectionSourceLatestAt(result = null, trace = null, projection = null) { + return result?.traceSummary?.updatedAt + ?? result?.agentRun?.updatedAt + ?? result?.providerTrace?.updatedAt + ?? result?.runnerTrace?.updatedAt + ?? trace?.updatedAt + ?? result?.updatedAt + ?? projection?.updatedAt + ?? null; +} + +export function workbenchTerminalSourceAt(result = null, projection = null) { + if (projection?.terminal !== true) return null; + return result?.traceSummary?.updatedAt + ?? result?.agentRun?.updatedAt + ?? result?.updatedAt + ?? projection?.updatedAt + ?? null; +} + +export function workbenchProjectionReason(diagnostic = null) { + if (diagnostic?.blocker) return diagnostic.blocker.code ?? "projection_blocked"; + if (diagnostic?.projectionStatus === "projecting") return "projecting"; + if (diagnostic?.projectionStatus === "caught-up") return "none"; + return diagnostic?.projectionStatus ?? "unknown"; +} + +export function workbenchTurnDegradedReason(body = {}) { + if (body?.projection?.blocker?.code) return body.projection.blocker.code; + if (body?.blocker?.code) return body.blocker.code; + if (body?.error?.code) return body.error.code; + if (body?.projectionStatus === "projecting") return "projecting"; + if (body?.projectionStatus === "blocked") return "projection_blocked"; + return "none"; +} + +export function responseBodyBytes(body = {}) { + try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; } +} + +export function statusClassLabel(value) { + const status = Number(value); + if (!Number.isFinite(status) || status <= 0) return "unknown"; + return `${Math.floor(status / 100)}xx`; +} + +export async function authenticateWorkbenchRead(request, response, options) { + const access = options.accessController; + if (!access?.authenticate) { + sendJson(response, 503, workbenchError("workbench_access_controller_missing", "Workbench read model requires access controller.")); + return null; + } + await access.ensureBootstrap?.(); + const auth = await access.authenticate(request, { required: true }); + if (!auth?.ok) { + sendJson(response, auth?.status ?? 401, auth ?? workbenchError("auth_required", "Authentication is required.")); + return null; + } + return auth; +} + +export async function handleWorkbenchSessionList(request, response, url, options, actor) { + const startedAt = nowMs(); + if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) return sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench session list is keyed by sessionId only.")); + const limit = boundedSessionListLimit(url.searchParams.get("limit")); + const offset = cursorOffset(url.searchParams.get("cursor") ?? url.searchParams.get("after")); + const includeSessionId = safeSessionId(url.searchParams.get("includeSessionId")); + const includeRouteId = includeSessionId ?? safeConversationId(url.searchParams.get("includeSessionId")); + const runtime = workbenchRuntimeReader(request, options); + if (typeof runtime?.listWorkbenchSessions !== "function") { + if (typeof runtime?.queryWorkbenchFacts !== "function") { + throw runtimeDependencyError("workbench_runtime_unconfigured", "Workbench runtime service is required for session list.", false); + } + const readModel = workbenchRuntimeFactsReadModel(request, options, actor); + const payload = await handleWorkbenchSessionListFromFacts(readModel, url, options, actor, { limit, offset, includeRouteId }); + recordWorkbenchCacheMetric(options, url, payload, startedAt); + emitWorkbenchSessionListReadOtel(request, options, { + startTimeMs: startedAt, + statusCode: 200, + source: "facts", + limit, + offset, + includeSessionId: includeRouteId, + payload + }); + return sendJson(response, 200, payload); + } + const payload = await runtime.listWorkbenchSessions({ + limit, + offset, + cursor: url.searchParams.get("cursor") ?? url.searchParams.get("after") ?? null, + includeSessionId: includeRouteId ?? null, + actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } + }); + recordWorkbenchCacheMetric(options, url, payload, startedAt); + emitWorkbenchSessionListReadOtel(request, options, { + startTimeMs: startedAt, + statusCode: 200, + source: "runtime", + limit, + offset, + includeSessionId: includeRouteId, + payload + }); + return sendJson(response, 200, payload); +} + +export async function handleWorkbenchSessionListFromFacts(readModel, url, options, actor, { limit, offset, includeRouteId }) { + const perf = options.backendPerformance; + const pageQuery = { ownerUserId: actor.role === "admin" ? undefined : actor.id, limit: limit + offset + 1, families: WORKBENCH_SESSION_LIST_PAGE_FAMILIES, sessionsOrder: "updated_desc", sessionProjection: "summary" }; + const includeQueryPromise = includeRouteId ? queryWorkbenchSessionInclude(readModel, includeRouteId, perf) : null; + const pageResult = await (perf ? perf.measure("workbench_session_page_query", () => readModel.queryFacts(pageQuery)) : readModel.queryFacts(pageQuery)); + if (pageResult.error) throw pageResult.error; + let pageFacts = pageResult.facts; + let naturalPage = visibleFactSessions(pageFacts, actor).slice(offset, offset + limit + 1); + if (includeRouteId && !naturalPage.some((session) => factSessionMatchesRouteId(session, includeRouteId))) { + const includeResult = await includeQueryPromise; + if (includeResult.error) throw includeResult.error; + const included = visibleFactSessions(includeResult.facts, actor)[0] ?? null; + if (included) { + pageFacts = mergeFactSets(pageFacts, includeResult.facts); + naturalPage = [included, ...naturalPage]; + } + } + const pageSessions = naturalPage.slice(0, limit); + const summaryResult = await (perf ? perf.measure("workbench_session_summary_query", () => queryFactsForSessionSummaries(readModel, pageSessions)) : queryFactsForSessionSummaries(readModel, pageSessions)); + if (summaryResult.error) throw summaryResult.error; + pageFacts = mergeFactSets(pageFacts, summaryResult.facts); + const summaries = pageSessions.map((session) => factSessionSummary(session, pageFacts)).filter(Boolean); + const hasMore = naturalPage.length > limit; + return { + ok: true, + status: "succeeded", + contractVersion: "workbench-sessions-v1", + sessions: summaries, + count: summaries.length, + cursor: offset > 0 ? cursorFromOffset(offset) : null, + hasMore, + nextCursor: hasMore ? cursorFromOffset(offset + limit) : null, + cache: pageResult.cache ?? summaryResult.cache ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +export function runtimeDependencyError(code, message, retryable = true) { + const error = new Error(message || code); + error.name = "WorkbenchRuntimeDependencyError"; + error.code = code; + error.data = { serviceId: "hwlab-workbench-runtime", retryable, transient: retryable, valuesRedacted: true }; + return error; +} + +export function workbenchRuntimeReader(request, options) { + return options.workbenchRuntime + ?? options.runtimeStore + ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger, traceparent: request?.hwlabHttpRequestContext?.traceparent }); +} + +export function workbenchRuntimeFactsReadModel(request, options, actor) { + const runtime = workbenchRuntimeReader(request, options); + if (typeof runtime?.queryWorkbenchFacts !== "function") { + throw runtimeDependencyError("workbench_runtime_unconfigured", "Workbench runtime service is required for Workbench facts reads.", false); + } + return { + async queryFacts(query = {}) { + try { + return await runtime.queryWorkbenchFacts({ + ...query, + actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } + }); + } catch (error) { + return { facts: emptyFactSet(), count: 0, durable: true, error: workbenchRuntimeFactsQueryError(error), persistence: null }; + } + } + }; +} + +export function workbenchRuntimeFactsQueryError(error) { + const normalized = error instanceof Error ? error : new Error(String(error ?? "Workbench facts query failed.")); + const data = objectValue(normalized.data); + normalized.data = { + ...data, + retryable: data.retryable !== false, + transient: data.transient !== false, + retryAfterMs: data.retryAfterMs ?? 5000, + valuesRedacted: true + }; + return normalized; +} + +export function queryWorkbenchSessionInclude(readModel, includeRouteId, perf = null) { + const run = () => queryFactsByRouteId(readModel, includeRouteId, { families: WORKBENCH_SESSION_LIST_PAGE_FAMILIES }); + const promise = perf ? perf.measure("workbench_session_include_query", run) : run(); + return promise.catch((error) => ({ error })); +} + +export async function queryFactsByRouteId(readModel, routeId, query = {}) { + const baseQuery = { ...query, limit: query.limit ?? MAX_PAGE_LIMIT }; + const sessionId = safeSessionId(routeId); + if (sessionId) return readModel.queryFacts({ ...baseQuery, sessionId }); + const conversationId = safeConversationId(routeId); + if (conversationId) { + const sessionResult = await readModel.queryFacts({ ...baseQuery, families: WORKBENCH_SESSION_LIST_PAGE_FAMILIES, conversationId }); + if (sessionResult.error || queryRequestsOnlySessionFacts(baseQuery)) return sessionResult; + const resolvedSessionId = factSessionId(factArray(sessionResult.facts?.sessions)[0] ?? null); + if (!resolvedSessionId) return sessionResult; + const relatedResult = await readModel.queryFacts({ ...baseQuery, sessionId: resolvedSessionId }); + if (relatedResult.error) return relatedResult; + const facts = mergeFactSets(sessionResult.facts, relatedResult.facts); + return { facts, count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0), durable: relatedResult.durable ?? sessionResult.durable, persistence: relatedResult.persistence ?? sessionResult.persistence ?? null }; + } + return { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; +} + +export function queryRequestsOnlySessionFacts(query = {}) { + const families = Array.isArray(query.families) ? query.families : []; + return families.length === 1 && families[0] === "sessions"; +} + +export async function queryFactsForSessionSummaries(readModel, sessions = []) { + const sessionIds = uniqueText(sessions.map(factSessionId)); + if (sessionIds.length === 0 && uniqueText(sessions.map(factLastTraceId)).length === 0) return { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; + // Session list summary is hot and is also sampled by the two-page web-probe + // observer. Avoid parallel fan-out here so one list request cannot consume + // multiple durable-read connections while another page is refreshing. + const bySession = sessionIds.length > 0 + ? await readModel.queryFacts({ sessionIds, families: ["messages", "parts", "turns"] }) + : { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; + if (bySession.error) return bySession; + const traceIds = summaryTraceIds(sessions, bySession.facts); + const byTrace = traceIds.length > 0 + ? await readModel.queryFacts({ traceIds, families: ["checkpoints"] }) + : { facts: emptyFactSet(), count: 0, durable: false, persistence: null }; + if (byTrace.error) return byTrace; + const facts = mergeFactSets(bySession.facts, byTrace.facts); + return { facts, count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0), durable: bySession.durable ?? byTrace.durable, persistence: bySession.persistence ?? byTrace.persistence ?? null }; +} + +export function summaryTraceIds(sessions = [], facts = {}) { + return uniqueText([ + ...sessions.map(factLastTraceId), + ...sessions.map((session) => factCurrentTraceForSession(facts, factSessionId(session), session)?.traceId) + ]); +} + +export function visibleFactSessions(facts, actor) { + return hideShadowedAgentRunAliasSessions(factArray(facts?.sessions) + .filter((session) => canActorReadFactSession(session, actor)) + .sort(compareFactSessionsDesc)); +} + +export function hideShadowedAgentRunAliasSessions(sessions = []) { + const canonicalTraceIds = new Set(sessions + .filter((session) => !isAgentRunAliasSessionId(factSessionId(session))) + .map(factLastTraceId) + .filter(Boolean)); + if (canonicalTraceIds.size === 0) return sessions; + return sessions.filter((session) => !isAgentRunAliasSessionId(factSessionId(session)) || !canonicalTraceIds.has(factLastTraceId(session))); +} + +export function isAgentRunAliasSessionId(sessionId) { + return /^ses_agentrun_/u.test(textValue(sessionId)); +} + +export function canActorReadFactSession(session, actor) { + if (!session || !actor) return false; + if (normalizeStatus(session.status) === "archived") return false; + if (actor.role === "admin") return true; + return session.ownerUserId === actor.id; +} + +export async function visibleFactSessionForTrace(readModel, facts, actor, traceId) { + const visible = visibleFactSessions(facts, actor); + const byLastTrace = visible.find((item) => item.lastTraceId === traceId) ?? null; + if (byLastTrace) return { session: byLastTrace, facts }; + const relatedSessionIds = uniqueText([ + factTurnForTrace(facts, traceId)?.sessionId, + factArray(facts.messages).find((message) => message.traceId === traceId)?.sessionId, + factCheckpointForTrace(facts, traceId)?.sessionId + ]); + const alreadyLoaded = visible.find((item) => relatedSessionIds.includes(factSessionId(item))) ?? null; + if (alreadyLoaded) return { session: alreadyLoaded, facts }; + for (const sessionId of relatedSessionIds) { + const result = await queryFactsByRouteId(readModel, sessionId, { families: ["sessions"], limit: 1 }); + if (result.error) return { error: result.error }; + const session = visibleFactSessions(result.facts, actor)[0] ?? null; + if (session) return { session, facts: mergeFactSets(facts, result.facts) }; + } + return { session: null, facts }; +} + +export function factSessionMatchesRouteId(session, routeId) { + const sessionId = safeSessionId(routeId); + if (sessionId && factSessionId(session) === sessionId) return true; + const conversationId = safeConversationId(routeId); + return Boolean(conversationId && session?.conversationId === conversationId); +} + + +export async function handleWorkbenchSessionDetail(request, response, url, options, actor, sessionId) { + const readModel = workbenchRuntimeFactsReadModel(request, options, actor); + const includeMessages = includeMessagesForSessionDetail(url.searchParams); + const result = await queryFactsByRouteId(readModel, sessionId, { families: WORKBENCH_SESSION_DETAIL_FAMILIES }); + if (result.error) return sendJson(response, 503, workbenchProjectionStoreError(result.error)); + const session = visibleFactSessions(result.facts, actor)[0] ?? null; + if (!session) { + const body = workbenchError("workbench_session_not_found", "Workbench session is not visible to the current actor.", { sessionId }); + attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/sessions/:id", code: "workbench_session_not_found", sessionId, count: result.count }); + return sendJson(response, 404, body); + } + sendJson(response, 200, { + ok: true, + status: "found", + contractVersion: "workbench-session-detail-v1", + session: factSessionDetail(session, result.facts, { includeMessages }), + detailMode: includeMessages ? "full" : "metadata-only", + messagesIncluded: includeMessages, + valuesRedacted: true, + secretMaterialStored: false + }); +} + +export async function handleWorkbenchMessagePage(request, response, url, options, actor, sessionId) { + const startedAt = nowMs(); + const readModel = workbenchRuntimeFactsReadModel(request, options, actor); + const limit = boundedLimit(url.searchParams.get("limit")); + const cursorParam = url.searchParams.get("cursor") ?? url.searchParams.get("after"); + const hasCursor = Boolean(textValue(cursorParam)); + const result = await queryFactsByRouteId(readModel, sessionId, { + families: WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES, + ...(hasCursor ? {} : { limit: limit + 1, messagesOrder: "updated_desc", partsOrder: "updated_desc", turnsOrder: "updated_desc", checkpointsOrder: "updated_desc" }) + }); + if (result.error) return sendJson(response, 503, workbenchProjectionStoreError(result.error)); + const session = visibleFactSessions(result.facts, actor)[0] ?? null; + if (!session) { + const body = workbenchError("workbench_session_not_found", "Workbench session is not visible to the current actor.", { sessionId }); + attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/sessions/:id/messages", code: "workbench_session_not_found", sessionId, count: result.count }); + return sendJson(response, 404, body); + } + const messages = factMessagesForSession(session, result.facts); + const offset = hasCursor ? cursorOffset(cursorParam) : 0; + const page = messages.slice(offset, offset + limit); + const nextOffset = offset + page.length; + const resolvedSessionId = factSessionId(session); + const body = { + ok: true, + status: "succeeded", + contractVersion: "workbench-message-page-v1", + sessionId: resolvedSessionId, + messages: page, + count: page.length, + total: messages.length, + cursor: offset > 0 ? cursorFromOffset(offset) : null, + nextCursor: nextOffset < messages.length ? cursorFromOffset(nextOffset) : null, + hasMore: nextOffset < messages.length, + timelineDigest: workbenchTimelineDigest(page), + roleSequencePrefix: messagesRoleSequence(page).slice(0, 32), + adjacentSameRoleCount: adjacentSameRoleCount(page), + traceIds: uniqueText(page.map((message) => message?.traceId)).slice(0, 16), + valuesRedacted: true, + secretMaterialStored: false + }; + emitWorkbenchSessionMessagesReadOtel(request, options, { + startTimeMs: startedAt, + statusCode: 200, + sessionId: resolvedSessionId, + limit, + offset, + messages: page, + total: messages.length, + hasMore: body.hasMore + }); + sendJson(response, 200, body); +} + +export async function handleWorkbenchTurnSnapshot(request, response, url, options, actor, rawTurnId) { + const startedAt = nowMs(); + const queryTraceId = safeTraceId(url.searchParams.get("traceId")); + const traceId = safeTraceId(rawTurnId) ?? queryTraceId; + const turnId = safeTurnId(rawTurnId) || traceId || rawTurnId; + if (!traceId) { + const body = workbenchError("invalid_turn_id", "turnId must currently be a trace id or include traceId query.", { turnId }); + recordWorkbenchTurnReadMetric(options, url, 400, body, startedAt); + return sendJson(response, 400, body); + } + const readModel = workbenchRuntimeFactsReadModel(request, options, actor); + const result = await readModel.queryFacts({ traceId, limit: MAX_PAGE_LIMIT, families: [...WORKBENCH_TRACE_EVENT_METADATA_FAMILIES, ...WORKBENCH_TRACE_EVENT_PAGE_FAMILIES] }); + if (result.error) { + const body = workbenchProjectionStoreError(result.error); + emitWorkbenchTurnStatusReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 503, + phase: "query_failed", + turnId, + errorCode: result.error?.code ?? "workbench_turn_query_failed" + }, result.error); + recordWorkbenchTurnReadMetric(options, url, 503, body, startedAt); + return sendJson(response, 503, body); + } + const session = visibleFactSessions(result.facts, actor).find((item) => item.lastTraceId === traceId) ?? visibleFactSessions(result.facts, actor)[0] ?? null; + const turn = factTurnForTrace(result.facts, traceId, turnId); + const found = Boolean(session || turn); + if (!found) { + const body = workbenchError("workbench_turn_not_found", "Workbench turn is not visible to the current actor.", { turnId, traceId }); + attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/turns/:id", code: "workbench_turn_not_found", traceId, turnId, count: result.count }); + emitWorkbenchTurnStatusReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 404, + phase: "not_found", + turnId, + readCount: result.count, + errorCode: "workbench_turn_not_found" + }); + recordWorkbenchTurnReadMetric(options, url, 404, body, startedAt); + return sendJson(response, 404, body); + } + const projection = factProjectionForTrace(result.facts, traceId); + const snapshot = factTurnSnapshot({ turn, session, facts: result.facts, traceId, turnId }); + const status = snapshot.status; + const body = { + ok: true, + status, + contractVersion: "workbench-turn-snapshot-v1", + turn: snapshot, + projection, + cache: result.cache ?? null, + projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, + lastProjectedSeq: projection.lastProjectedSeq, + sourceRunId: projection.sourceRunId, + sourceCommandId: projection.sourceCommandId, + staleMs: projection.staleMs, + blocker: projection.blocker, + valuesRedacted: true, + secretMaterialStored: false + }; + recordWorkbenchProjectionMetric(options, { + projection: { ...projection, status, eventCount: snapshot.trace?.eventCount ?? projection.lastProjectedSeq ?? 0 }, + diagnostic: projection + }); + emitWorkbenchTurnStatusReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 200, + phase: "succeeded", + status, + terminal: snapshot.terminal, + sessionId: snapshot.sessionId, + turnId: snapshot.turnId, + traceEventCount: snapshot.trace?.eventCount ?? null, + latestProjectedSeq: projection.lastProjectedSeq, + projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, + sourceRunId: projection.sourceRunId, + sourceCommandId: projection.sourceCommandId, + readCount: result.count + }); + recordWorkbenchTurnReadMetric(options, url, 200, body, startedAt); + recordWorkbenchCacheMetric(options, url, body, startedAt); + sendJson(response, 200, body); +} + +export async function handleWorkbenchTraceEventPage(request, response, url, options, actor, rawTraceId) { + const startedAt = nowMs(); + const traceId = safeTraceId(rawTraceId); + if (!traceId) return sendJson(response, 400, workbenchError("invalid_trace_id", "traceId must start with trc_.", { traceId: rawTraceId })); + const readModel = workbenchRuntimeFactsReadModel(request, options, actor); + const pageOptions = tracePageOptions(url); + const metadata = await readModel.queryFacts({ traceId, families: WORKBENCH_TRACE_EVENT_METADATA_FAMILIES, limit: 1 }); + if (metadata.error) { + emitWorkbenchTraceEventsReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 503, + phase: "metadata_query_failed", + pageOptions, + errorCode: metadata.error?.code ?? "workbench_trace_metadata_query_failed" + }, metadata.error); + return sendJson(response, 503, workbenchProjectionStoreError(metadata.error)); + } + const resolved = await visibleFactSessionForTrace(readModel, metadata.facts, actor, traceId); + if (resolved.error) { + emitWorkbenchTraceEventsReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 503, + phase: "session_lookup_failed", + pageOptions, + metadataCount: metadata.count, + errorCode: resolved.error?.code ?? "workbench_trace_session_lookup_failed" + }, resolved.error); + return sendJson(response, 503, workbenchProjectionStoreError(resolved.error)); + } + const session = resolved.session; + const metadataFacts = resolved.facts ?? metadata.facts; + if (!session) { + const context = await readModel.queryFacts({ traceId, families: ["sessions", "turns", "checkpoints"], limit: MAX_PAGE_LIMIT }); + if (context.error) { + emitWorkbenchTraceEventsReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 503, + phase: "context_query_failed", + pageOptions, + metadataCount: metadata.count, + errorCode: context.error?.code ?? "workbench_trace_context_query_failed" + }, context.error); + return sendJson(response, 503, workbenchProjectionStoreError(context.error)); + } + const contextSessions = visibleFactSessions(context.facts, actor); + const contextSession = contextSessions.find((item) => item.lastTraceId === traceId) ?? contextSessions[0] ?? null; + const contextTurn = factTurnForTrace(context.facts, traceId, traceId); + if (contextSession || contextTurn) { + const projection = factProjectionForTrace(context.facts, traceId); + const blocker = traceEventsReadModelBlocker("workbench_trace_metadata_missing", "Workbench trace metadata is missing from the trace events read model.", { traceId, projection, session: contextSession, turn: contextTurn, route: url.pathname }); + attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/traces/:id/events", code: "workbench_trace_metadata_missing", sessionId: factSessionId(contextSession), traceId, count: context.count }); + emitWorkbenchTraceEventsReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 404, + phase: "metadata_missing", + pageOptions, + projection, + sessionId: factSessionId(contextSession), + turnId: factTurnId(contextTurn, traceId), + readCount: context.count, + metadataCount: metadata.count, + errorCode: "workbench_trace_metadata_missing" + }); + return sendJson(response, 404, workbenchTraceEventsReadModelError(blocker, { traceId, projection, session: contextSession, turn: contextTurn })); + } + attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/traces/:id/events", code: "workbench_trace_not_found", traceId, count: context.count }); + emitWorkbenchTraceEventsReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 404, + phase: "not_found", + pageOptions, + readCount: context.count, + metadataCount: metadata.count, + errorCode: "workbench_trace_not_found" + }); + return sendJson(response, 404, workbenchError("workbench_trace_not_found", "Workbench trace is not visible to the current actor.", { traceId })); + } + const projection = factProjectionForTrace(metadataFacts, traceId); + const pageResult = await readModel.queryFacts({ traceId, families: WORKBENCH_TRACE_EVENT_PAGE_FAMILIES, afterProjectedSeq: pageOptions.afterProjectedSeq, limit: pageOptions.limit + 1 }); + if (pageResult.error) { + emitWorkbenchTraceEventsReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 503, + phase: "events_query_failed", + pageOptions, + projection, + sessionId: factSessionId(session), + metadataCount: metadata.count, + errorCode: pageResult.error?.code ?? "workbench_trace_events_query_failed" + }, pageResult.error); + return sendJson(response, 503, workbenchProjectionStoreError(pageResult.error)); + } + const traceTurn = factTurnForTrace(metadataFacts, traceId, traceId); + const turnTraceStatus = normalizeStatus(traceTurn?.status); + const traceStatus = turnTraceStatus !== "unknown" ? turnTraceStatus : durableTraceStatus(pageResult.facts.traceEvents); + const page = traceEventPageFromFacts(pageResult.facts.traceEvents, pageOptions, { total: projection.lastProjectedSeq, traceLastSeq: projection.lastProjectedSeq, traceStatus }); + const missingTraceEvents = traceEventPageMissing(page, projection, pageOptions); + const missingTraceEventsDiagnostic = missingTraceEvents + ? traceEventsReadModelBlocker("workbench_trace_events_missing", "Workbench trace events are still catching up in the durable read model.", { traceId, projection, session, turn: traceTurn, route: url.pathname }) + : null; + if (missingTraceEvents) { + attachWorkbenchReadModelDiagnosticOtel(request, { route: "/v1/workbench/traces/:id/events", code: "workbench_trace_events_missing", sessionId: factSessionId(session), traceId, count: pageResult.count }); + } + const responseProjection = page.blocker ? blockedTraceEventProjection(projection, page.blocker) : missingTraceEvents ? catchingUpTraceEventProjection(projection, missingTraceEventsDiagnostic) : projection; + const body = { + ok: true, + status: page.blocker ? "blocked" : missingTraceEvents ? "projecting" : "succeeded", + contractVersion: "workbench-trace-events-v1", + traceId, + sessionId: factSessionId(session), + threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), + ...page, + events: factArray(page.events).map((event) => ({ ...event, detailProjection: true, authority: "trace-detail-only" })), + projection: responseProjection, + detailProjection: true, + authority: "trace-detail-only", + realtimeAuthority: "workbench-realtime-authority-v2", + cache: pageResult.cache ?? metadata.cache ?? null, + projectionStatus: responseProjection.projectionStatus, + projectionHealth: responseProjection.projectionHealth, + lastProjectedSeq: responseProjection.lastProjectedSeq, + sourceRunId: responseProjection.sourceRunId, + sourceCommandId: responseProjection.sourceCommandId, + staleMs: responseProjection.staleMs, + blocker: responseProjection.blocker, + timing: responseProjection.timing, + startedAt: responseProjection.startedAt, + lastEventAt: responseProjection.lastEventAt, + finishedAt: responseProjection.finishedAt, + durationMs: responseProjection.durationMs, + diagnostic: responseProjection.diagnostic ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; + recordWorkbenchCacheMetric(options, url, body, startedAt); + emitWorkbenchTraceEventsReadOtel(traceId, options, { + startTimeMs: startedAt, + statusCode: 200, + phase: page.blocker ? "blocked" : missingTraceEvents ? "events_catching_up" : "succeeded", + pageOptions, + page, + projection: responseProjection, + sessionId: body.sessionId, + turnId: factTurnId(traceTurn, traceId), + readCount: pageResult.count, + rawEventCount: factArray(pageResult.facts?.traceEvents).length, + metadataCount: metadata.count, + traceStatus + }); + sendJson(response, 200, body); +} + +export function emitWorkbenchTurnStatusReadOtel(traceId, options = {}, fields = {}, error = null) { + const safeId = safeTraceId(traceId); + if (!safeId) return; + const statusCode = Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : 0; + void emitCodeAgentOtelSpan("turn_status_read", safeId, options.env ?? process.env, { + startTimeMs: fields.startTimeMs, + status: statusCode >= 500 || error ? "error" : "ok", + error, + attributes: { + "http.method": "GET", + "http.route": "/v1/workbench/turns/:id", + "http.status_code": statusCode || null, + phase: fields.phase ?? null, + status: fields.status ?? null, + terminal: typeof fields.terminal === "boolean" ? fields.terminal : null, + sessionId: fields.sessionId ?? null, + turnId: fields.turnId ?? safeId, + readCount: nonNegativeInteger(fields.readCount), + traceEventCount: nonNegativeInteger(fields.traceEventCount), + latestProjectedSeq: nonNegativeInteger(fields.latestProjectedSeq), + projectionStatus: fields.projectionStatus ?? null, + projectionHealth: fields.projectionHealth ?? null, + sourceRunId: fields.sourceRunId ?? null, + sourceCommandId: fields.sourceCommandId ?? null, + errorCode: fields.errorCode ?? null, + valuesRedacted: true + } + }); +} + +export function emitWorkbenchTraceEventsReadOtel(traceId, options = {}, fields = {}, error = null) { + const safeId = safeTraceId(traceId); + if (!safeId) return; + const page = fields.page ?? {}; + const range = page.range ?? {}; + const projection = fields.projection ?? {}; + const pageOptions = fields.pageOptions ?? {}; + const statusCode = Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : 0; + const returnedEvents = Array.isArray(page.events) ? page.events.length : nonNegativeInteger(fields.returnedEvents); + const afterProjectedSeq = nonNegativeInteger(pageOptions.afterProjectedSeq ?? range.afterProjectedSeq); + const limit = nonNegativeInteger(pageOptions.limit ?? range.limit); + const hasMore = typeof page.hasMore === "boolean" ? page.hasMore : null; + void emitCodeAgentOtelSpan("trace_events_read", safeId, options.env ?? process.env, { + startTimeMs: fields.startTimeMs, + status: statusCode >= 500 || error ? "error" : "ok", + error, + attributes: { + "http.method": "GET", + "http.route": "/v1/workbench/traces/:id/events", + "http.status_code": statusCode || null, + phase: fields.phase ?? null, + sessionId: fields.sessionId ?? null, + turnId: fields.turnId ?? safeId, + returnedEvents, + rawEventCount: nonNegativeInteger(fields.rawEventCount), + readCount: nonNegativeInteger(fields.readCount), + metadataCount: nonNegativeInteger(fields.metadataCount), + sinceSeq: afterProjectedSeq, + afterProjectedSeq, + limit, + fromSeq: nonNegativeInteger(range.fromProjectedSeq), + toSeq: nonNegativeInteger(range.toProjectedSeq), + totalEvents: nonNegativeInteger(range.total ?? page.eventCount), + hasMore, + fullTraceLoaded: typeof page.fullTraceLoaded === "boolean" ? page.fullTraceLoaded : hasMore === null ? null : !hasMore && !page.blocker, + traceLastSeq: nonNegativeInteger(page.traceLastSeq ?? projection.lastProjectedSeq), + projectionStatus: projection.projectionStatus ?? null, + projectionHealth: projection.projectionHealth ?? null, + sourceRunId: projection.sourceRunId ?? null, + sourceCommandId: projection.sourceCommandId ?? null, + traceStatus: fields.traceStatus ?? page.traceStatus ?? null, + errorCode: fields.errorCode ?? null, + valuesRedacted: true + } + }); +} + +export function emitWorkbenchSessionListReadOtel(request, options = {}, fields = {}, error = null) { + const payload = fields.payload && typeof fields.payload === "object" ? fields.payload : {}; + const sessions = Array.isArray(payload.sessions) ? payload.sessions : []; + const fallbackTitleCount = sessions.filter(sessionSummaryNeedsTitleFallback).length; + const traceIds = uniqueText(sessions.map((session) => session?.lastTraceId)).slice(0, 8); + const businessTraceId = (safeTraceId(traceIds[0]) ?? textValue(fields.includeSessionId)) || "workbench_session_list"; + void emitCodeAgentOtelSpan("session_list_read", businessTraceId, options.env ?? process.env, { + startTimeMs: fields.startTimeMs, + status: error ? "error" : "ok", + error, + attributes: { + "http.method": "GET", + "http.route": "/v1/workbench/sessions", + "http.status_code": Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : null, + source: fields.source ?? null, + limit: nonNegativeInteger(fields.limit), + offset: nonNegativeInteger(fields.offset), + includeSessionId: fields.includeSessionId ?? null, + sessionCount: sessions.length, + fallbackTitleCount, + fallbackTitleRatio: sessions.length > 0 ? Number((fallbackTitleCount / sessions.length).toFixed(4)) : 0, + emptyPreviewCount: sessions.filter((session) => !sessionPreviewText(session)).length, + sessionIds: uniqueText(sessions.map((session) => session?.sessionId)).slice(0, 8).join(","), + traceIds: traceIds.join(","), + hasMore: payload.hasMore === true, + valuesRedacted: true + } + }); +} + +export function emitWorkbenchSessionMessagesReadOtel(request, options = {}, fields = {}, error = null) { + const messages = Array.isArray(fields.messages) ? fields.messages : []; + const traceIds = uniqueText(messages.map((message) => message?.traceId)).slice(0, 8); + const businessTraceId = (safeTraceId(traceIds[0]) ?? textValue(fields.sessionId)) || "workbench_session_messages"; + const roleSequence = messagesRoleSequence(messages); + void emitCodeAgentOtelSpan("session_messages_read", businessTraceId, options.env ?? process.env, { + startTimeMs: fields.startTimeMs, + status: error ? "error" : "ok", + error, + attributes: { + "http.method": "GET", + "http.route": "/v1/workbench/sessions/:id/messages", + "http.status_code": Number.isInteger(Number(fields.statusCode)) ? Number(fields.statusCode) : null, + sessionId: fields.sessionId ?? null, + limit: nonNegativeInteger(fields.limit), + offset: nonNegativeInteger(fields.offset), + returnedMessages: messages.length, + totalMessages: nonNegativeInteger(fields.total), + hasMore: fields.hasMore === true, + roleSequencePrefix: roleSequence.slice(0, 32), + consecutiveUserPrefix: consecutiveRolePrefix(messages, "user"), + adjacentSameRoleCount: adjacentSameRoleCount(messages), + timelineDigest: workbenchTimelineDigest(messages), + userCount: messages.filter((message) => normalizeRole(message?.role) === "user").length, + agentCount: messages.filter((message) => isAssistantLikeRole(message?.role)).length, + traceIds: traceIds.join(","), + valuesRedacted: true + } + }); +} + +export function messagesRoleSequence(messages = []) { + return messages.map((message) => roleSequenceSymbol(message?.role)).join(""); +} + +export function workbenchTimelineDigest(messages = []) { + const payload = messages.map((message) => { + const terminal = isTerminalProjectionStatus(normalizeStatus(message?.status)) || Boolean(message?.finishedAt); + return { + messageId: message?.messageId ?? null, + role: normalizeRole(message?.role), + sessionId: message?.sessionId ?? null, + traceId: message?.traceId ?? null, + turnId: message?.turnId ?? null, + status: normalizeStatus(message?.status), + partIds: factArray(message?.parts).map((part) => part?.partId ?? null), + textHash: textValue(message?.text || message?.textPreview) ? hash(textValue(message?.text || message?.textPreview)).slice(0, 16) : null, + startedAt: message?.startedAt ?? null, + finishedAt: terminal ? message?.finishedAt ?? null : null, + durationMs: terminal && Number.isFinite(Number(message?.durationMs)) ? Math.trunc(Number(message.durationMs)) : null + }; + }); + return `sha256:${hash(JSON.stringify(payload))}`; +} + +export function sessionSummaryNeedsTitleFallback(session) { + if (!session || typeof session !== "object") return true; + return !sessionPreviewText(session); +} + +export function sessionPreviewText(session) { + return textValue( + session?.firstUserMessagePreview + ?? session?.snapshot?.firstUserMessagePreview + ?? session?.userPreview + ?? session?.title + ?? session?.name + ); +} + +export function roleSequenceSymbol(role) { + const normalized = normalizeRole(role); + if (normalized === "user") return "U"; + if (normalized === "agent" || normalized === "assistant") return "A"; + if (normalized === "system") return "S"; + return "?"; +} + +export function normalizeRole(role) { + return textValue(role).toLowerCase(); +} + +export function consecutiveRolePrefix(messages = [], role) { + let count = 0; + for (const message of messages) { + if (normalizeRole(message?.role) !== role) break; + count += 1; + } + return count; +} + +export function adjacentSameRoleCount(messages = []) { + let count = 0; + for (let index = 1; index < messages.length; index += 1) { + const previous = normalizeRole(messages[index - 1]?.role); + if (previous && previous === normalizeRole(messages[index]?.role)) count += 1; + } + return count; +} + +export function traceEventPageFromFacts(sourceEvents, options, metadata = {}) { + const rows = factArray(sourceEvents).filter((event) => factProjectedSeq(event)).sort(compareFactTraceEventsAsc); + const collision = projectedSeqCollision(rows); + if (collision) return blockedTraceEventPage(options, metadata, collision); + const pageRows = rows.slice(0, options.limit); + const events = pageRows.map(factTraceEventDto).filter(Boolean); + const toProjectedSeq = events.length ? events.at(-1).projectedSeq : null; + const hasMore = rows.length > options.limit; + const total = Number.isFinite(Number(metadata.total)) ? Math.trunc(Number(metadata.total)) : Math.max(options.afterProjectedSeq, toProjectedSeq ?? options.afterProjectedSeq); + const traceLastSeq = Number.isFinite(Number(metadata.traceLastSeq)) ? Math.trunc(Number(metadata.traceLastSeq)) : total; + return { + events, + eventCount: total ?? Math.max(options.afterProjectedSeq, toProjectedSeq ?? options.afterProjectedSeq), + range: { + afterProjectedSeq: options.afterProjectedSeq, + fromProjectedSeq: events.length ? events[0].projectedSeq : null, + toProjectedSeq, + limit: options.limit, + returned: events.length, + total + }, + hasMore, + fullTraceLoaded: !hasMore && (toProjectedSeq ?? options.afterProjectedSeq) >= traceLastSeq, + traceLastSeq, + nextProjectedSeq: events.length ? toProjectedSeq : options.afterProjectedSeq, + nextCursor: hasMore && toProjectedSeq ? `projected:${toProjectedSeq}` : null, + traceStatus: metadata.traceStatus ?? "unknown", + updatedAt: events.at(-1)?.updatedAt ?? events.at(-1)?.createdAt ?? null + }; +} + +export function projectedSeqCollision(rows = []) { + const seen = new Map(); + for (const row of rows) { + const projectedSeq = factProjectedSeq(row); + if (!projectedSeq) continue; + const id = textValue(row?.id ?? row?.sourceEventId) || null; + const existing = seen.get(projectedSeq); + if (existing && existing.id !== id) { + return { + code: "projected_seq_collision", + category: "workbench-projection", + layer: "workbench-trace-events", + message: "Trace projection contains duplicate projectedSeq values; canonical event order is blocked until projection is rebuilt.", + projectedSeq, + eventIds: [existing.id, id].filter(Boolean), + retryable: false, + valuesPrinted: false + }; + } + if (!existing) seen.set(projectedSeq, { id }); + } + return null; +} + +export function blockedTraceEventPage(options, metadata = {}, blocker) { + const total = Number.isFinite(Number(metadata.total)) ? Math.trunc(Number(metadata.total)) : null; + return { + events: [], + eventCount: total ?? options.afterProjectedSeq, + range: { + afterProjectedSeq: options.afterProjectedSeq, + fromProjectedSeq: null, + toProjectedSeq: null, + limit: options.limit, + returned: 0, + total + }, + hasMore: false, + nextProjectedSeq: options.afterProjectedSeq, + nextCursor: null, + traceStatus: metadata.traceStatus ?? "unknown", + updatedAt: null, + blocker + }; +} + +export function blockedTraceEventProjection(projection = {}, blocker) { + return { + ...projection, + projectionStatus: "blocked", + projectionHealth: "degraded", + blocker, + valuesRedacted: true + }; +} + +export function catchingUpTraceEventProjection(projection = {}, diagnostic = null) { + const status = projection.projectionStatus === "caught-up" || !projection.projectionStatus ? "projecting" : projection.projectionStatus; + const health = projection.projectionHealth === "caught-up" || projection.projectionHealth === "healthy" || !projection.projectionHealth ? "projecting" : projection.projectionHealth; + return { + ...projection, + projectionStatus: status, + projectionHealth: health, + blocker: null, + diagnostic: diagnostic ? { ...diagnostic, projectionStatus: status, projectionHealth: health } : null, + valuesRedacted: true + }; +} + +export function traceEventPageMissing(page = {}, projection = {}, options = {}) { + if (page.blocker) return false; + const expectedSeq = Number(projection?.lastProjectedSeq); + if (!Number.isFinite(expectedSeq) || expectedSeq <= 0) return false; + return factArray(page.events).length === 0 && expectedSeq > Number(options.afterProjectedSeq ?? 0); +} + +export function traceEventsReadModelBlocker(code, message, { traceId, projection = {}, session = null, turn = null, route = null } = {}) { + return { + code, + message, + userMessage: message, + layer: "workbench-read-model", + category: "trace-events", + retryable: true, + route, + traceId, + sessionId: factSessionId(session) ?? turn?.sessionId ?? null, + turnId: factTurnId(turn, traceId), + projectionStatus: projection.projectionStatus ?? null, + projectionHealth: "degraded", + lastProjectedSeq: projection.lastProjectedSeq ?? null, + sourceRunId: projection.sourceRunId ?? null, + sourceCommandId: projection.sourceCommandId ?? null, + retryAfterMs: 5000, + valuesPrinted: false + }; +} + +export function workbenchTraceEventsReadModelError(blocker, { traceId, projection = {}, session = null, turn = null } = {}) { + const responseProjection = blockedTraceEventProjection(projection, blocker); + return workbenchError(blocker.code, blocker.message, { + traceId, + sessionId: blocker.sessionId ?? factSessionId(session) ?? turn?.sessionId ?? null, + turnId: blocker.turnId ?? factTurnId(turn, traceId), + route: blocker.route ?? null, + layer: blocker.layer, + category: blocker.category, + retryable: blocker.retryable, + projection: responseProjection, + projectionStatus: responseProjection.projectionStatus, + projectionHealth: responseProjection.projectionHealth, + lastProjectedSeq: responseProjection.lastProjectedSeq, + sourceRunId: responseProjection.sourceRunId, + sourceCommandId: responseProjection.sourceCommandId, + blocker, + diagnostic: { + contractVersion: "hwlab-error-diagnostic-v1", + route: blocker.route ?? null, + layer: blocker.layer, + category: blocker.category, + code: blocker.code, + httpStatus: 404, + source: "server", + retryable: blocker.retryable, + valuesPrinted: false + } + }); +} + +export function projectionText(...values) { + for (const value of values) { + if (value && typeof value === "object") { + const nested = messageAuthorityTextValue(value.text ?? value.content ?? value.message ?? value.summary ?? value.preview ?? value.title); + if (nested) return nested; + continue; + } + const text = messageAuthorityTextValue(value); + if (text) return text; + } + return null; +} + +export function partFact(part, index, messageId, traceId) { + const type = textValue(part?.type) || "text"; + const text = messageAuthorityTextValue(part?.text ?? part?.content ?? part?.message); + return { + partId: safePartId(part?.partId ?? part?.id) || `prt_${hash(`${messageId}:${index}:${type}`).slice(0, 24)}`, + messageId, + traceId, + type, + text: text || null, + status: normalizeStatus(part?.status ?? "completed"), + toolName: textValue(part?.toolName ?? part?.name) || null, + createdAt: textValue(part?.createdAt ?? part?.timestamp) || null, + valuesRedacted: part?.valuesRedacted !== false + }; +} + +export function tracePageOptions(url) { + const cursor = textValue(url.searchParams.get("cursor")); + const cursorProjectedSeq = cursor.startsWith("projected:") ? Number.parseInt(cursor.slice("projected:".length), 10) : NaN; + const afterProjectedSeq = Number.isInteger(cursorProjectedSeq) && cursorProjectedSeq >= 0 + ? cursorProjectedSeq + : nonNegativeInteger(url.searchParams.get("afterProjectedSeq"), 0); + return { afterProjectedSeq, limit: boundedLimit(url.searchParams.get("limit")) }; +} + + +export function handleWorkbenchReadModelFailure(response, url, options = {}, error) { + const classified = classifyWorkbenchReadModelFailure(error); + const route = workbenchReadRouteTemplate(url?.pathname ?? "/v1/workbench"); + logWorkbenchReadModelFailure(options.logger ?? console, { + event: "workbench_read_model_route_failed", + ok: false, + route, + statusCode: classified.statusCode, + errorCode: classified.code, + errorName: error?.name ?? "Error", + message: error instanceof Error ? error.message : String(error ?? "unknown"), + valuesRedacted: true + }); + if (response.headersSent || response.destroyed) return; + sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { + route, + errorName: error?.name ?? "Error", + retryable: classified.retryable === true, + transient: classified.transient === true, + serviceId: classified.serviceId ?? null, + runtimeStatus: classified.runtimeStatus ?? null + })); +} + +export function classifyWorkbenchReadModelFailure(error) { + const message = String(error?.message ?? error ?? ""); + const code = String(error?.code ?? "").toLowerCase(); + const runtimeStatus = Number(error?.data?.status ?? 0); + const runtimePayloadError = error?.data?.payload?.error ?? null; + if (error?.name === "WorkbenchRuntimeDependencyError" || code.startsWith("workbench_runtime_")) { + const retryable = error?.data?.retryable === true || error?.data?.transient === true || runtimePayloadError?.retryable === true; + return { + statusCode: retryable || (Number.isFinite(runtimeStatus) && runtimeStatus >= 500) ? 503 : 500, + code: String(runtimePayloadError?.code ?? error?.code ?? "workbench_runtime_dependency_failed"), + message: String(runtimePayloadError?.message ?? error?.message ?? "Workbench runtime dependency failed."), + retryable, + transient: retryable, + serviceId: String(error?.data?.serviceId ?? "hwlab-workbench-runtime"), + runtimeStatus: Number.isFinite(runtimeStatus) && runtimeStatus > 0 ? runtimeStatus : null + }; + } + if (/timeout|connection terminated|terminating connection|econnreset|econnrefused|etimedout/iu.test(message) || /timeout|econnreset|econnrefused|etimedout/u.test(code)) { + return { + statusCode: 503, + code: "workbench_read_model_store_unavailable", + message: "Workbench read model store is temporarily unavailable." + }; + } + return { + statusCode: 500, + code: "workbench_read_model_failed", + message: "Workbench read model failed." + }; +} + +export function workbenchReadRouteTemplate(pathname) { + const path = String(pathname ?? ""); + if (path === "/v1/workbench/sessions") return path; + if (/^\/v1\/workbench\/sessions\/[^/]+\/messages$/u.test(path)) return "/v1/workbench/sessions/:id/messages"; + if (/^\/v1\/workbench\/sessions\/[^/]+$/u.test(path)) return "/v1/workbench/sessions/:id"; + if (/^\/v1\/workbench\/turns\/[^/]+$/u.test(path)) return "/v1/workbench/turns/:id"; + if (/^\/v1\/workbench\/traces\/[^/]+\/events$/u.test(path)) return "/v1/workbench/traces/:id/events"; + return path || "/v1/workbench"; +} + +export function logWorkbenchReadModelFailure(logger, payload) { + try { + const line = JSON.stringify(payload); + if (typeof logger?.error === "function") logger.error(line); + else if (typeof logger?.warn === "function") logger.warn(line); + } catch { + // Read-model failure logging must not affect response generation. + } +} + +export function methodNotAllowed(response, allowed) { + sendJson(response, 405, workbenchError("method_not_allowed", `Use ${allowed} for this Workbench read model route.`)); +} + +export function workbenchError(code, message, extra = {}) { + return { + ok: false, + status: "failed", + error: { code, message, ...extra }, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +export function boundedLimit(value) { + return Math.min(MAX_PAGE_LIMIT, parsePositiveInteger(value, DEFAULT_PAGE_LIMIT)); +} + +export function boundedSessionListLimit(value) { + return Math.min(MAX_PAGE_LIMIT, parsePositiveInteger(value, DEFAULT_SESSION_LIST_LIMIT)); +} + +export function cursorOffset(value) { + const text = textValue(value); + if (!text) return 0; + if (text.startsWith("idx:")) return nonNegativeInteger(text.slice(4), 0); + return nonNegativeInteger(text, 0); +} + +export function cursorFromOffset(offset) { + return `idx:${Math.max(0, Math.trunc(Number(offset) || 0))}`; +} + +export function nonNegativeInteger(value, fallback) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} + + +export function nowMs() { + if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); + return Date.now(); +} diff --git a/internal/cloud/server-workbench-read-state.test.ts b/internal/cloud/server-workbench-read-state.test.ts new file mode 100644 index 00000000..b0dde3d5 --- /dev/null +++ b/internal/cloud/server-workbench-read-state.test.ts @@ -0,0 +1,800 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery. +// Responsibility: Workbench durable state and timeline regression tests. + +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { createCloudApiServer } from "./server.ts"; +import { createBackendPerformanceStore } from "./backend-performance.ts"; +import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts"; +import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; +import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts"; + +import { + ACTOR, + getJson, + waitForCondition, + createDurableFactsRuntimeStore, + createRuntimeStoreFromFacts, + buildDurableFactsForSession, + normalizeTestMessages, + testTimingProjection, + timestampIso, + elapsedMs, + normalizeTestEvents, + filterFacts, + workbenchTestFactOrder, + matchesFact, + matchesTraceFact, + testFactFamilySet, + mergeFacts, + emptyFacts, + normalizeTestStatus, + getSseEvents, + parseSseBlock, + createFakeKafkaFactory, + waitFor +} from "./server-workbench-http-test-helpers.ts"; + +test("workbench trace events reports catch-up page when session projection is visible", async () => { + const traceStore = createCodeAgentTraceStore(); + const traceId = "trc_workbench_trace_events_gap"; + const session = { + id: "ses_workbench_trace_events_gap", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "completed", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_trace_events_gap", + threadId: "thread-workbench-trace-events-gap", + lastTraceId: traceId, + updatedAt: "2026-06-20T13:05:00.000Z", + session: { sessionStatus: "completed", lastTraceId: traceId, messages: [{ role: "user", text: "events gap", traceId }, { role: "agent", text: "OK", traceId }] } + }; + const accessController = { + store: {}, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ session, events: [], status: "completed", finalText: "OK", runId: "run_workbench_trace_events_gap", commandId: "cmd_workbench_trace_events_gap", lastProjectedSeq: 2 }] + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); + assert.equal(turn.status, 200); + assert.equal(turn.body.turn.status, "completed"); + + const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); + assert.equal(trace.status, 200); + assert.equal(trace.body.status, "projecting"); + assert.equal(trace.body.diagnostic.code, "workbench_trace_events_missing"); + assert.equal(trace.body.sessionId, session.id); + assert.equal(trace.body.projectionStatus, "projecting"); + assert.equal(trace.body.projectionHealth, "projecting"); + assert.equal(trace.body.lastProjectedSeq, 2); + assert.equal(trace.body.sourceRunId, "run_workbench_trace_events_gap"); + assert.equal(trace.body.sourceCommandId, "cmd_workbench_trace_events_gap"); + assert.equal(trace.body.blocker, null); + assert.deepEqual(trace.body.events, []); + assert.equal(trace.body.fullTraceLoaded, false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model projects current turn running state from trace projection", async () => { + const traceStore = createCodeAgentTraceStore(); + const traceId = "trc_workbench_active_state"; + const session = { + id: "ses_workbench_active_state", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "active", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_active_state", + threadId: "thread-workbench-active-state", + lastTraceId: traceId, + updatedAt: "2026-06-17T00:01:00.000Z", + session: { sessionStatus: "active", lastTraceId: traceId, messages: [{ role: "user", text: "still running", traceId }] } + }; + traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false }); + const accessController = { + store: { + async listAgentSessionsForUser() { return [session]; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ + session, + events: [{ seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: session.updatedAt }], + status: "running", + lastProjectedSeq: 1 + }] + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const sessions = await getJson(port, "/v1/workbench/sessions"); + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].status, "running"); + assert.equal(sessions.body.sessions[0].running, true); + assert.equal(sessions.body.sessions[0].terminal, false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model projects current turn running state for idle session summary", async () => { + const traceStore = createCodeAgentTraceStore(); + const traceId = "trc_workbench_idle_with_running_trace"; + const session = { + id: "ses_workbench_idle_with_running_trace", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "idle", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_idle_with_running_trace", + threadId: "thread-workbench-idle-with-running-trace", + lastTraceId: traceId, + updatedAt: "2026-06-17T00:02:00.000Z", + session: { sessionStatus: "idle", lastTraceId: traceId, messages: [{ role: "user", text: "still running despite idle row", traceId, status: "sent" }] } + }; + traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false }); + const accessController = { + store: { + async listAgentSessionsForUser() { return [session]; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ + session, + events: [{ seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: session.updatedAt }], + status: "running", + lastProjectedSeq: 1 + }] + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].status, "running"); + assert.equal(sessions.body.sessions[0].running, true); + assert.equal(sessions.body.sessions[0].terminal, false); + assert.equal(sessions.body.sessions[0].turnSummary.status, "running"); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model projects newest running turn over stale completed session lastTraceId", async () => { + const oldTraceId = "trc_workbench_stale_completed_old"; + const newTraceId = "trc_workbench_new_running_current"; + const session = { + id: "ses_workbench_stale_completed_new_running", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "completed", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_stale_completed_new_running", + threadId: "thread-workbench-stale-completed-new-running", + lastTraceId: oldTraceId, + updatedAt: "2026-07-01T00:00:02.000Z", + session: { + sessionStatus: "completed", + lastTraceId: oldTraceId, + messages: [ + { messageId: "msg_workbench_old_user", role: "user", text: "first", traceId: oldTraceId, status: "sent", projectedSeq: 10 }, + { messageId: "msg_workbench_old_agent", role: "agent", text: "first done", traceId: oldTraceId, status: "completed", projectedSeq: 20 } + ] + } + }; + const facts = emptyFacts(); + mergeFacts(facts, buildDurableFactsForSession({ session, status: "completed", finalText: "first done", lastProjectedSeq: 20 })); + facts.messages.push( + { messageId: "msg_workbench_new_user", sessionId: session.id, turnId: newTraceId, traceId: newTraceId, role: "user", status: "sent", text: "second", projectedSeq: 30, sourceSeq: 30, sourceEventId: "evt_new_user", terminal: false, sealed: false, updatedAt: "2026-07-01T00:00:03.000Z", valuesRedacted: true }, + { messageId: "msg_workbench_new_agent", sessionId: session.id, turnId: newTraceId, traceId: newTraceId, role: "agent", status: "running", text: "", projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_agent", terminal: false, sealed: false, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true } + ); + facts.turns.push({ turnId: newTraceId, sessionId: session.id, traceId: newTraceId, messageId: "msg_workbench_new_agent", status: "running", projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_turn", terminal: false, sealed: false, diagnostic: { projectionStatus: "projecting", projectionHealth: "healthy", valuesRedacted: true }, timing: { startedAt: "2026-07-01T00:00:03.000Z", lastEventAt: "2026-07-01T00:00:04.000Z", finishedAt: null, durationMs: null, valuesRedacted: true }, startedAt: "2026-07-01T00:00:03.000Z", lastEventAt: "2026-07-01T00:00:04.000Z", finishedAt: null, durationMs: null, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true }); + facts.checkpoints.push({ traceId: newTraceId, sessionId: session.id, turnId: newTraceId, projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_checkpoint", projectionStatus: "projecting", projectionHealth: "healthy", terminal: false, sealed: false, diagnostic: { projectionStatus: "projecting", projectionHealth: "healthy", valuesRedacted: true }, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true }); + const runtimeStore = createRuntimeStoreFromFacts(facts); + const accessController = { + store: { async listAgentSessionsForUser() { return [session]; } }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const server = createCloudApiServer({ accessController, traceStore: createCodeAgentTraceStore(), runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].lastTraceId, newTraceId); + assert.equal(sessions.body.sessions[0].status, "running"); + assert.equal(sessions.body.sessions[0].running, true); + assert.equal(sessions.body.sessions[0].terminal, false); + assert.equal(sessions.body.sessions[0].turnSummary.traceId, newTraceId); + assert.equal(sessions.body.sessions[0].turnSummary.status, "running"); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model projects completed current turn for idle session summary", async () => { + const traceStore = createCodeAgentTraceStore(); + const results = createCodeAgentChatResultStore(); + const traceId = "trc_workbench_idle_with_completed_trace"; + const finalText = "OK"; + const session = { + id: "ses_workbench_idle_with_completed_trace", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "idle", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_idle_with_completed_trace", + threadId: "thread-workbench-idle-with-completed-trace", + lastTraceId: traceId, + updatedAt: "2026-06-17T00:03:00.000Z", + session: { + sessionStatus: "idle", + lastTraceId: traceId, + messages: [ + { role: "user", text: "reply OK", traceId, status: "sent" }, + { role: "agent", text: "partial assistant text", traceId, status: "running" } + ] + } + }; + traceStore.append(traceId, { type: "request", status: "accepted", label: "request:accepted" }); + traceStore.append(traceId, { type: "result", status: "completed", label: "result:completed", terminal: true }); + results.set(traceId, { + status: "completed", + traceId, + ownerUserId: ACTOR.id, + conversationId: session.conversationId, + sessionId: session.id, + threadId: session.threadId, + finalResponse: finalText, + agentRun: { runId: "run_workbench_idle_completed", commandId: "cmd_workbench_idle_completed", status: "completed" } + }); + const accessController = { + store: { + async listAgentSessionsForUser() { return [session]; }, + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } }); + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ + session, + events: [ + { seq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-17T00:02:58.000Z" }, + { seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-17T00:03:00.000Z" } + ], + status: "completed", + finalText, + runId: "run_workbench_idle_completed", + commandId: "cmd_workbench_idle_completed", + lastProjectedSeq: 2 + }] + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results, backendPerformanceStore }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].status, "completed"); + assert.equal(sessions.body.sessions[0].running, false); + assert.equal(sessions.body.sessions[0].terminal, true); + assert.equal(sessions.body.sessions[0].turnSummary.status, "completed"); + + const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`); + assert.equal(messages.status, 200); + assert.equal(messages.body.messages[1].status, "completed"); + assert.equal(messages.body.messages[1].text, finalText); + + const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); + assert.equal(turn.status, 200); + assert.equal(turn.body.turn.status, "completed"); + assert.equal(turn.body.turn.running, false); + assert.equal(turn.body.turn.terminal, true); + assert.equal(turn.body.turn.assistantText, finalText); + assert.equal(turn.body.projectionStatus, "caught-up"); + assert.equal(turn.body.sourceRunId, "run_workbench_idle_completed"); + assert.equal(turn.body.sourceCommandId, "cmd_workbench_idle_completed"); + const metricsText = backendPerformanceStore.metricsText(); + const lagCountLine = metricsText.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_lag_seconds_count") && line.includes('projection_status="caught_up"')); + const turnGetCountLine = metricsText.split("\n").find((line) => line.startsWith("hwlab_workbench_turn_get_duration_seconds_count") && line.includes('route="/v1/workbench/turns/:traceId"')); + assert.ok(lagCountLine?.endsWith(" 1")); + assert.ok(turnGetCountLine?.endsWith(" 1")); + + const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); + assert.equal(trace.status, 200); + assert.equal(trace.body.projectionStatus, "caught-up"); + assert.equal(trace.body.sourceRunId, "run_workbench_idle_completed"); + assert.equal(trace.body.sourceCommandId, "cmd_workbench_idle_completed"); + assert.equal(trace.body.lastProjectedSeq, 2); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model lets terminal result override stale running session state", async () => { + const traceStore = createCodeAgentTraceStore(); + const results = createCodeAgentChatResultStore(); + const traceId = "trc_workbench_stale_running_result_completed"; + const finalText = "OK"; + const session = { + id: "ses_workbench_stale_running_result_completed", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_stale_running_result_completed", + threadId: "thread-workbench-stale-running-result-completed", + lastTraceId: traceId, + updatedAt: "2026-06-18T16:31:04.000Z", + session: { + sessionStatus: "running", + lastTraceId: traceId, + messages: [ + { role: "user", text: "reply OK", traceId, status: "sent" }, + { role: "agent", text: "", traceId, status: "running" } + ] + } + }; + traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false }); + results.set(traceId, { + status: "completed", + traceId, + ownerUserId: ACTOR.id, + conversationId: session.conversationId, + sessionId: session.id, + threadId: session.threadId, + finalResponse: finalText, + agentRun: { runId: "run_workbench_stale_running_completed", commandId: "cmd_workbench_stale_running_completed", status: "completed" } + }); + const accessController = { + store: { + async listAgentSessionsForUser() { return [session]; }, + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ + session, + events: [ + { seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T16:30:00.000Z" }, + { seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-18T16:31:04.000Z" } + ], + status: "completed", + finalText, + runId: "run_workbench_stale_running_completed", + commandId: "cmd_workbench_stale_running_completed", + lastProjectedSeq: 2 + }] + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].status, "completed"); + assert.equal(sessions.body.sessions[0].running, false); + assert.equal(sessions.body.sessions[0].terminal, true); + assert.equal(sessions.body.sessions[0].turnSummary.status, "completed"); + + const detail = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}`); + assert.equal(detail.status, 200); + assert.equal(detail.body.session.status, "completed"); + assert.equal(detail.body.session.running, false); + assert.equal(detail.body.session.terminal, true); + + const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`); + assert.equal(messages.status, 200); + assert.equal(messages.body.messages[1].status, "completed"); + assert.equal(messages.body.messages[1].text, finalText); + + const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); + assert.equal(turn.status, 200); + assert.equal(turn.body.status, "completed"); + assert.equal(turn.body.turn.status, "completed"); + assert.equal(turn.body.turn.running, false); + assert.equal(turn.body.turn.terminal, true); + assert.equal(turn.body.turn.assistantText, finalText); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench session list uses durable projection checkpoint without trace/result hydration", async () => { + const traceStore = createCodeAgentTraceStore(); + const traceId = "trc_workbench_compact_list_summary"; + const finalText = "compact list summary OK"; + const session = { + id: "ses_workbench_compact_list_summary", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_compact_list_summary", + threadId: "thread-workbench-compact-list-summary", + lastTraceId: traceId, + updatedAt: "2026-06-19T15:40:00.000Z", + session: { + sessionStatus: "running", + lastTraceId: traceId, + messages: [ + { role: "user", text: "compact summary", traceId, status: "sent" }, + { role: "agent", text: "", traceId, status: "running" } + ], + traceResults: { + [traceId]: { + traceId, + status: "completed", + finalResponse: { text: finalText, status: "completed", traceId, valuesPrinted: false }, + traceSummary: { + traceId, + source: "agent-session-compact-summary", + sourceEventCount: 42, + terminalStatus: "completed", + agentRun: { runId: "run_workbench_compact_list_summary", commandId: "cmd_workbench_compact_list_summary", lastSeq: 42, valuesPrinted: false }, + valuesPrinted: false + }, + agentRun: { runId: "run_workbench_compact_list_summary", commandId: "cmd_workbench_compact_list_summary", status: "completed", terminalStatus: "completed", lastSeq: 42, valuesPrinted: false }, + updatedAt: "2026-06-19T15:40:00.000Z", + valuesRedacted: true, + secretMaterialStored: false + } + }, + valuesRedacted: true, + secretMaterialStored: false + } + }; + const accessController = { + store: { + async listAgentSessionsForUser() { return [session]; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const factQueries = []; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ + session, + events: [ + { seq: 1, sourceSeq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-19T15:39:10.000Z" }, + { seq: 42, sourceSeq: 42, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-19T15:40:00.000Z" } + ], + status: "completed", + finalText, + runId: "run_workbench_compact_list_summary", + commandId: "cmd_workbench_compact_list_summary", + lastProjectedSeq: 42 + }], + queries: factQueries + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].status, "completed"); + assert.equal(sessions.body.sessions[0].running, false); + assert.equal(sessions.body.sessions[0].terminal, true); + assert.equal(sessions.body.sessions[0].turnSummary.status, "completed"); + assert.equal(sessions.body.sessions[0].turnSummary.eventCount, 42); + assert.equal(sessions.body.sessions[0].projectionStatus, "caught-up"); + assert.ok(factQueries.length >= 1); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench session list overlaps include lookup with the page query", async () => { + const makeSession = (id, traceId, updatedAt) => ({ + id, + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: id.replace(/^ses_/u, "cnv_"), + threadId: `thread-${id}`, + lastTraceId: traceId, + updatedAt, + session: { + sessionStatus: "running", + lastTraceId: traceId, + messages: [{ role: "user", text: id, traceId, status: "sent" }] + } + }); + const included = makeSession("ses_parallel_include", "trc_parallel_include", "2026-06-19T14:00:00.000Z"); + const firstPage = makeSession("ses_parallel_page_a", "trc_parallel_page_a", "2026-06-19T16:00:00.000Z"); + const secondPage = makeSession("ses_parallel_page_b", "trc_parallel_page_b", "2026-06-19T15:00:00.000Z"); + const facts = emptyFacts(); + for (const session of [included, firstPage, secondPage]) mergeFacts(facts, buildDurableFactsForSession({ session, status: "running" })); + const accessController = { + store: {}, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + let pageReleased = false; + let releasePage = () => {}; + const pageGate = new Promise((resolve) => { releasePage = resolve; }); + const queries = []; + const runtimeStore = { + async queryWorkbenchFacts(params = {}) { + queries.push({ ...params, pageReleased }); + if (params.sessionsOrder === "updated_desc") await pageGate; + const filtered = filterFacts(facts, params); + return { + facts: filtered, + count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), + persistence: { adapter: "test-durable-workbench-facts", durable: true } + }; + } + }; + const server = createCloudApiServer({ accessController, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const pending = getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(included.id)}&limit=1`); + await waitForCondition(() => queries.some((query) => query.sessionId === included.id) && queries.some((query) => query.sessionsOrder === "updated_desc")); + const includeIndex = queries.findIndex((query) => query.sessionId === included.id); + assert.ok(includeIndex >= 0); + assert.equal(queries[includeIndex].pageReleased, false); + pageReleased = true; + releasePage(); + const sessions = await pending; + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].sessionId, included.id); + assert.equal(sessions.body.hasMore, true); + assert.equal(queries.find((query) => query.sessionsOrder === "updated_desc")?.sessionProjection, "summary"); + assert.equal(queries.find((query) => query.sessionId === included.id)?.sessionProjection, undefined); + } finally { + pageReleased = true; + releasePage(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model reads terminal session status from durable facts", async () => { + const traceStore = createCodeAgentTraceStore(); + const traceId = "trc_workbench_durable_terminal_after_memory_running"; + const session = { + id: "ses_workbench_durable_terminal_after_memory_running", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_durable_terminal_after_memory_running", + threadId: "thread-workbench-durable-terminal-after-memory-running", + lastTraceId: traceId, + updatedAt: "2026-06-18T17:22:00.000Z", + session: { + sessionStatus: "running", + lastTraceId: traceId, + messages: [ + { role: "user", text: "reply OK", traceId, status: "sent" }, + { role: "agent", text: "", traceId, status: "running" } + ] + } + }; + traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false, seq: 1 }); + const durableEvents = [ + { traceId, seq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-18T17:21:40.000Z", valuesPrinted: false }, + { traceId, seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-18T17:22:00.000Z", valuesPrinted: false } + ]; + const accessController = { + store: { + async listAgentSessionsForUser() { return [session]; }, + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ session, events: durableEvents, status: "completed", finalText: "OK", lastProjectedSeq: 2 }] + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`); + assert.equal(sessions.status, 200); + assert.equal(sessions.body.sessions[0].status, "completed"); + assert.equal(sessions.body.sessions[0].running, false); + assert.equal(sessions.body.sessions[0].terminal, true); + + const detail = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}`); + assert.equal(detail.status, 200); + assert.equal(detail.body.session.status, "completed"); + assert.equal(detail.body.session.running, false); + assert.equal(detail.body.session.terminal, true); + + const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`); + assert.equal(messages.status, 200); + assert.equal(messages.body.messages[1].status, "completed"); + + const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); + assert.equal(turn.status, 200); + assert.equal(turn.body.status, "completed"); + assert.equal(turn.body.turn.status, "completed"); + assert.equal(turn.body.turn.running, false); + assert.equal(turn.body.turn.terminal, true); + assert.equal(turn.body.turn.trace.eventCount, 2); + assert.equal(turn.body.projectionStatus, "caught-up"); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model keeps non-terminal durable tool completed events out of turn status", async () => { + const traceStore = createCodeAgentTraceStore(); + const results = createCodeAgentChatResultStore(); + const traceId = "trc_workbench_durable_tool_completed_nonterminal"; + const session = { + id: "ses_workbench_durable_tool_completed_nonterminal", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_durable_tool_completed_nonterminal", + threadId: "thread-workbench-durable-tool-completed-nonterminal", + lastTraceId: traceId, + updatedAt: "2026-06-18T19:35:51.787Z", + session: { + sessionStatus: "running", + lastTraceId: traceId, + messages: [ + { role: "user", text: "write benchmark", traceId, status: "sent" }, + { role: "agent", text: "", traceId, status: "running" } + ], + valuesRedacted: true, + secretMaterialStored: false + } + }; + traceStore.append(traceId, { seq: 1, type: "backend_status", status: "running", label: "runner:created", terminal: false }); + const durableEvents = [ + { traceId, seq: 1, type: "assistant_message", status: "running", label: "agentrun:assistant:message", terminal: false, message: "正在安装 Python。", createdAt: "2026-06-18T19:35:40.000Z", valuesPrinted: false }, + { traceId, seq: 2, type: "commandExecution", status: "completed", label: "item/commandExecution:completed", terminal: false, command: "ls -ld .", stdout: "OK\n", createdAt: "2026-06-18T19:35:51.787Z", valuesPrinted: false } + ]; + results.set(traceId, { + status: "running", + traceId, + ownerUserId: ACTOR.id, + conversationId: session.conversationId, + sessionId: session.id, + threadId: session.threadId, + finalResponse: null, + assistantText: null, + agentRun: { runId: "run_workbench_nonterminal_tool_completed", commandId: "cmd_workbench_nonterminal_tool_completed", status: "runner-job-created", runStatus: "pending", commandState: "pending" } + }); + const accessController = { + store: { + async listAgentSessionsForUser() { return [session]; }, + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ + session, + events: durableEvents, + status: "running", + assistantText: "partial assistant text", + runId: "run_workbench_nonterminal_tool_completed", + commandId: "cmd_workbench_nonterminal_tool_completed", + projectionStatus: "projecting", + lastProjectedSeq: 2 + }] + }); + const workbenchRuntime = { + async queryWorkbenchFacts(query) { return runtimeStore.queryWorkbenchFacts(query); } + }; + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, workbenchRuntime, codeAgentChatResults: results }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); + assert.equal(turn.status, 200); + assert.equal(turn.body.status, "running"); + assert.equal(turn.body.turn.status, "running"); + assert.equal(turn.body.turn.running, true); + assert.equal(turn.body.turn.terminal, false); + assert.equal(turn.body.turn.assistantText, null); + assert.equal(turn.body.turn.finalResponse, null); + assert.equal(turn.body.projectionStatus, "projecting"); + + const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); + assert.equal(trace.status, 200); + assert.equal(trace.body.events.at(-1).status, "completed"); + assert.equal(trace.body.events.at(-1).terminal, false); + assert.equal(trace.body.projectionStatus, "projecting"); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench trace events API blocks historical projectedSeq collisions instead of handing them to renderer", async () => { + const traceStore = createCodeAgentTraceStore(); + const traceId = "trc_workbench_projected_seq_collision"; + const session = { + id: "ses_workbench_projected_seq_collision", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_projected_seq_collision", + threadId: "thread-workbench-projected-seq-collision", + lastTraceId: traceId, + updatedAt: "2026-06-20T12:20:00.000Z", + session: { sessionStatus: "running", lastTraceId: traceId, messages: [{ role: "user", text: "collision", traceId, status: "sent" }] } + }; + const accessController = { + store: { + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = createDurableFactsRuntimeStore({ + sessions: [{ + session, + events: [ + { id: "wte_collision_a", projectedSeq: 1, sourceSeq: 1, sourceEventId: "source-a", type: "backend", status: "running", label: "projected:1:a", createdAt: "2026-06-20T12:19:58.000Z" }, + { id: "wte_collision_b", projectedSeq: 1, sourceSeq: 2, sourceEventId: "source-b", type: "assistant", status: "running", label: "projected:1:b", message: "later", createdAt: "2026-06-20T12:19:59.000Z" } + ], + status: "running", + projectionStatus: "projecting", + lastProjectedSeq: 1 + }] + }); + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); + assert.equal(trace.status, 200); + assert.equal(trace.body.status, "blocked"); + assert.equal(trace.body.projectionStatus, "blocked"); + assert.equal(trace.body.projectionHealth, "degraded"); + assert.equal(trace.body.blocker.code, "projected_seq_collision"); + assert.deepEqual(trace.body.events, []); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); diff --git a/internal/cloud/server-workbench-realtime-http.test.ts b/internal/cloud/server-workbench-realtime-http.test.ts new file mode 100644 index 00000000..ad278165 --- /dev/null +++ b/internal/cloud/server-workbench-realtime-http.test.ts @@ -0,0 +1,448 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery. +// Responsibility: Workbench realtime and projection blocker regression tests. + +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { createCloudApiServer } from "./server.ts"; +import { createBackendPerformanceStore } from "./backend-performance.ts"; +import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts"; +import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; +import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts"; + +import { + ACTOR, + getJson, + waitForCondition, + createDurableFactsRuntimeStore, + createRuntimeStoreFromFacts, + buildDurableFactsForSession, + normalizeTestMessages, + testTimingProjection, + timestampIso, + elapsedMs, + normalizeTestEvents, + filterFacts, + workbenchTestFactOrder, + matchesFact, + matchesTraceFact, + testFactFamilySet, + mergeFacts, + emptyFacts, + normalizeTestStatus, + getSseEvents, + parseSseBlock, + createFakeKafkaFactory, + waitFor +} from "./server-workbench-http-test-helpers.ts"; + +test("workbench realtime stream surfaces facts blocker instead of legacy trace fallback", async () => { + const traceStore = createCodeAgentTraceStore(); + const results = createCodeAgentChatResultStore(); + const traceId = "trc_workbench_realtime"; + const session = { + id: "ses_workbench_realtime", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_realtime", + threadId: "thread-workbench-realtime", + lastTraceId: traceId, + updatedAt: "2026-06-17T02:00:00.000Z", + session: { sessionStatus: "running", lastTraceId: traceId, messages: [{ role: "user", text: "stream", traceId }] } + }; + traceStore.append(traceId, { type: "assistant", status: "completed", label: "assistant:completed", terminal: true, message: "legacy trace answer" }); + results.set(traceId, { status: "completed", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId, finalResponse: { text: "legacy result answer" } }); + const accessController = { + store: { + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const server = createCloudApiServer({ accessController, traceStore, codeAgentChatResults: results, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 4); + const events = await eventsPromise; + assert.deepEqual(events.map((event) => event.event), [ + "workbench.connected", + "workbench.error", + "workbench.trace.snapshot", + "workbench.turn.snapshot" + ]); + assert.equal(events[0].data.filters.sessionId, session.id); + assert.equal(events[1].data.error.code, "workbench_facts_trace_missing"); + assert.equal(events[2].data.traceId, traceId); + assert.equal(events[2].data.snapshot.status, "unknown"); + assert.equal(events[2].data.snapshot.eventCount, 0); + assert.equal(events[3].data.turn.traceId, traceId); + assert.equal(events[3].data.turn.status, "unknown"); + assert.equal(events[3].data.turn.terminal, false); + assert.equal(events[3].data.turn.finalResponse, null); + assert.equal(JSON.stringify(events).includes("legacy result answer"), false); + assert.equal(JSON.stringify(events).includes("legacy trace answer"), false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); +test("workbench realtime stream forwards HWLAB Kafka events after initial connection", async () => { + const fakeKafka = createFakeKafkaFactory(); + const traceId = "trc_workbench_realtime_after_seq"; + const session = { + id: "ses_workbench_realtime_after_seq", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_realtime_after_seq", + threadId: "thread-workbench-realtime-after-seq", + lastTraceId: traceId, + updatedAt: "2026-06-24T14:00:00.000Z", + session: { sessionStatus: "running", lastTraceId: traceId } + }; + const outboxQueries = []; + const workbenchRuntime = { + async readWorkbenchProjectionOutbox(params = {}) { + outboxQueries.push({ ...params }); + return [ + { outboxSeq: 11, projectedSeq: 7, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, createdAt: "2026-06-24T14:00:01.000Z" } + ]; + } + }; + const accessController = { + store: { + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const serverWithKafka = createCloudApiServer({ + accessController, + workbenchRuntime, + kafkaFactory: fakeKafka.factory, + env: { + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", + HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + } + }); + await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = serverWithKafka.address(); + setTimeout(() => { void fakeKafka.emit({ + eventType: "hwlab.trace.event.projected", + sessionId: "ses_agentrun_workbench_realtime_after_seq", + traceId, + context: { sourceSeq: 7, runId: "run_workbench_realtime_after_seq", commandId: "cmd_workbench_realtime_after_seq" }, + event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:test", message: "Kafka realtime event", sourceSeq: 7 } + }); }, 25); + const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); + assert.equal(events[0].id, "10"); + assert.equal(events[1].id, "hwlab.event.v1:0:0"); + assert.equal(events[1].data.realtimeSource, "kafka"); + assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2"); + assert.equal(events[1].data.entity.family, "traceEvents"); + assert.equal(events[1].data.entity.version, 7); + assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0"); + assert.equal(events[1].data.event.message, "Kafka realtime event"); + assert.equal(events[1].data.kafka.topic, "hwlab.event.v1"); + assert.equal(events[1].data.cursor.traceSeq, 7); + assert.deepEqual(outboxQueries, []); + } finally { + await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench session realtime Kafka stream does not pin subscription to stale lastTraceId", async () => { + const fakeKafka = createFakeKafkaFactory(); + const staleTraceId = "trc_workbench_realtime_stale_trace"; + const liveTraceId = "trc_workbench_realtime_live_trace"; + const session = { + id: "ses_workbench_realtime_session_wide", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_realtime_session_wide", + threadId: "thread-workbench-realtime-session-wide", + lastTraceId: staleTraceId, + updatedAt: "2026-06-24T14:00:00.000Z", + session: { sessionStatus: "running", lastTraceId: staleTraceId } + }; + const accessController = { + store: { + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId() { return null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const serverWithKafka = createCloudApiServer({ + accessController, + workbenchRuntime: { + async queryWorkbenchFacts(params = {}) { + return { + facts: { + sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, threadId: session.threadId, lastTraceId: staleTraceId, status: "running", valuesRedacted: true }], + messages: [], + parts: [], + turns: [], + checkpoints: [] + }, + count: 1, + persistence: { adapter: "test-session-wide-kafka", durable: true }, + params + }; + } + }, + kafkaFactory: fakeKafka.factory, + env: { + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", + HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + } + }); + await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = serverWithKafka.address(); + setTimeout(() => { void fakeKafka.emit({ + eventType: "hwlab.trace.event.projected", + sessionId: "ses_agentrun_workbench_realtime_session_wide", + traceId: liveTraceId, + context: { runId: "run_workbench_realtime_session_wide", commandId: "cmd_workbench_realtime_session_wide" }, + event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:live", message: "Kafka live trace event" } + }); }, 100); + const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}`, 4); + assert.equal(events[0].event, "workbench.connected"); + const liveEvent = events.find((event) => event.event === "workbench.trace.event" && event.data?.traceId === liveTraceId); + assert.ok(liveEvent, JSON.stringify(events.map((event) => ({ event: event.event, traceId: event.data?.traceId, message: event.data?.event?.message })))); + assert.equal(liveEvent.data.event.message, "Kafka live trace event"); + assert.equal(liveEvent.data.realtimeAuthority, "workbench-realtime-authority-v2"); + } finally { + await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench read model exposes runtime trace projection query failures as projection blockers", async () => { + const traceStore = createCodeAgentTraceStore(); + const results = createCodeAgentChatResultStore(); + const traceId = "trc_workbench_projection_store_unavailable"; + const session = { + id: "ses_workbench_projection_store_unavailable", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_projection_store_unavailable", + threadId: "thread-workbench-projection-store-unavailable", + lastTraceId: traceId, + updatedAt: "2026-06-18T01:10:00.000Z", + session: { + sessionStatus: "running", + lastTraceId: traceId, + messages: [ + { role: "user", text: "projection store unavailable", traceId, status: "sent", createdAt: "2026-06-18T01:09:58.000Z" }, + { role: "agent", text: "", traceId, status: "running", createdAt: "2026-06-18T01:09:59.000Z" } + ], + valuesRedacted: true, + secretMaterialStored: false + } + }; + traceStore.append(traceId, { seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T01:10:00.000Z" }); + results.set(traceId, { status: "running", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId }); + const accessController = { + store: { + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = { + async queryWorkbenchFacts() { + const error = new Error("runtime query failed"); + error.code = "TEST_RUNTIME_QUERY_FAILED"; + throw error; + } + }; + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); + assert.equal(turn.status, 503); + assert.equal(turn.body.error.code, "projection_store_unavailable"); + + const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); + assert.equal(trace.status, 503); + assert.equal(trace.body.error.code, "projection_store_unavailable"); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench trace events remain visible after session lastTraceId moves", async () => { + const oldTraceId = "trc_workbench_history_old_trace"; + const newTraceId = "trc_workbench_history_new_trace"; + const session = { + id: "ses_workbench_history_trace", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_history_trace", + threadId: "thread-workbench-history-trace", + lastTraceId: newTraceId, + updatedAt: "2026-06-20T10:10:00.000Z", + session: { + messages: [ + { role: "user", text: "old turn", traceId: oldTraceId, status: "sent", createdAt: "2026-06-20T10:00:00.000Z" }, + { role: "agent", text: "old final", traceId: oldTraceId, status: "completed", createdAt: "2026-06-20T10:00:01.000Z" } + ], + valuesRedacted: true + } + }; + const oldFacts = buildDurableFactsForSession({ + session: { ...session, status: "completed", lastTraceId: oldTraceId, updatedAt: "2026-06-20T10:01:00.000Z" }, + status: "completed", + finalText: "old final", + runId: "run_workbench_history_old_trace", + commandId: "cmd_workbench_history_old_trace", + events: [ + { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:00:00.000Z" }, + { projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:01:00.000Z" } + ], + lastProjectedSeq: 2 + }); + const newFacts = buildDurableFactsForSession({ + session: { ...session, session: { messages: [{ role: "user", text: "new turn", traceId: newTraceId, status: "sent" }] } }, + status: "running", + runId: "run_workbench_history_new_trace", + commandId: "cmd_workbench_history_new_trace", + events: [{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:10:00.000Z" }], + lastProjectedSeq: 1 + }); + const facts = emptyFacts(); + mergeFacts(facts, oldFacts); + mergeFacts(facts, newFacts); + facts.sessions = newFacts.sessions; + const queries = []; + const runtimeStore = { + async queryWorkbenchFacts(params = {}) { + queries.push(params); + const filtered = filterFacts(facts, params); + return { + facts: filtered, + count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), + persistence: { adapter: "test-durable-workbench-facts", durable: true } + }; + } + }; + const accessController = { + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const server = createCloudApiServer({ accessController, runtimeStore }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(oldTraceId)}/events?limit=10`); + assert.equal(response.status, 200); + assert.equal(response.body.status, "succeeded"); + assert.equal(response.body.sessionId, session.id); + assert.equal(response.body.traceId, oldTraceId); + assert.equal(response.body.events.length, 2); + assert.equal(response.body.events.at(-1).status, "completed"); + assert.equal(response.body.error, undefined); + assert.equal(queries.some((query) => query.sessionId === session.id && Array.isArray(query.families) && query.families.includes("sessions")), true); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench trace event page keeps per-trace terminal status after later session cancel", async () => { + const completedTraceId = "trc_workbench_completed_before_cancel"; + const canceledTraceId = "trc_workbench_later_cancel"; + const session = { + id: "ses_workbench_completed_before_cancel", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "canceled", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_completed_before_cancel", + threadId: "thread-workbench-completed-before-cancel", + lastTraceId: canceledTraceId, + updatedAt: "2026-06-20T10:20:30.000Z", + session: { + messages: [ + { role: "user", text: "completed first", traceId: completedTraceId, status: "sent", createdAt: "2026-06-20T10:20:00.000Z" }, + { role: "agent", text: "completed final", traceId: completedTraceId, status: "completed", createdAt: "2026-06-20T10:20:10.000Z" }, + { role: "user", text: "cancel later", traceId: canceledTraceId, status: "sent", createdAt: "2026-06-20T10:20:20.000Z" }, + { role: "agent", text: "hwlab-user-cancel", traceId: canceledTraceId, status: "canceled", createdAt: "2026-06-20T10:20:30.000Z" } + ], + valuesRedacted: true + } + }; + const facts = emptyFacts(); + mergeFacts(facts, buildDurableFactsForSession({ + session: { ...session, status: "completed", lastTraceId: completedTraceId, updatedAt: "2026-06-20T10:20:10.000Z" }, + status: "completed", + finalText: "completed final", + events: [ + { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:00.000Z" }, + { projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:20:10.000Z" } + ], + lastProjectedSeq: 2 + })); + mergeFacts(facts, buildDurableFactsForSession({ + session: { ...session, status: "canceled", lastTraceId: canceledTraceId, session: { messages: session.session.messages.slice(2), valuesRedacted: true } }, + status: "canceled", + finalText: "hwlab-user-cancel", + events: [ + { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:20.000Z" }, + { projectedSeq: 2, sourceSeq: 2, type: "cancel", status: "canceled", label: "runner:canceled", terminal: true, createdAt: "2026-06-20T10:20:30.000Z" } + ], + lastProjectedSeq: 2 + })); + facts.sessions = facts.sessions.filter((item) => item.lastTraceId === canceledTraceId); + const runtimeStore = { + async queryWorkbenchFacts(params = {}) { + const filtered = filterFacts(facts, params); + return { + facts: filtered, + count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0), + persistence: { adapter: "test-durable-workbench-facts", durable: true } + }; + } + }; + const accessController = { + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(completedTraceId)}/events?limit=10`); + assert.equal(response.status, 200); + assert.equal(response.body.traceStatus, "completed"); + assert.equal(response.body.events.at(-1).status, "completed"); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); diff --git a/internal/cloud/server-workbench-realtime-http.ts b/internal/cloud/server-workbench-realtime-http.ts new file mode 100644 index 00000000..548b8ec8 --- /dev/null +++ b/internal/cloud/server-workbench-realtime-http.ts @@ -0,0 +1,987 @@ +/* + * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-read-model-timeline-contract; draft-2026-06-28-p0-d518-session-timeline-consistency; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-api-contract; PJ2026-010401 Web工作台 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-model-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 + * 职责: Workbench SSE and Kafka realtime transport over durable read-model facts. + */ +import { createHash } from "node:crypto"; + +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { + parsePositiveInteger, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson +} from "./server-http-utils.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; +import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; +import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; +import { openKafkaEventStream } from "./kafka-event-bridge.ts"; +import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; +import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; +import * as workbenchFacts from "./server-workbench-facts.ts"; + +const DEFAULT_PAGE_LIMIT = 50; +const DEFAULT_SESSION_LIST_LIMIT = 20; +const MAX_PAGE_LIMIT = 100; +const DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS = 15000; +const WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS = 2500; +const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; +const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]); +const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); +const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); +const WORKBENCH_TRACE_EVENT_METADATA_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); +const WORKBENCH_TRACE_EVENT_PAGE_FAMILIES = Object.freeze(["traceEvents"]); +const activeWorkbenchRealtimeConnections = new Set(); + +const { + factSessionSummary, + factSessionDetail, + factMessagesForSession, + canonicalFactMessageGroupsForSession, + factMessageCanonicalGroupKey, + selectCanonicalFactMessage, + factMessageCanonicalScore, + factMessageHasFinalResponsePart, + workbenchLifecycleMessageId, + factMessageDto, + factMessageAuthorityText, + factPartsHaveFinalResponse, + factSyntheticTerminalFinalResponse, + isTerminalProjectionStatus, + firstFactPartText, + factMessageFinalResponseText, + factTerminalMessageForTrace, + factTerminalStatusFromMessageProjection, + factPartDto, + factTurnForTrace, + factTurnSnapshot, + factTraceSnapshot, + factTraceTimingProjection, + factTraceEventIsTerminalAuthority, + factTraceEventTerminalTimes, + factTraceEventDto, + factProjectionForTrace, + factTimingProjection, + factCombinedTimingProjection, + factTimingSource, + firstTimestampIso, + latestTimestampIso, + elapsedFactMs, + timestampIso, + factCheckpointForTrace, + workbenchProjectionStoreError, + nonNegativeIntegerOrNull, + factSessionId, + factLastTraceId, + factLatestTraceIdForSession, + factCurrentTraceForSession, + factMessageTraceCandidate, + factTraceCandidate, + compareTraceCandidatesDesc, + nonUnknownStatus, + factTurnId, + factUpdatedAt, + factSeq, + factProjectedSeq, + normalizeProjectionStatus, + normalizeProjectionHealth, + compareFactSessionsDesc, + compareFactMessagesAsc, + compareFactMessagesForSessionAsc, + factMessageTimelineKey, + factTimelineAnchorMessageForTrace, + factMessageTimelineRoleRank, + firstFiniteNumber, + compareOptionalNumberAsc, + compareFactPartsAsc, + compareFactTraceEventsAsc, + compareFactRecordsDesc, + compareTimestampAsc, + compareTimestampDesc, + compareOptionalTimestampAsc, + compareNumberAsc, + compareNumberDesc, + compareText, + timestampMs, + optionalTimestampMs, + numericValue, + mergeFactSets, + emptyFactSet, + factArray, + uniqueText, + eventSeq, + normalizeStatus, + normalizeTerminalStatus, + firstUserPreview, + latestUserPreview, + latestValidMessagePreview, + latestAssistantPreview, + sessionTitleFromMessages, + sessionPreviewFromMessages, + boundedPreviewText, + isAssistantLikeRole, + objectValue, + textValue, + messageAuthorityTextValue, + safeTurnId, + safeMessageId, + safePartId, + hash +} = workbenchFacts; + +import { + authenticateWorkbenchRead, + blockedTraceEventProjection, + classifyWorkbenchReadModelFailure, + isAgentRunAliasSessionId, + logWorkbenchReadModelFailure, + methodNotAllowed, + nonNegativeInteger, + queryFactsByRouteId, + traceEventsReadModelBlocker, + visibleFactSessionForTrace, + visibleFactSessions, + workbenchError +} from "./server-workbench-read-http.ts"; + +export async function drainWorkbenchRealtimeConnections(options = {}) { + const reason = textValue(options.reason) || "server_shutdown"; + const signal = textValue(options.signal) || null; + const timeoutMs = parsePositiveInteger(options.timeoutMs, WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS); + const connections = [...activeWorkbenchRealtimeConnections].filter((connection) => connection?.isActive?.()); + for (const connection of connections) { + try { + connection.close({ reason, signal }); + } catch { + // Best effort: one broken client must not block shutdown drain for the rest. + } + } + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const activeAfterClose = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length; + if (activeAfterClose === 0) break; + await sleepMs(Math.min(50, Math.max(1, deadline - Date.now()))); + } + const activeAfter = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length; + return { + ok: activeAfter === 0, + activeBefore: connections.length, + closed: Math.max(0, connections.length - activeAfter), + activeAfter, + timeout: activeAfter > 0, + reason, + signal + }; +} + + +export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) { + try { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); + const requestedTraceId = safeTraceId(url.searchParams.get("traceId")); + const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS); + attachWorkbenchRealtimeOtelContext(request, { + sessionId: requestedSessionId, + traceId: requestedTraceId, + heartbeatMs, + realtimeSource: "kafka" + }); + const perf = options.backendPerformance; + const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); + if (!auth) return; + + if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) { + sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench read model is keyed by sessionId/threadId/traceId only.")); + return; + } + + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const readModel = createWorkbenchReadModel(options, auth.actor); + const requestedAfterSeq = workbenchRealtimeAfterSeq(request, url); + let closed = false; + let realtimeCloseReason = "client_close"; + let realtimeCloseSignal = null; + let realtimeSessionId = requestedSessionId ?? null; + let realtimeTraceId = requestedTraceId ?? null; + let realtimeThreadId = null; + const realtimeStartedAtMs = Date.now(); + const cleanup = []; + + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-store, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + "x-content-type-options": "nosniff" + }); + if (typeof response.flushHeaders === "function") response.flushHeaders(); + + const writeEvent = (name, payload = {}) => { + if (closed || response.destroyed) return; + const startedAt = nowMs(); + const eventCreatedAt = realtimeEventCreatedAt(payload); + const traceSeq = realtimeTraceSeq(payload); + const eventId = realtimeEventId(payload); + response.write(`event: ${name}\n`); + if (eventId) response.write(`id: ${eventId}\n`); + response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`); + perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: "ok" }); + }; + + const realtimeConnection = { + close(fields = {}) { + if (closed || response.destroyed || response.writableEnded) return false; + const reason = textValue(fields.reason) || "server_shutdown"; + realtimeCloseReason = reason; + realtimeCloseSignal = textValue(fields.signal) || null; + try { + writeEvent("workbench.server_draining", { + type: "server.draining", + status: "closing", + reason, + signal: realtimeCloseSignal, + sessionId: realtimeSessionId, + threadId: realtimeThreadId, + traceId: realtimeTraceId, + observedAt: new Date().toISOString() + }); + response.end(); + return true; + } catch { + try { response.destroy(); } catch {} + return false; + } + }, + isActive() { + return !closed && !response.destroyed && !response.writableEnded; + } + }; + activeWorkbenchRealtimeConnections.add(realtimeConnection); + cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); + + writeEvent("workbench.connected", { + type: "connected", + status: "connected", + realtimeSource: "kafka", + snapshotSource: "read-model", + heartbeatMs, + cursor: { outboxSeq: requestedAfterSeq }, + filters: { + sessionId: requestedSessionId, + traceId: requestedTraceId + } + }); + + const resolveInitialSession = requestedSessionId && requestedAfterSeq <= 0; + const initialContext = resolveInitialSession + ? await realtimeInitialSessionContext(readModel, auth.actor, requestedSessionId) + : { facts: emptyFactSet(), session: null, blocker: null }; + if (initialContext.blocker) { + writeEvent("workbench.error", { + type: "error", + reason: "initial-session", + sessionId: requestedSessionId, + traceId: requestedTraceId, + error: initialContext.blocker + }); + } + const initialFactSession = initialContext.session; + const streamSessionId = factSessionId(initialFactSession) ?? requestedSessionId ?? null; + const streamThreadId = safeOpaqueId(initialFactSession?.threadId) ?? (textValue(initialFactSession?.threadId) || null); + const activeTraceId = requestedTraceId + ?? factLastTraceId(initialFactSession) + ?? factLatestTraceIdForSession(initialContext.facts, streamSessionId) + ?? null; + realtimeSessionId = streamSessionId ?? requestedSessionId ?? null; + realtimeTraceId = activeTraceId ?? requestedTraceId ?? null; + realtimeThreadId = streamThreadId ?? null; + attachWorkbenchRealtimeOtelContext(request, { + sessionId: streamSessionId ?? requestedSessionId, + traceId: activeTraceId, + threadId: streamThreadId, + heartbeatMs, + realtimeSource: "kafka" + }); + emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); + if (activeTraceId && requestedAfterSeq <= 0) { + await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })); + } + const kafkaFilters = resolveWorkbenchRealtimeKafkaFilters({ traceId: requestedTraceId, sessionId: streamSessionId, traceStore }); + try { + const kafkaStream = await openKafkaEventStream({ + env: options.env ?? process.env, + stream: "hwlab", + fromBeginning: false, + ...kafkaFilters, + kafkaFactory: options.kafkaFactory, + onEvent: async (record) => { + if (closed || response.destroyed) return; + const realtime = workbenchRealtimeEventFromKafka(record, { sessionId: streamSessionId, threadId: streamThreadId, traceId: activeTraceId }); + if (!realtime) return; + writeEvent("workbench.trace.event", realtime.traceEvent); + if (realtime.turnSnapshot) writeEvent("workbench.turn.snapshot", realtime.turnSnapshot); + }, + onError: (error) => { + writeEvent("workbench.error", { + type: "error", + realtimeSource: "kafka", + sessionId: streamSessionId, + threadId: streamThreadId, + traceId: activeTraceId, + error: { code: "workbench_kafka_stream_failed", message: error?.message ?? "Workbench Kafka realtime stream failed." } + }); + } + }); + cleanup.push(() => { void kafkaStream.stop?.(); }); + } catch (kafkaError) { + writeEvent("workbench.error", { + type: "error", + realtimeSource: "kafka", + sessionId: streamSessionId, + threadId: streamThreadId, + traceId: activeTraceId, + error: { code: "workbench_kafka_stream_unavailable", message: kafkaError?.message ?? "Workbench Kafka realtime stream is unavailable." } + }); + } + + const heartbeat = setInterval(async () => { + try { + writeEvent("workbench.heartbeat", { + type: "heartbeat", + sessionId: requestedSessionId ?? null, + traceId: activeTraceId ?? null, + observedAt: new Date().toISOString() + }); + } catch (error) { + writeEvent("workbench.error", { + type: "error", + error: { code: "workbench_realtime_heartbeat_failed", message: error?.message ?? "Workbench realtime heartbeat failed." } + }); + } + }, Math.max(1000, heartbeatMs)); + cleanup.push(() => clearInterval(heartbeat)); + + response.on("close", () => { + if (closed) return; + closed = true; + emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + startedAtMs: realtimeStartedAtMs, + sessionId: realtimeSessionId, + threadId: realtimeThreadId, + traceId: realtimeTraceId, + activeConnectionCount: activeWorkbenchRealtimeConnections.size + }); + for (const item of cleanup.splice(0)) item(); + }); + } catch (error) { + handleWorkbenchRealtimeFailure(request, response, url, options, error); + } +} + +export function attachWorkbenchRealtimeOtelContext(request, fields = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context) return; + context.route = "/v1/workbench/events"; + context.otelAttributes = { + ...(context.otelAttributes ?? {}), + "workbench.route": "events", + "workbench.session_id": fields.sessionId ?? null, + traceId: fields.traceId ?? null, + "workbench.trace_id": fields.traceId ?? null, + "workbench.thread_id": fields.threadId ?? null, + "workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null, + "workbench.sse.realtime_source": fields.realtimeSource ?? null, + "workbench.sse.outbox_mode": false + }; +} + +export function resolveWorkbenchRealtimeKafkaFilters({ traceId = null, sessionId = null, traceStore = null } = {}) { + const resolved = traceId && typeof traceStore?.snapshot === "function" ? collectTraceLinkedIds(traceStore.snapshot(traceId)) : {}; + const hasAgentRunKey = Boolean(resolved.runId || resolved.commandId); + const fallbackSessionId = resolved.sessionId || agentRunScopedSessionIdFromWorkbenchSession(sessionId) || sessionId; + return compactObject({ + traceId: hasAgentRunKey ? null : traceId, + sessionId: hasAgentRunKey ? null : fallbackSessionId, + runId: resolved.runId, + commandId: resolved.commandId + }); +} + +export function agentRunScopedSessionIdFromWorkbenchSession(sessionId) { + const safeId = safeSessionId(sessionId); + if (!safeId || isAgentRunAliasSessionId(safeId)) return safeId; + const base = safeId.replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session"; + return safeSessionId(`ses_agentrun_${base}`); +} + +export function collectTraceLinkedIds(snapshot) { + const out = { sessionId: null, runId: null, commandId: null }; + const events = Array.isArray(snapshot?.events) ? snapshot.events : []; + for (const event of events) collectTraceLinkedIdsFromRecord(event, out); + collectTraceLinkedIdsFromRecord(snapshot?.lastEvent, out); + return out; +} + +export function collectTraceLinkedIdsFromRecord(record, out) { + const value = record && typeof record === "object" && !Array.isArray(record) ? record : {}; + const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {}; + const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {}; + out.sessionId ||= safeSessionId(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId); + out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId); + out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId); +} + +export function workbenchRealtimeEventFromKafka(record, context = {}) { + const value = record?.value && typeof record.value === "object" && !Array.isArray(record.value) ? record.value : null; + if (!value) return null; + const hwlabEvent = value.event && typeof value.event === "object" && !Array.isArray(value.event) ? value.event : {}; + const sourceContext = value.context && typeof value.context === "object" && !Array.isArray(value.context) ? value.context : {}; + const traceId = safeTraceId(value.traceId ?? hwlabEvent.traceId ?? context.traceId) ?? textValue(value.traceId ?? hwlabEvent.traceId ?? context.traceId); + const sessionId = safeSessionId(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId) ?? textValue(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId); + const threadId = safeOpaqueId(context.threadId ?? sourceContext.threadId) ?? textValue(context.threadId ?? sourceContext.threadId); + if (!traceId && !sessionId) return null; + const sourceSeq = integerValue(hwlabEvent.sourceSeq ?? sourceContext.sourceSeq); + const kafka = { + topic: textValue(record.topic), + partition: integerValue(record.partition), + offset: textValue(record.offset), + key: textValue(record.key), + timestamp: textValue(record.timestamp), + valuesRedacted: true + }; + const event = { + ...hwlabEvent, + traceId: traceId ?? hwlabEvent.traceId ?? null, + sessionId: sessionId ?? hwlabEvent.sessionId ?? null, + threadId: threadId ?? sourceContext.threadId ?? null, + runId: textValue(hwlabEvent.runId ?? sourceContext.runId), + commandId: textValue(hwlabEvent.commandId ?? sourceContext.commandId), + source: textValue(hwlabEvent.source) || "hwlab.kafka", + sourceSeq, + kafka, + valuesPrinted: false + }; + const status = normalizeStatus(event.status); + const terminal = event.terminal === true || TERMINAL_STATUSES.has(status); + const cursor = { + traceSeq: sourceSeq ?? integerValue(record.offset), + kafkaOffset: textValue(record.offset), + kafkaPartition: integerValue(record.partition) + }; + const entityVersion = sourceSeq ?? integerValue(record.offset) ?? 0; + const projectionRevision = ["kafka", kafka.topic, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"); + const traceEntity = { + family: "traceEvents", + id: [traceId ?? "trace", event.runId, event.commandId, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"), + version: entityVersion, + traceSeq: cursor.traceSeq, + projectionRevision, + authority: WORKBENCH_REALTIME_AUTHORITY_VERSION + }; + const traceEvent = { + id: kafka.topic && kafka.partition !== null && kafka.offset ? `${kafka.topic}:${kafka.partition}:${kafka.offset}` : null, + type: "trace.event", + contractVersion: "workbench-sync-v1", + realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, + realtimeSource: "kafka", + sessionId, + threadId, + traceId, + event, + snapshot: { traceId, sessionId, threadId, status: status || event.status || null, events: [event], eventCount: 1 }, + cursor, + entity: traceEntity, + projectionRevision, + kafka, + valuesPrinted: false + }; + const turnSnapshot = terminal ? { + type: "turn.snapshot", + contractVersion: "workbench-sync-v1", + realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, + realtimeSource: "kafka", + sessionId, + threadId, + traceId, + reason: "kafka-terminal", + cursor, + turn: { + traceId, + sessionId, + threadId, + status: status || event.status || "completed", + running: false, + terminal: true, + finalResponse: terminalFinalResponse(status || event.status || "completed", { + traceId, + status: status || event.status || "completed", + finalResponse: event.text ? { text: event.text, status: status || event.status || "completed", traceId, source: "kafka-terminal-event", valuesPrinted: false } : null, + finalText: event.text ?? event.message ?? null, + error: event.errorCode ? { message: event.message ?? event.errorCode } : null + }), + agentRun: compactObject({ runId: event.runId, commandId: event.commandId }), + valuesPrinted: false + }, + entity: { + family: "turns", + id: traceId ?? event.runId ?? event.commandId ?? traceEntity.id, + version: entityVersion, + traceSeq: cursor.traceSeq, + projectionRevision, + authority: WORKBENCH_REALTIME_AUTHORITY_VERSION + }, + projectionRevision, + kafka, + valuesPrinted: false + } : null; + return { traceEvent, turnSnapshot }; +} + +export function compactObject(value) { + return Object.fromEntries(Object.entries(value || {}).filter(([, entry]) => textValue(entry))); +} + +export function integerValue(value) { + const number = Number(value); + return Number.isFinite(number) ? Math.trunc(number) : null; +} + +export function attachWorkbenchReadModelDiagnosticOtel(request, fields = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context) return; + const code = textValue(fields.code) || "workbench_read_model_not_found"; + const route = textValue(fields.route) || context.route || "/v1/workbench"; + context.route = route; + context.lastHttpError = { + code, + category: "workbench_read_model", + layer: "workbench.read_model" + }; + context.otelAttributes = { + ...(context.otelAttributes ?? {}), + "workbench.route": route, + "workbench.read_model.route": route, + "workbench.read_model.result": code, + "workbench.read_model.count": Number.isFinite(Number(fields.count)) ? Math.trunc(Number(fields.count)) : null, + "workbench.session_id": fields.sessionId ?? null, + "workbench.trace_id": fields.traceId ?? null, + "workbench.turn_id": fields.turnId ?? null, + traceId: fields.traceId ?? null, + "error.code": code, + "error.category": "workbench_read_model", + "error.layer": "workbench.read_model" + }; +} + +export function emitWorkbenchRealtimeAcceptedOtelSpan(request, env = process.env) { + const context = request?.hwlabHttpRequestContext; + if (!context?.traceId) return; + const endedAtMs = Date.now(); + void emitHttpServerRequestSpan(context, env, { + startTimeMs: context.startedAtMs, + endTimeMs: endedAtMs, + statusCode: 200, + method: "GET", + route: "/v1/workbench/events", + attributes: { + ...(context.otelAttributes ?? {}), + "http.closed_by": "sse-accepted", + "workbench.sse.lifecycle": "accepted", + "workbench.sse.accept_latency_ms": Math.max(0, endedAtMs - Number(context.startedAtMs ?? endedAtMs)) + } + }).catch(() => undefined); +} + +export function emitWorkbenchRealtimeClosedOtelSpan(request, env = process.env, fields = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context?.traceId) return; + const endedAtMs = Date.now(); + const startedAtMs = Number(fields.startedAtMs ?? context.startedAtMs ?? endedAtMs); + const reason = textValue(fields.reason) || "client_close"; + void emitHttpServerRequestSpan(context, env, { + startTimeMs: Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs, + endTimeMs: endedAtMs, + statusCode: 200, + method: "GET", + route: "/v1/workbench/events", + attributes: { + ...(context.otelAttributes ?? {}), + "http.closed_by": reason, + "workbench.sse.lifecycle": "closed", + "workbench.sse.close_reason": reason, + "workbench.sse.close_signal": fields.signal ?? null, + "workbench.sse.duration_ms": Math.max(0, endedAtMs - (Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs)), + "workbench.sse.session_id": fields.sessionId ?? null, + "workbench.sse.thread_id": fields.threadId ?? null, + "workbench.sse.trace_id": fields.traceId ?? null, + "workbench.sse.active_connection_count": Number.isFinite(Number(fields.activeConnectionCount)) ? Number(fields.activeConnectionCount) : null + } + }).catch(() => undefined); +} + +export function handleWorkbenchRealtimeFailure(request, response, url, options = {}, error) { + const classified = classifyWorkbenchReadModelFailure(error); + const context = request?.hwlabHttpRequestContext; + if (context) { + context.route = "/v1/workbench/events"; + context.otelAttributes = { + ...(context.otelAttributes ?? {}), + "workbench.route": "events", + "workbench.sse.lifecycle": "failed", + "error.code": classified.code, + "error.layer": "workbench.realtime" + }; + } + logWorkbenchReadModelFailure(options.logger ?? console, { + event: "workbench_realtime_route_failed", + ok: false, + route: "/v1/workbench/events", + statusCode: classified.statusCode, + errorCode: classified.code, + errorName: error?.name ?? "Error", + message: error instanceof Error ? error.message : String(error ?? "unknown"), + valuesRedacted: true + }); + if (response.headersSent && !response.destroyed) { + try { + response.write(`event: workbench.error\n`); + response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), type: "error", error: { code: classified.code, message: classified.message, valuesRedacted: true } })}\n\n`); + response.end(); + } catch { + try { response.destroy(); } catch {} + } + return; + } + if (!response.destroyed) sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { route: "/v1/workbench/events", errorName: error?.name ?? "Error" })); +} + +export async function writeMessageRealtimeSnapshotSafe({ writeEvent, readModel, actor, sessionId, threadId = null, traceId = null, reason = "message", cursor = null } = {}) { + const safeSession = safeSessionId(sessionId); + if (!safeSession || typeof readModel?.queryFacts !== "function") return; + try { + const result = await queryFactsByRouteId(readModel, safeSession, { families: WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES }); + if (result.error) return; + const session = visibleFactSessions(result.facts, actor)[0] ?? null; + if (!session) return; + const messages = factMessagesForSession(session, result.facts); + const message = [...messages].reverse().find((item) => { + if (!isAssistantLikeRole(item?.role)) return false; + return !traceId || item.traceId === traceId; + }) ?? null; + if (!message) return; + writeEvent("workbench.message.snapshot", { + type: "message.snapshot", + sessionId: factSessionId(session), + threadId: threadId ?? session.threadId ?? null, + traceId: message.traceId ?? traceId ?? null, + reason, + cursor, + message + }); + } catch (error) { + writeEvent("workbench.error", { + type: "error", + error: { code: "workbench_message_snapshot_failed", message: error?.message ?? "Workbench realtime message snapshot failed." } + }); + } +} + +export function realtimeEventCreatedAt(payload) { + const candidates = [payload?.event?.createdAt, payload?.snapshot?.lastEvent?.createdAt, payload?.turn?.updatedAt]; + for (const value of candidates) { + const text = textValue(value); + if (text) return text; + } + return null; +} + +export function realtimeTraceSeq(payload) { + const seq = Number(payload?.cursor?.traceSeq ?? payload?.event?.seq ?? payload?.snapshot?.lastEvent?.seq); + return Number.isFinite(seq) && seq >= 0 ? seq : null; +} + +export function realtimeEventId(payload) { + const raw = payload?.cursor?.outboxSeq ?? payload?.outboxSeq ?? payload?.id; + const text = textValue(raw); + if (!text || /[\r\n]/u.test(text)) return null; + return text.slice(0, 128); +} + +export function workbenchRealtimeAfterSeq(request, url) { + for (const value of [url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq"), request?.headers?.["last-event-id"]]) { + const parsed = nonNegativeInteger(value, 0); + if (parsed > 0) return parsed; + } + return 0; +} + +export function nowMs() { + if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); + return Date.now(); +} + +export function sleepMs(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + + +export async function realtimeInitialSessionContext(readModel, actor, sessionId) { + const safeId = safeSessionId(sessionId); + if (!safeId || typeof readModel?.queryFacts !== "function") { + const blocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench realtime initial session must be read from durable facts; the facts query interface is unavailable.", { session: { sessionId: safeId }, route: "/v1/workbench/events" }); + return { facts: emptyFactSet(), session: null, blocker }; + } + try { + const result = await readModel.queryFacts({ sessionId: safeId, families: WORKBENCH_SESSION_DETAIL_FAMILIES, limit: MAX_PAGE_LIMIT }); + if (result.error || result.durable === false) { + const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" }); + blocker.causeCode = result.error?.code ?? null; + blocker.causeName = result.error?.name ?? null; + blocker.message = result.error?.message ? `${blocker.message} Cause: ${result.error.message}` : result.durable === false ? `${blocker.message} Cause: durable facts reader unavailable.` : blocker.message; + return { facts: emptyFactSet(), session: null, blocker }; + } + const facts = result.facts ?? emptyFactSet(); + const session = visibleFactSessions(facts, actor).find((item) => factSessionId(item) === safeId) ?? null; + return { facts, session, blocker: null }; + } catch (error) { + const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" }); + blocker.causeCode = error?.code ?? null; + blocker.causeName = error?.name ?? null; + blocker.message = error?.message ? `${blocker.message} Cause: ${error.message}` : blocker.message; + return { facts: emptyFactSet(), session: null, blocker }; + } +} + +export async function writeTraceRealtimeSnapshot({ writeEvent, options, actor, traceId, reason }) { + const context = await visibleTraceContext(options, actor, traceId); + if (!context.visible) { + writeEvent("workbench.trace.unavailable", { + type: "trace.unavailable", + reason, + traceId, + error: { code: "workbench_trace_not_found", message: "Workbench trace is not visible to the current actor." } + }); + return; + } + if (context.factsReadBlocker) { + writeEvent("workbench.error", { + type: "error", + traceId, + reason, + error: context.factsReadBlocker + }); + } + const sessionId = factSessionId(context.factSession) ?? safeSessionId(context.session?.id) ?? null; + const threadId = safeOpaqueId(context.factSession?.threadId ?? context.session?.threadId) ?? (textValue(context.factSession?.threadId ?? context.session?.threadId) || null); + const realtimeTraceSnapshot = context.factsReadBlocker ? blockedRealtimeTraceSnapshot(context) : traceSnapshotForRealtime(context.trace); + const traceSeq = context.factsReadBlocker ? 0 : traceSnapshotLastSeq(context.trace); + writeEvent("workbench.trace.snapshot", { + type: "trace.snapshot", + reason, + sessionId, + threadId, + traceId, + snapshot: { ...realtimeTraceSnapshot, sessionId, threadId }, + cursor: { traceSeq } + }); + writeEvent("workbench.turn.snapshot", { + type: "turn.snapshot", + reason, + sessionId, + threadId, + traceId, + turn: realtimeTurnSnapshot(context), + cursor: { traceSeq } + }); +} + +export function realtimeTurnSnapshot(context) { + if (context?.factsReadBlocker) return blockedRealtimeTurnSnapshot(context, context.factsReadBlocker); + const facts = context?.facts; + const traceId = safeTraceId(context?.traceId); + if (traceId && facts && context?.factSession) { + const factTurn = factTurnForTrace(facts, traceId); + const projected = factTurnSnapshot({ turn: factTurn, session: context.factSession, facts, traceId, turnId: context.turnId }); + if (projected) return projected; + } + const blocker = traceEventsReadModelBlocker("workbench_facts_turn_missing", "Workbench durable facts did not contain a turn snapshot; realtime must surface the read-model gap instead of falling back to legacy trace projection.", { traceId, session: context?.factSession, route: "/v1/workbench/events" }); + return blockedRealtimeTurnSnapshot(context, blocker); +} + +export function blockedRealtimeTraceSnapshot(context) { + const traceId = safeTraceId(context?.traceId ?? context?.factsReadBlocker?.traceId) ?? null; + const blocker = context?.factsReadBlocker ?? traceEventsReadModelBlocker("workbench_facts_unavailable", "Workbench durable facts are unavailable for realtime trace snapshot.", { traceId, route: "/v1/workbench/events" }); + const projection = blockedTraceEventProjection({}, blocker); + return { + traceId, + status: "unknown", + traceStatus: "unknown", + events: [], + eventCount: 0, + fullTraceLoaded: true, + hasMore: false, + nextProjectedSeq: 0, + updatedAt: null, + projection, + projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, + staleMs: projection.staleMs ?? null, + blocker, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +export function blockedRealtimeTurnSnapshot(context, blocker) { + const traceId = safeTraceId(context?.traceId ?? blocker?.traceId) ?? null; + const turnId = safeTurnId(context?.turnId ?? blocker?.turnId) || traceId; + const sessionId = safeSessionId(blocker?.sessionId ?? factSessionId(context?.factSession) ?? context?.session?.id) ?? null; + const threadId = safeOpaqueId(context?.factSession?.threadId ?? context?.session?.threadId) ?? (textValue(context?.session?.threadId) || null); + const projection = blockedTraceEventProjection({}, blocker); + return { + turnId, + traceId, + status: "unknown", + running: false, + terminal: false, + sessionId, + threadId, + userMessageId: null, + assistantMessageId: null, + assistantText: null, + finalResponse: null, + timing: null, + startedAt: null, + lastEventAt: null, + finishedAt: null, + durationMs: null, + agentRun: null, + trace: { + traceId, + status: "unknown", + eventCount: 0, + updatedAt: null + }, + urls: { + self: turnId ? `/v1/workbench/turns/${encodeURIComponent(turnId)}` : null, + traceEvents: traceId ? `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` : null + }, + projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, + staleMs: projection.staleMs ?? null, + blocker, + projection, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +export async function writeTraceRealtimeSnapshotSafe(input) { + try { + await writeTraceRealtimeSnapshot(input); + } catch (error) { + input.writeEvent("workbench.error", { + type: "error", + traceId: input.traceId, + reason: input.reason, + error: { + code: "workbench_trace_snapshot_failed", + causeCode: error?.code ?? null, + retryable: error?.data?.retryable === true, + message: error?.message ?? "Workbench trace snapshot failed.", + valuesRedacted: true + } + }); + } +} + +export async function visibleTraceContext(options, actor, traceId) { + const readModel = createWorkbenchReadModel(options, actor); + if (typeof readModel.queryFacts !== "function") { + const projection = factProjectionForTrace(emptyFactSet(), traceId); + const factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench durable facts query is unavailable; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); + return { visible: true, turnId: traceId, traceId, status: "unknown", projection, result: null, session: null, trace: factTraceSnapshot(emptyFactSet(), traceId), facts: emptyFactSet(), factSession: null, factsReadBlocker }; + } + + let facts = emptyFactSet(); + let factsReadBlocker = null; + try { + const factResult = await readModel.queryFacts({ traceId, families: [...WORKBENCH_SESSION_DETAIL_FAMILIES, ...WORKBENCH_TRACE_EVENT_PAGE_FAMILIES], limit: MAX_PAGE_LIMIT }); + if (factResult.error || factResult.durable === false) { + const projection = factProjectionForTrace(facts, traceId); + factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); + factsReadBlocker.causeCode = factResult.error?.code ?? null; + factsReadBlocker.causeName = factResult.error?.name ?? null; + factsReadBlocker.message = factResult.error?.message ? `${factsReadBlocker.message} Cause: ${factResult.error.message}` : factResult.durable === false ? `${factsReadBlocker.message} Cause: durable facts reader unavailable.` : factsReadBlocker.message; + } else { + facts = factResult?.facts ?? facts; + } + } catch (error) { + const projection = factProjectionForTrace(facts, traceId); + factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); + factsReadBlocker.causeCode = error?.code ?? null; + factsReadBlocker.causeName = error?.name ?? null; + factsReadBlocker.message = error?.message ? `${factsReadBlocker.message} Cause: ${error.message}` : factsReadBlocker.message; + } + + const resolved = factsReadBlocker ? { session: null, facts } : await visibleFactSessionForTrace(readModel, facts, actor, traceId); + if (resolved.error) { + const projection = factProjectionForTrace(facts, traceId); + factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts session lookup failed; realtime must surface the read-model error instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); + factsReadBlocker.causeCode = resolved.error?.code ?? null; + factsReadBlocker.causeName = resolved.error?.name ?? null; + factsReadBlocker.message = resolved.error?.message ? `${factsReadBlocker.message} Cause: ${resolved.error.message}` : factsReadBlocker.message; + } else if (resolved.facts) { + facts = resolved.facts; + } + + const factSession = resolved.session ?? null; + const trace = factTraceSnapshot(facts, traceId); + const turn = factTurnForTrace(facts, traceId, traceId); + let projection = factsReadBlocker ? blockedTraceEventProjection(factProjectionForTrace(facts, traceId), factsReadBlocker) : factProjectionForTrace(facts, traceId); + const visible = Boolean(factsReadBlocker || factSession || turn || factCheckpointForTrace(facts, traceId) || trace.eventCount > 0); + if (!visible) { + factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_trace_missing", "Workbench realtime could not find durable facts for the requested trace; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" }); + projection = blockedTraceEventProjection(projection, factsReadBlocker); + } + if (!factsReadBlocker && !factSession) { + factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_session_missing", "Workbench durable facts did not contain the trace session; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, turn, route: "/v1/workbench/events" }); + projection = blockedTraceEventProjection(projection, factsReadBlocker); + } + const snapshot = factTurnSnapshot({ turn, session: factSession, facts, traceId, turnId: traceId }); + return { visible: true, turnId: snapshot.turnId ?? traceId, traceId, status: snapshot.status, projection, result: null, session: null, trace, facts, factSession, factsReadBlocker }; +} + +export function traceSnapshotForRealtime(snapshot) { + const events = Array.isArray(snapshot?.events) ? snapshot.events : []; + return { + ...snapshot, + events, + eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length, + lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +export function traceSnapshotSummary(snapshot) { + const events = Array.isArray(snapshot?.events) ? snapshot.events : []; + return { + traceId: snapshot?.traceId ?? null, + status: snapshot?.status ?? "missing", + eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length, + lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null, + updatedAt: snapshot?.updatedAt ?? null, + valuesRedacted: true + }; +} + +export function traceSnapshotLastSeq(snapshot) { + const events = Array.isArray(snapshot?.events) ? snapshot.events : []; + return events.reduce((max, event, index) => Math.max(max, eventSeq(event, index)), 0); +} diff --git a/internal/db/runtime-store-core.ts b/internal/db/runtime-store-core.ts new file mode 100644 index 00000000..07678476 --- /dev/null +++ b/internal/db/runtime-store-core.ts @@ -0,0 +1,1558 @@ +/* + * SPEC: PJ2026-010403 API契约 draft-2026-06-17-r0; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p1-zero-split-durable-realtime; draft-2026-06-24-p0-aggregate-event-stream; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract. + * 职责: Cloud runtime durable store。Code Agent trace/session/projection cursor 投影事实必须从同一持久化适配器写入和恢复。 + */ +import { createHash, randomUUID } from "node:crypto"; + +import { + ENVIRONMENT_DEV, + ERROR_CODES, + HwlabProtocolError, + assertProtocolRecord +} from "../protocol/index.mjs"; +import { + CLOUD_API_SERVICE_ID, + createProtocolAuditEvent, + deriveProtocolActorFromMeta +} from "../audit/index.mjs"; +import { + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS +} from "./schema.ts"; + +export const RUNTIME_STORE_KIND = "memory"; +export const RUNTIME_STORE_KIND_POSTGRES = "postgres"; +export const RUNTIME_DURABLE_ADAPTER_MISSING = "runtime_durable_adapter_missing"; +export const RUNTIME_DURABLE_ADAPTER_UNCONFIGURED = "runtime_durable_adapter_unconfigured"; +export const RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING = "runtime_durable_adapter_driver_missing"; +export const RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED = "runtime_durable_adapter_ssl_blocked"; +export const RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED = "runtime_durable_adapter_auth_blocked"; +export const RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED = "runtime_durable_adapter_schema_blocked"; +export const RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED = "runtime_durable_adapter_migration_blocked"; +export const RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED = "runtime_durable_adapter_query_blocked"; +export const RUNTIME_DURABILITY_REQUIRED_EVIDENCE = "runtime_adapter_schema_migration_read_query"; + +export const RUNTIME_ADAPTER_ENV = "HWLAB_CLOUD_RUNTIME_ADAPTER"; +export const RUNTIME_DURABLE_ENV = "HWLAB_CLOUD_RUNTIME_DURABLE"; +export const RUNTIME_DB_POOL_MAX_ENV = "HWLAB_CLOUD_DB_POOL_MAX"; + +export const requiredPostgresSchema = CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS; + +export const postgresCountTables = Object.freeze([ + "users", + "user_sessions", + "gateway_sessions", + "box_resources", + "box_capabilities", + "hardware_operations", + "audit_events", + "evidence_records", + "agent_sessions", + "account_workspaces", + "worker_sessions", + "agent_trace_events", + "workbench_projection_state", + "workbench_sessions", + "workbench_messages", + "workbench_parts", + "workbench_turns", + "workbench_trace_events", + "workbench_event_sequences", + "workbench_events", + "workbench_session_inputs", + "workbench_projection_checkpoints" +]); + +export const postgresRuntimeReadIndexes = Object.freeze([ + "CREATE INDEX IF NOT EXISTS idx_agent_trace_events_trace_order ON agent_trace_events(trace_id, occurred_at, id)", + "CREATE INDEX IF NOT EXISTS idx_agent_trace_events_session_order ON agent_trace_events(agent_session_id, occurred_at, id)", + "CREATE INDEX IF NOT EXISTS idx_agent_trace_events_worker_order ON agent_trace_events(worker_session_id, occurred_at, id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_projection_state_status_retry_updated ON workbench_projection_state(projection_status, next_retry_at, updated_at)", + "CREATE INDEX IF NOT EXISTS idx_workbench_projection_state_run_command ON workbench_projection_state(run_id, command_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_projection_state_session_updated ON workbench_projection_state(session_id, updated_at DESC)", + "CREATE INDEX IF NOT EXISTS idx_workbench_sessions_updated ON workbench_sessions(updated_at DESC, session_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_sessions_owner_updated ON workbench_sessions(owner_user_id, updated_at DESC, session_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_sessions_status_updated ON workbench_sessions(status, updated_at DESC, session_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_messages_session_updated ON workbench_messages(session_id, updated_at ASC, message_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_messages_trace ON workbench_messages(trace_id, message_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_messages_turn ON workbench_messages(turn_id, message_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_parts_message_index ON workbench_parts(message_id, part_index, part_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_parts_session_trace ON workbench_parts(session_id, trace_id, part_index)", + "CREATE INDEX IF NOT EXISTS idx_workbench_turns_session_updated ON workbench_turns(session_id, updated_at DESC, turn_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_turns_trace ON workbench_turns(trace_id, turn_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_trace_events_trace_projected_seq ON workbench_trace_events(trace_id, projected_seq)", + "CREATE INDEX IF NOT EXISTS idx_workbench_trace_events_session_projected_seq ON workbench_trace_events(session_id, projected_seq)", + "CREATE INDEX IF NOT EXISTS idx_workbench_event_sequences_session ON workbench_event_sequences(session_id, updated_at DESC, aggregate_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_event_sequences_trace ON workbench_event_sequences(trace_id, updated_at DESC, aggregate_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_events_trace_seq ON workbench_events(trace_id, event_seq)", + "CREATE INDEX IF NOT EXISTS idx_workbench_events_session_seq ON workbench_events(session_id, event_seq)", + "CREATE INDEX IF NOT EXISTS idx_workbench_events_type_seq ON workbench_events(event_type, event_seq)", + "CREATE INDEX IF NOT EXISTS idx_workbench_session_inputs_session_seq ON workbench_session_inputs(session_id, admitted_seq, input_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_session_inputs_trace ON workbench_session_inputs(trace_id, input_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_session_inputs_command ON workbench_session_inputs(command_id, input_id) WHERE command_id IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_workbench_projection_checkpoints_status_updated ON workbench_projection_checkpoints(projection_status, updated_at DESC, trace_id)", + "CREATE INDEX IF NOT EXISTS idx_workbench_projection_checkpoints_session_updated ON workbench_projection_checkpoints(session_id, updated_at DESC, trace_id)" +]); + +export function buildPostgresPoolConfig({ dbUrl, sslMode, timeoutMs, poolMax, queryTimeoutMs } = {}) { + const normalizedSslMode = normalizePostgresSslMode(sslMode ?? postgresSslModeFromConnectionString(dbUrl)); + const normalizedPoolMax = normalizeRuntimePoolMax(poolMax); + const normalizedQueryTimeoutMs = queryTimeoutMs === undefined || queryTimeoutMs === null || String(queryTimeoutMs).trim() === "" + ? null + : normalizeRuntimeTimeoutMs(queryTimeoutMs); + return { + connectionString: normalizePostgresConnectionStringForSslMode(dbUrl, normalizedSslMode), + ssl: postgresSslOption(normalizedSslMode), + connectionTimeoutMillis: normalizeRuntimeTimeoutMs(timeoutMs), + ...(normalizedQueryTimeoutMs !== null ? { query_timeout: normalizedQueryTimeoutMs, statement_timeout: normalizedQueryTimeoutMs } : {}), + ...(normalizedPoolMax !== null ? { max: normalizedPoolMax } : {}) + }; +} + +export const RUNTIME_DB_QUERY_TIMEOUT_MS_ENV = "HWLAB_CLOUD_DB_QUERY_TIMEOUT_MS"; +export const RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS"; +export const RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS"; +export const RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS"; +export const RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV = "HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS"; +export const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS = 5; +export const DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS = 250; +export const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS = 5_000; +export const DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS = 10_000; + +export function normalizePositiveInteger(value, fallback) { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? Math.trunc(number) : fallback; +} + +export function delayRuntimeDbQueryRetry(ms) { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.trunc(Number(ms) || 0)))); +} + +export function isReadyForDurableRuntimeRead(readiness = {}) { + return readiness.ready === true && readiness.durable === true && readiness.liveRuntimeEvidence === true; +} + +export function normalizeCapabilityInputs(params) { + const value = params.capabilities ?? params.capability ?? params; + if (Array.isArray(value)) { + if (value.length === 0) { + throw new HwlabProtocolError("capabilities must not be empty", { + code: ERROR_CODES.invalidParams + }); + } + return value.map((item) => asObject(item, "capability")); + } + return [asObject(value, "capability")]; +} + +export function normalizeShellInput(params) { + const input = normalizeJsonObject(params.input); + const command = params.command ?? input.command; + if (command === undefined || command === null || command === "") { + throw new HwlabProtocolError("hardware.invoke.shell requires input.command or command", { + code: ERROR_CODES.invalidParams, + data: { + required: ["projectId", "gatewaySessionId", "resourceId", "capabilityId", "input.command"] + } + }); + } + if (Array.isArray(command)) { + return { + command: command.map((item) => String(item)), + cwd: input.cwd, + timeoutMs: input.timeoutMs + }; + } + return { + command: String(command), + cwd: input.cwd, + timeoutMs: input.timeoutMs + }; +} + +export function normalizeAgentTraceEvent(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); + if (!traceId) { + throw new HwlabProtocolError("agent trace event requires traceId", { + code: ERROR_CODES.invalidParams, + data: { required: ["traceId"] } + }); + } + const seq = positiveIntegerOrNull(input.seq); + const occurredAt = timestampOr(input.occurredAt ?? input.createdAt ?? input.timestamp, now); + const status = textOr(input.status, "observed"); + const type = textOr(input.type ?? input.kind, "event"); + const level = traceEventLevel(input.level ?? status ?? type); + const message = textOr(input.message ?? input.label ?? type, ""); + const agentSessionId = textOr(input.agentSessionId ?? input.sessionId ?? requestMeta.agentSessionId ?? requestMeta.sessionId, ""); + const workerSessionId = textOr(input.workerSessionId ?? input.workerSession?.id ?? input.workerId, ""); + const id = textOr(input.id ?? input.eventId, "") || `tev_${sha256Hex({ + traceId, + seq, + source: input.source ?? null, + sourceSeq: input.sourceSeq ?? null, + type, + status, + label: input.label ?? null, + message, + occurredAt + }).slice(0, 32)}`; + const event = pruneUndefined({ + ...input, + traceId, + seq: seq ?? undefined, + agentSessionId: agentSessionId || undefined, + workerSessionId: workerSessionId || undefined, + level, + createdAt: timestampOr(input.createdAt ?? occurredAt, occurredAt), + valuesPrinted: false + }); + return { + id, + traceId, + agentSessionId: agentSessionId || null, + workerSessionId: workerSessionId || null, + level, + message, + occurredAt, + event + }; +} + +export function normalizeWorkbenchProjectionState(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); + const runId = textOr(input.runId ?? input.sourceRunId, ""); + const commandId = textOr(input.commandId ?? input.sourceCommandId, ""); + if (!traceId || !runId || !commandId) { + throw new HwlabProtocolError("workbench projection state requires traceId, runId, and commandId", { + code: ERROR_CODES.invalidParams, + data: { required: ["traceId", "runId", "commandId"] } + }); + } + const timestamp = timestampOr(input.updatedAt, now); + const lastAgentRunSeq = nonNegativeInteger(input.lastAgentRunSeq ?? input.lastSourceSeq); + const lastProjectedSeq = nonNegativeInteger(input.lastProjectedSeq ?? lastAgentRunSeq); + const upstreamLatestSeq = nonNegativeIntegerOrNull(input.upstreamLatestSeq ?? input.sourceLatestSeq); + return pruneUndefined({ + ...input, + traceId, + sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, + conversationId: textOr(input.conversationId, "") || null, + threadId: textOr(input.threadId, "") || null, + ownerUserId: textOr(input.ownerUserId, "") || null, + ownerRole: textOr(input.ownerRole, "") || null, + sourceRunId: runId, + sourceCommandId: commandId, + runId, + commandId, + lastSourceSeq: lastAgentRunSeq, + lastAgentRunSeq, + lastProjectedSeq, + sourceLatestSeq: upstreamLatestSeq ?? lastAgentRunSeq, + upstreamLatestSeq: upstreamLatestSeq ?? lastAgentRunSeq, + projectionStatus: projectionStatusOr(input.projectionStatus, "projecting"), + projectionHealth: projectionHealthOr(input.projectionHealth, "healthy"), + resultSyncState: resultSyncStateOr(input.resultSyncState, "not_started"), + lastProjectedAt: timestampOr(input.lastProjectedAt, timestamp), + lastResultSyncAt: input.lastResultSyncAt ? timestampOr(input.lastResultSyncAt, timestamp) : null, + lastErrorCode: textOr(input.lastErrorCode, "") || null, + lastErrorMessage: textOr(input.lastErrorMessage, "") || null, + failureCount: nonNegativeInteger(input.failureCount), + nextRetryAt: input.nextRetryAt ? timestampOr(input.nextRetryAt, timestamp) : null, + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false + }); +} + +export function normalizeWorkbenchProjectionAllocation(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); + const sourceEventId = textOr(input.sourceEventId ?? input.eventId, ""); + if (!traceId || !sourceEventId) { + throw new HwlabProtocolError("workbench projectedSeq allocation requires traceId and sourceEventId", { + code: ERROR_CODES.invalidParams, + data: { required: ["traceId", "sourceEventId"] } + }); + } + const timestamp = timestampOr(input.updatedAt ?? input.occurredAt ?? input.createdAt, now); + return pruneUndefined({ + ...input, + traceId, + sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, + turnId: textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null, + runId: textOr(input.runId ?? input.sourceRunId, "") || null, + commandId: textOr(input.commandId ?? input.sourceCommandId, "") || null, + sourceSeq: nonNegativeInteger(input.sourceSeq ?? input.lastSourceSeq ?? input.lastAgentRunSeq), + sourceEventId, + eventType: textOr(input.eventType ?? input.type, "event"), + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false + }); +} + +export function normalizeWorkbenchFacts(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + return { + inputs: arrayInput(input.inputs ?? input.input).map((item) => normalizeWorkbenchSessionInputFact(item, requestMeta, now)), + sessions: arrayInput(input.sessions ?? input.session).map((item) => normalizeWorkbenchSessionFact(item, requestMeta, now)), + messages: arrayInput(input.messages ?? input.message).map((item) => normalizeWorkbenchMessageFact(item, requestMeta, now)), + parts: arrayInput(input.parts ?? input.part).map((item) => normalizeWorkbenchPartFact(item, requestMeta, now)), + turns: arrayInput(input.turns ?? input.turn).map((item) => normalizeWorkbenchTurnFact(item, requestMeta, now)), + traceEvents: arrayInput(input.traceEvents ?? input.traceEvent).map((item) => normalizeWorkbenchTraceEventFact(item, requestMeta, now)), + checkpoints: arrayInput(input.checkpoints ?? input.checkpoint).map((item) => normalizeWorkbenchProjectionCheckpoint(item, requestMeta, now)) + }; +} + +export function normalizeWorkbenchAggregateEventsForFacts(facts = {}, requestMeta = {}, now) { + const events = []; + for (const fact of arrayInput(facts.inputs)) appendWorkbenchFactEvent(events, "inputs", fact, requestMeta, now); + for (const fact of arrayInput(facts.sessions)) appendWorkbenchFactEvent(events, "sessions", fact, requestMeta, now); + for (const fact of arrayInput(facts.messages)) appendWorkbenchFactEvent(events, "messages", fact, requestMeta, now); + for (const fact of arrayInput(facts.parts)) appendWorkbenchFactEvent(events, "parts", fact, requestMeta, now); + for (const fact of arrayInput(facts.traceEvents)) appendWorkbenchFactEvent(events, "traceEvents", fact, requestMeta, now); + for (const fact of arrayInput(facts.checkpoints)) appendWorkbenchFactEvent(events, "checkpoints", fact, requestMeta, now); + for (const fact of arrayInput(facts.turns)) appendWorkbenchFactEvent(events, "turns", fact, requestMeta, now); + return events; +} + +export function appendWorkbenchFactEvent(events, factFamily, fact, requestMeta, now) { + const aggregate = workbenchAggregateForFact(fact, requestMeta); + const factId = textOr(fact.id ?? fact.inputId ?? fact.sessionId ?? fact.messageId ?? fact.partId ?? fact.turnId ?? fact.traceId, ""); + const rawSourceEventId = textOr(fact.sourceEventId ?? fact.eventId, "") || factId; + const sourceEventId = rawSourceEventId ? `${factFamily}:${rawSourceEventId}` : null; + const eventType = workbenchAggregateEventType(factFamily, fact); + const projectedSeq = nonNegativeInteger(fact.projectedSeq); + const sourceSeq = nonNegativeInteger(fact.sourceSeq); + const occurredAt = timestampOr(fact.occurredAt ?? fact.updatedAt ?? fact.createdAt, now); + events.push(normalizeWorkbenchAggregateEvent({ + ...aggregate, + factFamily, + factId, + factKey: `${factFamily.slice(0, -1) || factFamily}:${factId}`, + eventId: stableWorkbenchFactId("wbe", { aggregateId: aggregate.aggregateId, factFamily, sourceEventId, eventType, factId }), + messageId: textOr(fact.messageId, "") || null, + sourceRunId: textOr(fact.sourceRunId ?? fact.runId, "") || null, + sourceCommandId: textOr(fact.sourceCommandId ?? fact.commandId, "") || null, + sourceSeq, + sourceEventId, + eventType, + projectionRevision: projectedSeq || sourceSeq, + terminal: Boolean(fact.terminal), + sealed: Boolean(fact.sealed), + payload: { + factFamily, + factId, + status: textOr(fact.status ?? fact.projectionStatus, "") || null, + eventType, + projectedSeq, + sourceSeq, + originalSourceEventId: rawSourceEventId || null, + valuesRedacted: true + }, + occurredAt, + committedAt: timestampOr(fact.updatedAt ?? occurredAt, now) + }, requestMeta, now)); +} + +export function workbenchAggregateForFact(fact = {}, requestMeta = {}) { + const sessionId = textOr(fact.sessionId ?? requestMeta.sessionId, "") || null; + const turnId = textOr(fact.turnId ?? requestMeta.turnId, "") || null; + const traceId = textOr(fact.traceId ?? requestMeta.traceId, "") || null; + if (sessionId) return { aggregateId: `session:${sessionId}`, aggregateType: "session", sessionId, turnId, traceId }; + if (turnId) return { aggregateId: `turn:${turnId}`, aggregateType: "turn", sessionId, turnId, traceId }; + if (traceId) return { aggregateId: `trace:${traceId}`, aggregateType: "trace", sessionId, turnId, traceId }; + throw requiredWorkbenchFactError("workbench aggregate event", ["sessionId", "turnId", "traceId"]); +} + +export function normalizeWorkbenchAggregateEvent(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const aggregateId = textOr(input.aggregateId, ""); + if (!aggregateId) throw requiredWorkbenchFactError("workbench aggregate event", ["aggregateId"]); + const timestamp = timestampOr(input.committedAt ?? input.occurredAt ?? input.updatedAt ?? input.createdAt, now); + const sourceSeq = nonNegativeInteger(input.sourceSeq); + const projectionRevision = nonNegativeInteger(input.projectionRevision ?? input.projectedSeq ?? sourceSeq); + return pruneUndefined({ + ...input, + eventSeq: nonNegativeInteger(input.eventSeq), + eventId: textOr(input.eventId ?? input.id, "") || stableWorkbenchFactId("wbe", { aggregateId, eventType: input.eventType, sourceEventId: input.sourceEventId, sourceSeq, projectionRevision }), + aggregateId, + aggregateType: textOr(input.aggregateType, "trace"), + aggregateSeq: nonNegativeInteger(input.aggregateSeq), + sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, + turnId: textOr(input.turnId ?? requestMeta.turnId, "") || null, + traceId: textOr(input.traceId ?? requestMeta.traceId, "") || null, + messageId: textOr(input.messageId ?? requestMeta.messageId, "") || null, + sourceRunId: textOr(input.sourceRunId ?? input.runId, "") || null, + sourceCommandId: textOr(input.sourceCommandId ?? input.commandId, "") || null, + sourceSeq, + sourceEventId: textOr(input.sourceEventId, "") || null, + eventType: textOr(input.eventType ?? input.type, "event"), + projectionRevision, + terminal: Boolean(input.terminal), + sealed: Boolean(input.sealed), + payload: normalizeJsonObject(input.payload), + occurredAt: timestampOr(input.occurredAt, timestamp), + committedAt: timestamp, + valuesPrinted: false + }); +} + +export function workbenchAggregateEventType(factFamily, fact = {}) { + if (factFamily === "inputs") return `input_${textOr(fact.delivery, "queue")}_${textOr(fact.status, "admitted")}`; + if (factFamily === "sessions") return "session_upsert"; + if (factFamily === "messages") return `${textOr(fact.role, "agent")}_message`; + if (factFamily === "parts") return `part_${textOr(fact.partType ?? fact.type, "text")}`; + if (factFamily === "turns") return fact.terminal || fact.sealed ? "turn_terminal" : "turn_update"; + if (factFamily === "traceEvents") return `trace_${textOr(fact.eventType ?? fact.type, "event")}`; + if (factFamily === "checkpoints") return `checkpoint_${textOr(fact.projectionStatus, "projecting")}`; + return "event"; +} + +export function workbenchInputDelivery(value) { + const text = textOr(value, "queue").toLowerCase(); + return ["queue", "steer", "cancel"].includes(text) ? text : "queue"; +} + +export function workbenchInputStatus(value) { + const text = textOr(value, "admitted").toLowerCase().replace(/-/gu, "_"); + return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; +} + +export function indexWorkbenchAggregateEvents(events = []) { + const index = new Map(); + for (const event of events) { + if (event?.factKey) index.set(event.factKey, event); + if (event?.factFamily && event?.factId) index.set(`${event.factFamily}:${event.factId}`, event); + if (event?.factFamily === "inputs" && event?.factId) index.set(`input:${event.factId}`, event); + if (event?.factFamily === "turns" && event?.turnId) index.set(`turn:${event.turnId}`, event); + if (event?.factFamily === "traceEvents" && event?.factId) index.set(`traceEvent:${event.factId}`, event); + if (event?.sourceEventId) index.set(`sourceEvent:${event.sourceEventId}`, event); + } + return index; +} + +export function normalizeWorkbenchSessionInputFact(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); + if (!sessionId) throw requiredWorkbenchFactError("workbench session input fact", ["sessionId"]); + const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); + const turnId = textOr(input.turnId ?? requestMeta.turnId ?? input.traceId ?? requestMeta.traceId, "") || null; + const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; + const messageId = textOr(input.messageId ?? requestMeta.messageId, "") || null; + const commandId = textOr(input.commandId ?? input.sourceCommandId, "") || null; + const delivery = workbenchInputDelivery(input.delivery); + const status = workbenchInputStatus(input.status); + const sourceEventId = textOr(input.sourceEventId ?? input.eventId, "") || `${sessionId}:${turnId ?? "turn-unassigned"}:${delivery}:${traceId ?? commandId ?? messageId ?? "input"}`; + const inputId = textOr(input.inputId ?? input.id, "") || stableWorkbenchFactId("wsi", { sessionId, turnId, traceId, messageId, commandId, delivery, sourceEventId }); + const admittedSeq = nonNegativeInteger(input.admittedSeq ?? input.aggregateSeq ?? input.sourceSeq); + const promotedSeq = workbenchSessionInputPromotedSeq({ ...input, status }, admittedSeq); + return pruneUndefined({ + ...input, + id: inputId, + inputId, + sessionId, + turnId, + traceId, + messageId, + commandId, + delivery, + admittedSeq, + promotedSeq, + sourceSeq: admittedSeq, + projectedSeq: admittedSeq, + status, + errorCode: textOr(input.errorCode ?? input.error?.code, "") || null, + sourceEventId, + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false, + valuesRedacted: true + }); +} + +export function memoryWorkbenchSessionInputWithSeq(fact, records) { + const existingFactSeq = nonNegativeInteger(fact.admittedSeq); + if (existingFactSeq > 0) return workbenchSessionInputWithSeq(fact, existingFactSeq); + const existing = records.get(fact.inputId) ?? null; + const existingSeq = nonNegativeInteger(existing?.admittedSeq); + if (existingSeq > 0) return workbenchSessionInputWithSeq(fact, existingSeq); + const nextSeq = [...records.values()] + .filter((record) => record.sessionId === fact.sessionId) + .reduce((max, record) => Math.max(max, nonNegativeInteger(record.admittedSeq)), 0) + 1; + return workbenchSessionInputWithSeq(fact, nextSeq); +} + +export function inputFactWithAggregateSeq(fact, eventIndex) { + const existingFactSeq = nonNegativeInteger(fact.admittedSeq); + if (existingFactSeq > 0) return workbenchSessionInputWithSeq(fact, existingFactSeq); + const event = eventIndex.get(`input:${fact.inputId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + const admittedSeq = nonNegativeInteger(event?.aggregateSeq); + return admittedSeq > 0 ? workbenchSessionInputWithSeq(fact, admittedSeq) : fact; +} + +export function workbenchSessionInputWithSeq(fact, admittedSeq) { + return { + ...fact, + admittedSeq, + promotedSeq: workbenchSessionInputPromotedSeq(fact, admittedSeq), + sourceSeq: admittedSeq, + projectedSeq: admittedSeq + }; +} + +export function workbenchSessionInputPromotedSeq(fact, admittedSeq) { + const existing = nonNegativeIntegerOrNull(fact.promotedSeq); + if (existing !== null && existing > 0) return existing; + const seq = nonNegativeInteger(admittedSeq); + return textOr(fact.status, "") === "promoted" && seq > 0 ? seq : existing; +} + +export function normalizeWorkbenchSessionFact(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const sessionId = textOr(input.sessionId ?? input.id ?? requestMeta.sessionId, ""); + if (!sessionId) throw requiredWorkbenchFactError("workbench session fact", ["sessionId"]); + const sourceSeq = nonNegativeInteger(input.sourceSeq); + const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); + const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); + return pruneUndefined({ + ...input, + id: sessionId, + sessionId, + ownerUserId: textOr(input.ownerUserId ?? requestMeta.ownerUserId, "") || null, + projectId: textOr(input.projectId ?? requestMeta.projectId, "") || null, + conversationId: textOr(input.conversationId ?? requestMeta.conversationId, "") || null, + threadId: textOr(input.threadId ?? requestMeta.threadId, "") || null, + status: textOr(input.status, "unknown"), + lastTraceId: textOr(input.lastTraceId ?? input.traceId ?? requestMeta.traceId, "") || null, + projectedSeq, + sourceSeq, + sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, + terminal: Boolean(input.terminal), + sealed: Boolean(input.sealed), + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false + }); +} + +export function normalizeWorkbenchMessageFact(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); + if (!sessionId) throw requiredWorkbenchFactError("workbench message fact", ["sessionId"]); + const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; + const turnId = textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null; + const role = textOr(input.role, "agent"); + const sourceSeq = nonNegativeInteger(input.sourceSeq); + const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); + const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); + const messageId = textOr(input.messageId ?? input.id, "") || stableWorkbenchFactId("msg", { sessionId, turnId, traceId, role, sourceSeq, projectedSeq }); + return pruneUndefined({ + ...input, + id: messageId, + messageId, + sessionId, + turnId, + traceId, + role, + status: textOr(input.status, "unknown"), + projectedSeq, + sourceSeq, + sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, + terminal: Boolean(input.terminal), + sealed: Boolean(input.sealed), + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false + }); +} + +export function normalizeWorkbenchPartFact(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); + const messageId = textOr(input.messageId ?? requestMeta.messageId, ""); + if (!sessionId || !messageId) throw requiredWorkbenchFactError("workbench part fact", ["sessionId", "messageId"]); + const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; + const turnId = textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null; + const partIndex = nonNegativeInteger(input.partIndex ?? input.index); + const sourceSeq = nonNegativeInteger(input.sourceSeq); + const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); + const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); + const partType = textOr(input.partType ?? input.type, "text"); + const partId = textOr(input.partId ?? input.id, "") || stableWorkbenchFactId("wpt", { sessionId, messageId, turnId, traceId, partIndex, partType }); + return pruneUndefined({ + ...input, + id: partId, + partId, + messageId, + sessionId, + turnId, + traceId, + partIndex, + partType, + status: textOr(input.status, "unknown"), + projectedSeq, + sourceSeq, + sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, + terminal: Boolean(input.terminal), + sealed: Boolean(input.sealed), + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false + }); +} + +export function normalizeWorkbenchTurnFact(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); + const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; + const turnId = textOr(input.turnId ?? input.id ?? requestMeta.turnId ?? traceId, ""); + if (!sessionId || !turnId) throw requiredWorkbenchFactError("workbench turn fact", ["sessionId", "turnId"]); + const sourceSeq = nonNegativeInteger(input.sourceSeq); + const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); + const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); + const status = textOr(input.status, "unknown"); + const terminal = Boolean(input.terminal); + const sealed = Boolean(input.sealed); + const finalResponse = input.finalResponse ?? null; + assertWorkbenchTurnFinalResponseInvariant({ status, terminal, sealed, finalResponse }); + return pruneUndefined({ + ...input, + id: turnId, + turnId, + sessionId, + traceId, + messageId: textOr(input.messageId ?? requestMeta.messageId, "") || null, + status, + projectedSeq, + sourceSeq, + sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, + terminal, + sealed, + finalResponse, + diagnostic: normalizeJsonObject(input.diagnostic ?? input.projection ?? input.blocker), + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false + }); +} + +export function assertWorkbenchTurnFinalResponseInvariant({ status, terminal, sealed, finalResponse } = {}) { + if (normalizeWorkbenchFactStatus(status) !== "completed") return; + if (!terminal && !sealed) return; + if (workbenchFactFinalResponseText(finalResponse)) return; + const error = new Error("completed Workbench turn facts must include an authoritative final response before sealing"); + error.code = "workbench_terminal_final_response_missing"; + error.layer = "workbench-read-model"; + error.category = "projection-invariant"; + throw error; +} + +export function normalizeWorkbenchFactStatus(value) { + return textOr(value, "").toLowerCase().replace(/_/gu, "-"); +} + +export function workbenchFactFinalResponseText(value) { + if (value && typeof value === "object" && !Array.isArray(value)) { + return workbenchFactFinalResponseText(value.text ?? value.content ?? value.message ?? value.summary ?? value.preview); + } + const text = textOr(value, ""); + return text && text !== "[object Object]" ? text : null; +} + +export function normalizeWorkbenchTraceEventFact(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); + if (!traceId) throw requiredWorkbenchFactError("workbench trace event fact", ["traceId"]); + const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, "") || null; + const turnId = textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null; + const sourceSeq = nonNegativeInteger(input.sourceSeq); + const projectedSeq = nonNegativeInteger(input.projectedSeq ?? input.seq); + const occurredAt = timestampOr(input.occurredAt ?? input.createdAt ?? input.timestamp, now); + const eventType = textOr(input.eventType ?? input.type ?? input.kind, "event"); + const id = textOr(input.id ?? input.eventId, "") || stableWorkbenchFactId("wte", { traceId, sourceSeq, projectedSeq, eventType, sourceEventId: input.sourceEventId ?? null, occurredAt }); + return pruneUndefined({ + ...input, + id, + traceId, + sessionId, + turnId, + messageId: textOr(input.messageId ?? requestMeta.messageId, "") || null, + sourceSeq, + sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, + projectedSeq, + eventType, + terminal: Boolean(input.terminal), + sealed: Boolean(input.sealed), + occurredAt, + updatedAt: timestampOr(input.updatedAt, occurredAt), + valuesPrinted: false + }); +} + +export function normalizeWorkbenchProjectionCheckpoint(value, requestMeta = {}, now) { + const input = normalizeJsonObject(value); + const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); + if (!traceId) throw requiredWorkbenchFactError("workbench projection checkpoint", ["traceId"]); + const sourceSeq = nonNegativeInteger(input.sourceSeq ?? input.lastSourceSeq ?? input.lastAgentRunSeq); + const projectedSeq = nonNegativeInteger(input.projectedSeq ?? input.lastProjectedSeq ?? sourceSeq); + const timestamp = timestampOr(input.updatedAt ?? input.lastProjectedAt, now); + return pruneUndefined({ + ...input, + id: traceId, + traceId, + sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, + turnId: textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null, + runId: textOr(input.runId ?? input.sourceRunId, "") || null, + commandId: textOr(input.commandId ?? input.sourceCommandId, "") || null, + projectedSeq, + sourceSeq, + sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, + projectionStatus: projectionStatusOr(input.projectionStatus, "projecting"), + projectionHealth: projectionHealthOr(input.projectionHealth, "healthy"), + terminal: Boolean(input.terminal), + sealed: Boolean(input.sealed), + diagnostic: normalizeJsonObject(input.diagnostic ?? input.blocker), + createdAt: timestampOr(input.createdAt, timestamp), + updatedAt: timestamp, + valuesPrinted: false + }); +} + +export function arrayInput(value) { + if (Array.isArray(value)) return value; + if (value && typeof value === "object") return [value]; + return []; +} + +export function stableWorkbenchFactId(prefix, value) { + return `${prefix}_${sha256Hex(value).slice(0, 32)}`; +} + +export function requiredWorkbenchFactError(label, fields) { + return new HwlabProtocolError(`${label} requires ${fields.join(", ")}`, { + code: ERROR_CODES.invalidParams, + data: { required: fields } + }); +} + +export function compareTraceEventRecords(left, right) { + const leftSeq = positiveIntegerOrNull(left?.event?.seq); + const rightSeq = positiveIntegerOrNull(right?.event?.seq); + if (leftSeq && rightSeq && leftSeq !== rightSeq) return leftSeq - rightSeq; + return String(left?.occurredAt ?? "").localeCompare(String(right?.occurredAt ?? "")) || + String(left?.id ?? "").localeCompare(String(right?.id ?? "")); +} + +export function compareProjectionStateRecords(left, right) { + return String(left?.updatedAt ?? "").localeCompare(String(right?.updatedAt ?? "")) || + String(left?.traceId ?? "").localeCompare(String(right?.traceId ?? "")); +} + +export function compareWorkbenchFactRecords(left, right) { + return String(left?.updatedAt ?? left?.createdAt ?? "").localeCompare(String(right?.updatedAt ?? right?.createdAt ?? "")) || + String(left?.id ?? left?.sessionId ?? left?.messageId ?? left?.turnId ?? left?.traceId ?? "").localeCompare(String(right?.id ?? right?.sessionId ?? right?.messageId ?? right?.turnId ?? right?.traceId ?? "")); +} + +export function compareWorkbenchInputFactRecords(left, right) { + return nonNegativeInteger(left?.admittedSeq) - nonNegativeInteger(right?.admittedSeq) || + String(left?.inputId ?? left?.id ?? "").localeCompare(String(right?.inputId ?? right?.id ?? "")); +} + +export function compareWorkbenchPartFactRecords(left, right) { + return String(left?.messageId ?? "").localeCompare(String(right?.messageId ?? "")) || + nonNegativeInteger(left?.partIndex) - nonNegativeInteger(right?.partIndex) || + compareWorkbenchFactRecords(left, right); +} + +export function compareWorkbenchTraceEventFacts(left, right) { + return nonNegativeInteger(left?.projectedSeq) - nonNegativeInteger(right?.projectedSeq); +} + +export function sortWorkbenchFactRows(records, params = {}, family, fallbackCompare) { + const sorted = [...records]; + if (workbenchFactOrder(params, family) === "updated_desc") { + return sorted.sort((left, right) => String(right?.updatedAt ?? right?.createdAt ?? "").localeCompare(String(left?.updatedAt ?? left?.createdAt ?? "")) || fallbackCompare(left, right)); + } + if (family === "inputs") return sorted.sort(compareWorkbenchInputFactRecords); + return sorted.sort(fallbackCompare); +} + +export const WORKBENCH_FACT_FAMILIES = Object.freeze(["inputs", "sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"]); + +export function workbenchFactFamilySet(params = {}) { + const raw = params.families ?? params.factFamilies; + if (raw === undefined || raw === null || raw === "") return new Set(WORKBENCH_FACT_FAMILIES); + const values = Array.isArray(raw) ? raw : String(raw).split(","); + const selected = values.map((value) => String(value ?? "").trim()).filter((value) => WORKBENCH_FACT_FAMILIES.includes(value)); + return new Set(selected.length > 0 ? selected : WORKBENCH_FACT_FAMILIES); +} + +export function workbenchFactFamilyForTable(table) { + if (table === "workbench_session_inputs") return "inputs"; + if (table === "workbench_sessions") return "sessions"; + if (table === "workbench_messages") return "messages"; + if (table === "workbench_parts") return "parts"; + if (table === "workbench_turns") return "turns"; + if (table === "workbench_trace_events") return "traceEvents"; + if (table === "workbench_projection_checkpoints") return "checkpoints"; + return "unknown"; +} + +export function workbenchSessionSummarySelectClause() { + return [ + "session_id", + "owner_user_id", + "project_id", + "conversation_id", + "thread_id", + "status", + "last_trace_id", + "projected_seq", + "source_seq", + "source_event_id", + "terminal", + "sealed", + "created_at", + "updated_at", + "COALESCE(NULLIF(session_json::jsonb ->> 'providerProfile', ''), NULLIF(session_json::jsonb #>> '{sessionJson,providerProfile}', '')) AS provider_profile" + ].join(", "); +} + +export function workbenchSessionSummaryFactFromRow(row = {}) { + const sessionId = textOr(row.session_id, ""); + if (!sessionId) return null; + const providerProfile = textOr(row.provider_profile, ""); + return { + id: sessionId, + sessionId, + ownerUserId: textOr(row.owner_user_id, null), + projectId: textOr(row.project_id, null), + conversationId: textOr(row.conversation_id, null), + threadId: textOr(row.thread_id, null), + status: textOr(row.status, "unknown"), + lastTraceId: textOr(row.last_trace_id, null), + projectedSeq: nonNegativeInteger(row.projected_seq), + sourceSeq: nonNegativeInteger(row.source_seq), + sourceEventId: textOr(row.source_event_id, null), + terminal: row.terminal === true || row.terminal === "t", + sealed: row.sealed === true || row.sealed === "t", + ...(providerProfile ? { + providerProfile, + sessionJson: { + providerProfile, + valuesRedacted: true, + secretMaterialStored: false + } + } : {}), + createdAt: textOr(row.created_at, null), + updatedAt: textOr(row.updated_at, row.created_at ?? null), + valuesPrinted: false, + valuesRedacted: true + }; +} + +export function workbenchFactOrder(params = {}, family = "") { + const value = params[`${family}Order`] ?? params.order; + return value === "updated_desc" ? "updated_desc" : "updated_asc"; +} + +export function traceEventLevel(value) { + const text = textOr(value, "info").toLowerCase(); + if (["failed", "failure", "error", "timeout"].includes(text)) return "error"; + if (["warn", "warning", "blocked", "canceled", "cancelled"].includes(text)) return "warn"; + if (["debug", "trace"].includes(text)) return "debug"; + return "info"; +} + +export function positiveIntegerOrNull(value) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +} + +export function nonNegativeInteger(value) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +export function nonNegativeIntegerOrNull(value) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : null; +} + +export function projectionStatusOr(value, fallback) { + const text = textOr(value, fallback).toLowerCase(); + return ["projecting", "caught_up", "terminal", "degraded", "blocked", "stalled"].includes(text) ? text : fallback; +} + +export function projectionHealthOr(value, fallback) { + const text = textOr(value, fallback).toLowerCase(); + return ["healthy", "degraded", "unavailable", "stalled"].includes(text) ? text : fallback; +} + +export function resultSyncStateOr(value, fallback) { + const text = textOr(value, fallback).toLowerCase(); + return ["not_started", "pending", "synced", "failed", "timed_out"].includes(text) ? text : fallback; +} + +export function timestampOr(value, fallback) { + const text = textOr(value, ""); + if (text) { + const ms = Date.parse(text); + if (Number.isFinite(ms)) return new Date(ms).toISOString(); + } + return textOr(fallback, "") || new Date().toISOString(); +} + +export function textOr(value, fallback) { + const text = String(value ?? "").trim(); + return text || fallback; +} + +export function textList(value) { + const raw = Array.isArray(value) ? value : []; + return [...new Set(raw.map((item) => textOr(item, "")).filter(Boolean))]; +} + +export function asObject(value, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new HwlabProtocolError(`${label} must be a JSON object`, { + code: ERROR_CODES.invalidParams + }); + } + return value; +} + +export function requireString(input, field) { + if (typeof input[field] !== "string" || input[field].trim() === "") { + throw new HwlabProtocolError(`${field} is required`, { + code: ERROR_CODES.invalidParams, + data: { field } + }); + } + return input[field]; +} + +export function requireProtocolId(input, field) { + const value = requireString(input, field); + return value; +} + +export function normalizeJsonObject(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return {}; + } + return { ...value }; +} + +export function pruneUndefined(input) { + return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)); +} + +export function makeId(prefix) { + return `${prefix}_${randomUUID()}`; +} + +export function stableJson(value) { + return JSON.stringify(sortJson(value)); +} + +export function sha256Hex(value) { + return createHash("sha256").update(stableJson(value)).digest("hex"); +} + +export function sortJson(value) { + if (Array.isArray(value)) { + return value.map(sortJson); + } + if (value && typeof value === "object") { + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sortJson(value[key])])); + } + return value; +} + +export function matchesQuery(record, params, fields) { + for (const field of fields) { + if (params[field] !== undefined && params[field] !== null && record[field] !== params[field]) { + return false; + } + const values = textList(params[`${field}s`]); + if (values.length > 0 && !values.includes(textOr(record[field], ""))) { + return false; + } + } + return true; +} + +export function matchesWorkbenchSessionFactQuery(record, params) { + if (!matchesQuery(record, params, ["sessionId", "ownerUserId", "projectId", "conversationId", "threadId", "status"])) return false; + const traceId = textOr(params.traceId ?? params.lastTraceId, ""); + if (traceId && record.lastTraceId !== traceId && record.traceId !== traceId) return false; + return true; +} + +export function limitResults(records, limit) { + const parsed = Number.parseInt(limit ?? "", 10); + if (!Number.isInteger(parsed) || parsed <= 0) { + return records; + } + return records.slice(0, parsed); +} + +export function normalizeRuntimeAdapter(value) { + if (typeof value !== "string" || value.trim() === "") { + return RUNTIME_STORE_KIND; + } + const normalized = value.trim().toLowerCase(); + if (["postgres", "postgresql", "pg", "db"].includes(normalized)) { + return RUNTIME_STORE_KIND_POSTGRES; + } + return RUNTIME_STORE_KIND; +} + +export function normalizePostgresSslMode(value) { + const normalized = String(value ?? "").trim().toLowerCase(); + if (["disable", "false", "0", "off", "no"].includes(normalized)) { + return "disable"; + } + return "require"; +} + +export function postgresSslModeFromConnectionString(dbUrl) { + if (typeof dbUrl !== "string" || dbUrl.trim() === "") return undefined; + try { + const url = new URL(dbUrl); + if (!["postgres:", "postgresql:"].includes(url.protocol)) return undefined; + return url.searchParams.get("sslmode") ?? url.searchParams.get("ssl") ?? undefined; + } catch { + return undefined; + } +} + +export function postgresSslOption(sslMode) { + return sslMode === "disable" ? false : { rejectUnauthorized: false }; +} + +export function normalizePostgresConnectionStringForSslMode(dbUrl, sslMode) { + if (typeof dbUrl !== "string" || dbUrl.trim() === "") { + return dbUrl; + } + + let url; + try { + url = new URL(dbUrl); + } catch { + return dbUrl; + } + + if (!["postgres:", "postgresql:"].includes(url.protocol)) { + return dbUrl; + } + + for (const key of ["ssl", "sslmode", "sslcert", "sslkey", "sslrootcert"]) { + url.searchParams.delete(key); + } + if (sslMode !== "disable") { + url.searchParams.set("sslmode", "require"); + url.searchParams.set("uselibpqcompat", "true"); + } + return url.toString(); +} + +export function postgresAdapterContract() { + return { + required: "Postgres-backed durable runtime adapter using HWLAB_CLOUD_DB_URL", + adapterEnv: RUNTIME_ADAPTER_ENV, + adapterEnvValue: RUNTIME_STORE_KIND_POSTGRES, + sourceIssue: "pikasTech/HWLAB#164", + secretMaterialRequiredInSource: false, + fixtureEvidenceAllowed: false + }; +} + +export function addRuntimeDurabilityContract(summary) { + return { + ...summary, + durabilityContract: { + ready: summary.durable === true && summary.ready === true && summary.liveRuntimeEvidence === true, + status: summary.durable === true && summary.ready === true && summary.liveRuntimeEvidence === true ? "ready" : "blocked", + requiredEvidence: RUNTIME_DURABILITY_REQUIRED_EVIDENCE, + dbLiveEvidenceIsDurabilityEvidence: false, + liveRuntimeEvidence: Boolean(summary.liveRuntimeEvidence), + adapterQueryRequired: true, + blockedLayer: summary.durable === true && summary.ready === true && summary.liveRuntimeEvidence === true + ? null + : durabilityBlockedLayer(summary), + blocker: summary.blocker ?? null, + secretMaterialRead: false + } + }; +} + +export function durabilityBlockedLayer(summary = {}) { + const blocker = summary.blocker; + if ( + blocker === RUNTIME_DURABLE_ADAPTER_MISSING || + blocker === RUNTIME_DURABLE_ADAPTER_UNCONFIGURED || + blocker === RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING + ) { + return "adapter"; + } + if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) { + return "ssl"; + } + if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) { + return "auth"; + } + if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) { + return "schema"; + } + if (blocker === RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED) { + return "migration"; + } + if (summary.connection?.queryAttempted || blocker === RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED) { + return "durability_query"; + } + return "adapter"; +} + +export function runtimeSafety() { + return { + devOnly: true, + secretsRead: false, + secretMaterialRead: false, + valuesRedacted: true, + endpointRedacted: true + }; +} + +export function classifyRuntimeDbError(error) { + const code = typeof error?.code === "string" ? error.code : "UNKNOWN"; + if (isRuntimeDbConnectTimeout(error)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + reason: "Postgres runtime adapter connection attempt timed out before completing the runtime query", + retryable: true, + transient: true, + retryAfterMs: 5000, + connection: { + queryAttempted: true, + queryResult: "connect_timeout", + errorCode: code + } + }; + } + if (code === "HWLAB_PG_DRIVER_MISSING") { + return { + blocker: RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + reason: "Postgres runtime adapter is selected, but the pg package is not installed in the runtime image", + connection: { + queryAttempted: false, + queryResult: "driver_missing", + errorCode: code + } + }; + } + if (isRuntimeDbSslError(error)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, + reason: "Postgres runtime adapter reached the DB driver but SSL negotiation or SSL mode compatibility blocked the schema query", + connection: { + queryAttempted: true, + queryResult: "ssl_negotiation_blocked", + errorCode: code + } + }; + } + if (["28P01", "28000", "42501"].includes(code)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + reason: "Postgres runtime adapter reached the DB driver but authentication or authorization blocked the schema query", + connection: { + queryAttempted: true, + queryResult: "auth_blocked", + errorCode: code + } + }; + } + if (["42P01", "42703", "3F000"].includes(code)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + reason: "Postgres runtime adapter reached the database, but the runtime schema is missing or incompatible", + connection: { + queryAttempted: true, + queryResult: "schema_blocked", + errorCode: code + } + }; + } + if (code === "53300") { + return { + blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + reason: "Postgres runtime adapter reached the database, but the server has exhausted available connection slots", + retryable: true, + transient: true, + retryAfterMs: 2000, + connection: { + queryAttempted: true, + queryResult: "too_many_connections", + errorCode: code + } + }; + } + return { + blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + reason: "Postgres runtime adapter could not complete a live runtime schema query", + connection: { + queryAttempted: true, + queryResult: "query_blocked", + errorCode: code + } + }; +} + +export function isRuntimeDbConnectTimeout(error) { + const message = String(error?.message ?? error ?? "").toLowerCase(); + const pgConnectTimeout = message.includes("timeout exceeded when trying to connect"); + const pgConnectionTimeoutTermination = message.includes("connection terminated due to connection timeout"); + if (!pgConnectTimeout && !pgConnectionTimeoutTermination) return false; + if (pgConnectionTimeoutTermination) return true; + const stack = String(error?.stack ?? "").toLowerCase(); + return !stack || stack.includes("pg-pool") || stack.includes("node_modules/pg"); +} + +export function durableRuntimeReadBlockedError(method, readiness = {}) { + return new HwlabProtocolError("Postgres durable runtime adapter is blocked for runtime evidence queries", { + code: ERROR_CODES.internalError, + data: durableRuntimeReadBlockedData(method, readiness), + context: { + result: "failed" + } + }); +} + +export function durableRuntimeReadBlockedData(method, readiness = {}) { + const durability = readiness.durabilityContract ?? {}; + const connection = readiness.connection ?? {}; + const blockedLayer = durability.blockedLayer ?? durabilityBlockedLayer(readiness); + const queryResult = connection.queryResult ?? (blockedLayer === "durability_query" ? "query_blocked" : "not_ready"); + + return pruneUndefined({ + method, + adapter: readiness.adapter ?? RUNTIME_STORE_KIND_POSTGRES, + status: "blocked", + blocker: readiness.blocker ?? RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + durable: false, + durableRequested: true, + liveRuntimeEvidence: false, + queryResult, + blockedLayer, + requiredEvidence: durability.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE, + dbLiveEvidenceIsDurabilityEvidence: false, + retryable: readiness.retryable === true || connection.queryResult === "connect_timeout", + transient: readiness.transient === true || connection.queryResult === "connect_timeout", + retryAfterMs: Number.isFinite(Number(readiness.retryAfterMs)) ? Math.max(0, Math.trunc(Number(readiness.retryAfterMs))) : undefined, + secretMaterialRead: false, + valueRedacted: true, + endpointRedacted: true, + errorCode: connection.errorCode ?? undefined, + redaction: { + valueRedacted: true, + valuesRedacted: true, + endpointRedacted: true, + secretMaterialRead: false + } + }); +} + +export function isRuntimeDbSslError(error) { + const code = typeof error?.code === "string" ? error.code : ""; + if (code.startsWith("ERR_SSL") || code.startsWith("ERR_TLS")) { + return true; + } + if ([ + "DEPTH_ZERO_SELF_SIGNED_CERT", + "SELF_SIGNED_CERT_IN_CHAIN", + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", + "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", + "CERT_HAS_EXPIRED" + ].includes(code)) { + return true; + } + const message = typeof error?.message === "string" ? error.message.toLowerCase() : ""; + return [ + "does not support ssl", + "ssl is not enabled", + "ssl negotiation", + "ssl off", + "ssl required", + "requires ssl", + "no ssl encryption", + "no encryption", + "hostssl", + "server requires ssl", + "before secure tls connection", + "tls handshake" + ].some((pattern) => message.includes(pattern)); +} + +export function summarizeRuntimeSchema(rows) { + const columnsByTable = new Map(); + for (const row of rows) { + const table = row.table_name; + const column = row.column_name; + if (!table || !column) continue; + if (!columnsByTable.has(table)) { + columnsByTable.set(table, new Set()); + } + columnsByTable.get(table).add(column); + } + + const missingTables = []; + const missingColumns = []; + for (const [table, columns] of Object.entries(requiredPostgresSchema)) { + const observed = columnsByTable.get(table); + if (!observed) { + missingTables.push(table); + missingColumns.push(...columns.map((column) => `${table}.${column}`)); + continue; + } + for (const column of columns) { + if (!observed.has(column)) { + missingColumns.push(`${table}.${column}`); + } + } + } + + return { + ready: missingTables.length === 0 && missingColumns.length === 0, + checked: true, + requiredTables: Object.keys(requiredPostgresSchema), + missingTables, + missingColumns + }; +} + +export function notCheckedRuntimeMigration() { + return { + checked: false, + ready: false, + table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + appliedMigrationId: null, + appliedSchemaVersion: null, + missing: true + }; +} + +export function blockedRuntimeMigration({ errorCode = null } = {}) { + return { + ...notCheckedRuntimeMigration(), + checked: true, + errorCode + }; +} + +export function runtimeGates({ + ssl = notCheckedGate(), + auth = notCheckedGate(), + schema = notCheckedGate(), + migration = notCheckedGate(), + durability = notCheckedGate() +} = {}) { + return { + ssl, + auth, + schema, + migration, + durability + }; +} + +export function defaultRuntimeGates({ blocker, connection, schema, migration } = {}) { + if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) { + return { + ssl: blockedGate({ checked: true, blocker }), + auth: notCheckedGate() + }; + } + if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) { + return { + ssl: readyGate(), + auth: blockedGate({ checked: true, blocker }) + }; + } + if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) { + return { + ssl: connection?.queryAttempted ? readyGate() : notCheckedGate(), + auth: connection?.queryAttempted ? readyGate() : notCheckedGate() + }; + } + if (blocker === RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED && schema?.checked && migration?.checked) { + return { + ssl: readyGate(), + auth: readyGate() + }; + } + return { + ssl: notCheckedGate(), + auth: notCheckedGate() + }; +} + +export function defaultSchemaGate({ blocker, schema } = {}) { + if (schema?.checked) { + return gateFromReadiness(schema.ready, { blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }); + } + if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) { + return blockedGate({ checked: true, blocker }); + } + return notCheckedGate(); +} + +export function defaultMigrationGate({ blocker, migration } = {}) { + if (migration?.checked) { + return gateFromReadiness(migration.ready, { blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }); + } + if (blocker === RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED) { + return blockedGate({ checked: true, blocker }); + } + return notCheckedGate(); +} + +export function readyGate() { + return { + checked: true, + ready: true, + status: "ready", + blocker: null + }; +} + +export function blockedGate({ checked = true, blocker = "runtime_durable_adapter_blocked" } = {}) { + return { + checked, + ready: false, + status: checked ? "blocked" : "not_checked", + blocker + }; +} + +export function notCheckedGate() { + return { + checked: false, + ready: false, + status: "not_checked", + blocker: null + }; +} + +export function gateFromReadiness(ready, { blocker } = {}) { + return ready ? readyGate() : blockedGate({ checked: true, blocker }); +} + +export function snapshotMemory(memory) { + return { + gatewaySessions: new Map(memory.gatewaySessions), + boxResources: new Map(memory.boxResources), + boxCapabilities: new Map(memory.boxCapabilities), + hardwareOperations: new Map(memory.hardwareOperations), + auditEvents: new Map(memory.auditEvents), + evidenceRecords: new Map(memory.evidenceRecords), + agentTraceEvents: new Map(memory.agentTraceEvents), + workbenchProjectionStates: new Map(memory.workbenchProjectionStates), + workbenchSessions: new Map(memory.workbenchSessions), + workbenchMessages: new Map(memory.workbenchMessages), + workbenchParts: new Map(memory.workbenchParts), + workbenchTurns: new Map(memory.workbenchTurns), + workbenchTraceEvents: new Map(memory.workbenchTraceEvents), + workbenchProjectionCheckpoints: new Map(memory.workbenchProjectionCheckpoints) + }; +} + +export function newRecords(current, before) { + return [...current.entries()] + .filter(([id]) => !before.has(id)) + .map(([, record]) => record); +} + +export function changedRecords(current, before) { + return [...current.entries()] + .filter(([id, record]) => stableJson(before.get(id)) !== stableJson(record)) + .map(([, record]) => record); +} + +export function withPersistence(result, persistence) { + return { + ...result, + persistence + }; +} + +export function parseJsonColumn(value, fallback) { + if (!value) return fallback; + if (typeof value === "object") return value; + try { + return JSON.parse(String(value)); + } catch { + return fallback; + } +} + +export function postgresPoolStats(pool) { + if (!pool || typeof pool !== "object") return null; + return { + totalCount: Number.isFinite(Number(pool.totalCount)) ? Number(pool.totalCount) : null, + idleCount: Number.isFinite(Number(pool.idleCount)) ? Number(pool.idleCount) : null, + waitingCount: Number.isFinite(Number(pool.waitingCount)) ? Number(pool.waitingCount) : null, + valuesRedacted: true + }; +} + +export function toCountKey(table) { + if (table === "users") return "users"; + if (table === "user_sessions") return "userSessions"; + if (table === "gateway_sessions") return "gatewaySessions"; + if (table === "box_resources") return "boxResources"; + if (table === "box_capabilities") return "boxCapabilities"; + if (table === "hardware_operations") return "hardwareOperations"; + if (table === "audit_events") return "auditEvents"; + if (table === "evidence_records") return "evidenceRecords"; + if (table === "agent_sessions") return "agentSessions"; + if (table === "account_workspaces") return "accountWorkspaces"; + if (table === "worker_sessions") return "workerSessions"; + if (table === "agent_trace_events") return "agentTraceEvents"; + if (table === "workbench_projection_state") return "workbenchProjectionStates"; + if (table === "workbench_sessions") return "workbenchSessions"; + if (table === "workbench_messages") return "workbenchMessages"; + if (table === "workbench_parts") return "workbenchParts"; + if (table === "workbench_turns") return "workbenchTurns"; + if (table === "workbench_trace_events") return "workbenchTraceEvents"; + if (table === "workbench_projection_checkpoints") return "workbenchProjectionCheckpoints"; + return table; +} + +export function normalizeRuntimeTimeoutMs(value) { + const parsed = Number.parseInt(String(value ?? ""), 10); + if (!Number.isInteger(parsed) || parsed <= 0) { + return 1200; + } + return Math.min(parsed, 5000); +} + +export function normalizeRuntimePoolMax(value) { + if (value === undefined || value === null || String(value).trim() === "") return null; + const parsed = Number.parseInt(String(value), 10); + if (!Number.isInteger(parsed) || parsed <= 0 || String(parsed) !== String(value).trim()) { + const error = new Error(`${RUNTIME_DB_POOL_MAX_ENV} must be a positive integer`); + error.code = "HWLAB_CLOUD_DB_POOL_MAX_INVALID"; + throw error; + } + return parsed; +} diff --git a/internal/db/runtime-store-memory.ts b/internal/db/runtime-store-memory.ts new file mode 100644 index 00000000..61fe2aff --- /dev/null +++ b/internal/db/runtime-store-memory.ts @@ -0,0 +1,815 @@ +/* + * In-memory Cloud runtime store implementation. + * Public construction remains in runtime-store.ts. + */ +import { createHash, randomUUID } from "node:crypto"; + +import { + ENVIRONMENT_DEV, + ERROR_CODES, + HwlabProtocolError, + assertProtocolRecord +} from "../protocol/index.mjs"; +import { + CLOUD_API_SERVICE_ID, + createProtocolAuditEvent, + deriveProtocolActorFromMeta +} from "../audit/index.mjs"; +import { + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS +} from "./schema.ts"; + + +import * as runtimeCore from "./runtime-store-core.ts"; + +const { + RUNTIME_STORE_KIND, + RUNTIME_STORE_KIND_POSTGRES, + RUNTIME_DURABLE_ADAPTER_MISSING, + RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, + RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, + RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + RUNTIME_DURABILITY_REQUIRED_EVIDENCE, + RUNTIME_ADAPTER_ENV, + RUNTIME_DURABLE_ENV, + RUNTIME_DB_POOL_MAX_ENV, + requiredPostgresSchema, + postgresCountTables, + postgresRuntimeReadIndexes, + buildPostgresPoolConfig, + RUNTIME_DB_QUERY_TIMEOUT_MS_ENV, + RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV, + RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV, + RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV, + RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV, + DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS, + DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS, + DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS, + DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS, + normalizePositiveInteger, + delayRuntimeDbQueryRetry, + isReadyForDurableRuntimeRead, + normalizeCapabilityInputs, + normalizeShellInput, + normalizeAgentTraceEvent, + normalizeWorkbenchProjectionState, + normalizeWorkbenchProjectionAllocation, + normalizeWorkbenchFacts, + normalizeWorkbenchAggregateEventsForFacts, + appendWorkbenchFactEvent, + workbenchAggregateForFact, + normalizeWorkbenchAggregateEvent, + workbenchAggregateEventType, + workbenchInputDelivery, + workbenchInputStatus, + indexWorkbenchAggregateEvents, + normalizeWorkbenchSessionInputFact, + memoryWorkbenchSessionInputWithSeq, + inputFactWithAggregateSeq, + workbenchSessionInputWithSeq, + workbenchSessionInputPromotedSeq, + normalizeWorkbenchSessionFact, + normalizeWorkbenchMessageFact, + normalizeWorkbenchPartFact, + normalizeWorkbenchTurnFact, + assertWorkbenchTurnFinalResponseInvariant, + normalizeWorkbenchFactStatus, + workbenchFactFinalResponseText, + normalizeWorkbenchTraceEventFact, + normalizeWorkbenchProjectionCheckpoint, + arrayInput, + stableWorkbenchFactId, + requiredWorkbenchFactError, + compareTraceEventRecords, + compareProjectionStateRecords, + compareWorkbenchFactRecords, + compareWorkbenchInputFactRecords, + compareWorkbenchPartFactRecords, + compareWorkbenchTraceEventFacts, + sortWorkbenchFactRows, + WORKBENCH_FACT_FAMILIES, + workbenchFactFamilySet, + workbenchFactFamilyForTable, + workbenchSessionSummarySelectClause, + workbenchSessionSummaryFactFromRow, + workbenchFactOrder, + traceEventLevel, + positiveIntegerOrNull, + nonNegativeInteger, + nonNegativeIntegerOrNull, + projectionStatusOr, + projectionHealthOr, + resultSyncStateOr, + timestampOr, + textOr, + textList, + asObject, + requireString, + requireProtocolId, + normalizeJsonObject, + pruneUndefined, + makeId, + stableJson, + sha256Hex, + sortJson, + matchesQuery, + matchesWorkbenchSessionFactQuery, + limitResults, + normalizeRuntimeAdapter, + normalizePostgresSslMode, + postgresSslModeFromConnectionString, + postgresSslOption, + normalizePostgresConnectionStringForSslMode, + postgresAdapterContract, + addRuntimeDurabilityContract, + durabilityBlockedLayer, + runtimeSafety, + classifyRuntimeDbError, + isRuntimeDbConnectTimeout, + durableRuntimeReadBlockedError, + durableRuntimeReadBlockedData, + isRuntimeDbSslError, + summarizeRuntimeSchema, + notCheckedRuntimeMigration, + blockedRuntimeMigration, + runtimeGates, + defaultRuntimeGates, + defaultSchemaGate, + defaultMigrationGate, + readyGate, + blockedGate, + notCheckedGate, + gateFromReadiness, + snapshotMemory, + newRecords, + changedRecords, + withPersistence, + parseJsonColumn, + postgresPoolStats, + toCountKey, + normalizeRuntimeTimeoutMs, + normalizeRuntimePoolMax +} = runtimeCore; + +export class CloudRuntimeStore { + constructor({ now = () => new Date().toISOString() } = {}) { + this.now = now; + this.gatewaySessions = new Map(); + this.boxResources = new Map(); + this.boxCapabilities = new Map(); + this.hardwareOperations = new Map(); + this.auditEvents = new Map(); + this.evidenceRecords = new Map(); + this.agentTraceEvents = new Map(); + this.workbenchProjectionStates = new Map(); + this.workbenchSessions = new Map(); + this.workbenchMessages = new Map(); + this.workbenchParts = new Map(); + this.workbenchTurns = new Map(); + this.workbenchTraceEvents = new Map(); + this.workbenchEventSequences = new Map(); + this.workbenchEvents = new Map(); + this.workbenchSessionInputs = new Map(); + this.workbenchProjectionCheckpoints = new Map(); + } + + summary() { + return addRuntimeDurabilityContract({ + adapter: RUNTIME_STORE_KIND, + durable: false, + status: "degraded", + blocker: RUNTIME_DURABLE_ADAPTER_MISSING, + reason: "L1 runtime writes are process-local only; no DB-backed durable runtime adapter is configured", + adapterContract: { + required: "DB-backed durable runtime adapter using HWLAB_CLOUD_DB_URL", + sourceIssue: "pikasTech/HWLAB#164", + secretMaterialRequiredInSource: false + }, + counts: { + gatewaySessions: this.gatewaySessions.size, + boxResources: this.boxResources.size, + boxCapabilities: this.boxCapabilities.size, + hardwareOperations: this.hardwareOperations.size, + auditEvents: this.auditEvents.size, + evidenceRecords: this.evidenceRecords.size, + agentTraceEvents: this.agentTraceEvents.size, + workbenchProjectionStates: this.workbenchProjectionStates.size, + workbenchSessions: this.workbenchSessions.size, + workbenchMessages: this.workbenchMessages.size, + workbenchParts: this.workbenchParts.size, + workbenchTurns: this.workbenchTurns.size, + workbenchTraceEvents: this.workbenchTraceEvents.size, + workbenchEventSequences: this.workbenchEventSequences.size, + workbenchEvents: this.workbenchEvents.size, + workbenchSessionInputs: this.workbenchSessionInputs.size, + workbenchProjectionCheckpoints: this.workbenchProjectionCheckpoints.size + } + }); + } + + registerGatewaySession(params = {}, requestMeta = {}) { + const now = this.now(); + const input = asObject(params.gatewaySession ?? params, "gatewaySession"); + const gatewaySession = pruneUndefined({ + gatewaySessionId: input.gatewaySessionId || makeId("gws"), + projectId: requireProtocolId(input, "projectId"), + serviceId: input.serviceId || "hwlab-gateway", + gatewayId: input.gatewayId || makeId("gtw"), + endpoint: input.endpoint, + status: input.status || "connected", + environment: ENVIRONMENT_DEV, + startedAt: input.startedAt || now, + lastSeenAt: input.lastSeenAt || now, + stoppedAt: input.stoppedAt, + labels: normalizeJsonObject(input.labels) + }); + + assertProtocolRecord("gatewaySession", gatewaySession); + this.gatewaySessions.set(gatewaySession.gatewaySessionId, gatewaySession); + + const auditEvent = this.recordAuditEvent({ + requestMeta, + params, + action: "gateway.session.register", + targetType: "gateway_session", + targetId: gatewaySession.gatewaySessionId, + projectId: gatewaySession.projectId, + gatewaySessionId: gatewaySession.gatewaySessionId, + outcome: "accepted", + metadata: { + gatewayId: gatewaySession.gatewayId, + serviceId: gatewaySession.serviceId + } + }); + + return { + registered: true, + gatewaySession, + auditEvent, + persistence: this.summary() + }; + } + + registerBoxResource(params = {}, requestMeta = {}) { + const now = this.now(); + const input = asObject(params.resource ?? params.boxResource ?? params, "boxResource"); + const gatewaySessionId = requireProtocolId(input, "gatewaySessionId"); + const gatewaySession = this.requireGatewaySession(gatewaySessionId); + const projectId = input.projectId || gatewaySession.projectId; + const resource = pruneUndefined({ + resourceId: input.resourceId || makeId("res"), + projectId, + gatewaySessionId, + boxId: input.boxId || makeId("box"), + resourceType: input.resourceType || "simulator_endpoint", + name: input.name, + state: input.state || "available", + environment: ENVIRONMENT_DEV, + metadata: normalizeJsonObject(input.metadata), + createdAt: input.createdAt || now, + updatedAt: input.updatedAt || now + }); + + assertProtocolRecord("boxResource", resource); + this.boxResources.set(resource.resourceId, resource); + + const auditEvent = this.recordAuditEvent({ + requestMeta, + params, + action: "box.resource.register", + targetType: "box_resource", + targetId: resource.resourceId, + projectId: resource.projectId, + gatewaySessionId: resource.gatewaySessionId, + outcome: "accepted", + metadata: { + boxId: resource.boxId, + resourceType: resource.resourceType + } + }); + + return { + registered: true, + resource, + auditEvent, + persistence: this.summary() + }; + } + + reportBoxCapabilities(params = {}, requestMeta = {}) { + const now = this.now(); + const inputs = normalizeCapabilityInputs(params); + const capabilities = []; + + for (const input of inputs) { + const resourceId = requireProtocolId(input, "resourceId"); + const resource = this.requireBoxResource(resourceId); + const capability = pruneUndefined({ + capabilityId: input.capabilityId || makeId("cap"), + resourceId, + projectId: input.projectId || resource.projectId, + name: requireString(input, "name"), + description: input.description, + direction: input.direction || "bidirectional", + valueType: input.valueType || "object", + unit: input.unit, + constraints: normalizeJsonObject(input.constraints), + mutatesState: input.mutatesState ?? true, + createdAt: input.createdAt || now, + updatedAt: input.updatedAt || now + }); + + assertProtocolRecord("boxCapability", capability); + this.boxCapabilities.set(capability.capabilityId, capability); + capabilities.push(capability); + } + + const first = capabilities[0]; + const auditEvent = this.recordAuditEvent({ + requestMeta, + params, + action: "box.capability.report", + targetType: "box_capability", + targetId: first.capabilityId, + projectId: first.projectId, + gatewaySessionId: this.boxResources.get(first.resourceId)?.gatewaySessionId, + outcome: "accepted", + metadata: { + capabilityIds: capabilities.map((capability) => capability.capabilityId) + } + }); + + return { + reported: true, + capabilities, + auditEvent, + persistence: this.summary() + }; + } + + requestHardwareOperation(params = {}, requestMeta = {}) { + const refs = this.requireOperationRefs(params, requestMeta); + const now = this.now(); + const operation = pruneUndefined({ + operationId: params.operationId || makeId("op"), + projectId: refs.projectId, + gatewaySessionId: refs.gatewaySession.gatewaySessionId, + agentSessionId: params.agentSessionId, + workerSessionId: params.workerSessionId, + resourceId: refs.resource.resourceId, + capabilityId: refs.capability.capabilityId, + requestedBy: refs.actor.actorId, + input: normalizeJsonObject(params.input), + status: "accepted", + environment: ENVIRONMENT_DEV, + requestedAt: params.requestedAt || now, + updatedAt: now + }); + + assertProtocolRecord("hardwareOperation", operation); + this.hardwareOperations.set(operation.operationId, operation); + + const auditEvent = this.recordAuditEvent({ + requestMeta, + params, + action: "hardware.operation.request", + targetType: "hardware_operation", + targetId: operation.operationId, + projectId: operation.projectId, + gatewaySessionId: operation.gatewaySessionId, + workerSessionId: operation.workerSessionId, + operationId: operation.operationId, + outcome: "accepted", + metadata: { + resourceId: operation.resourceId, + capabilityId: operation.capabilityId + } + }); + + return { + accepted: true, + status: operation.status, + operationId: operation.operationId, + operation, + auditEvent, + persistence: this.summary() + }; + } + + invokeHardwareShell(params = {}, requestMeta = {}) { + const shellInput = normalizeShellInput(params); + const operationResult = this.requestHardwareOperation( + { + ...params, + input: { + ...normalizeJsonObject(params.input), + shell: shellInput + } + }, + requestMeta + ); + const operation = { + ...operationResult.operation, + output: { + shellExecuted: false, + dispatchStatus: "not_connected", + message: "cloud-api recorded the shell invoke request; no gateway shell adapter is connected in L1" + }, + updatedAt: this.now() + }; + assertProtocolRecord("hardwareOperation", operation); + this.hardwareOperations.set(operation.operationId, operation); + + const evidenceRecord = this.writeEvidenceRecord( + { + projectId: operation.projectId, + operationId: operation.operationId, + kind: "trace", + mimeType: "application/json", + serviceId: CLOUD_API_SERVICE_ID, + metadata: { + traceId: requestMeta.traceId, + gatewaySessionId: operation.gatewaySessionId, + resourceId: operation.resourceId, + capabilityId: operation.capabilityId, + shellExecuted: false, + dispatchStatus: "not_connected" + }, + content: { + operationId: operation.operationId, + shell: shellInput, + dispatchStatus: "not_connected" + } + }, + requestMeta + ).evidenceRecord; + + const auditEvent = this.recordAuditEvent({ + requestMeta, + params, + action: "hardware.invoke.shell", + targetType: "hardware_operation", + targetId: operation.operationId, + projectId: operation.projectId, + gatewaySessionId: operation.gatewaySessionId, + workerSessionId: operation.workerSessionId, + operationId: operation.operationId, + outcome: "accepted", + metadata: { + evidenceId: evidenceRecord.evidenceId, + shellExecuted: false, + dispatchStatus: "not_connected" + } + }); + + return { + accepted: true, + status: operation.status, + operationId: operation.operationId, + operation, + auditEvent, + evidenceRecord, + dispatch: operation.output, + persistence: this.summary() + }; + } + + writeAuditEvent(params = {}, requestMeta = {}) { + const eventInput = params.event ?? params.auditEvent ?? params; + const auditEvent = eventInput.auditId + ? pruneUndefined({ + ...eventInput, + environment: ENVIRONMENT_DEV + }) + : this.buildAuditEventFromInput(eventInput, requestMeta); + + assertProtocolRecord("auditEvent", auditEvent); + this.auditEvents.set(auditEvent.auditId, auditEvent); + + return { + written: true, + auditEvent, + persistence: this.summary() + }; + } + + queryAuditEvents(params = {}) { + const events = [...this.auditEvents.values()].filter((event) => matchesQuery(event, params, [ + "auditId", + "traceId", + "projectId", + "gatewaySessionId", + "operationId", + "action", + "targetId" + ])); + + return { + events: limitResults(events, params.limit), + count: events.length, + persistence: this.summary() + }; + } + + writeEvidenceRecord(params = {}, requestMeta = {}) { + const now = this.now(); + const input = asObject(params.record ?? params.evidenceRecord ?? params, "evidenceRecord"); + const content = input.content ?? input.metadata ?? {}; + const sha256 = input.sha256 || sha256Hex(content); + const evidenceId = input.evidenceId || makeId("evd"); + const evidenceRecord = pruneUndefined({ + evidenceId, + projectId: requireProtocolId(input, "projectId"), + operationId: requireProtocolId(input, "operationId"), + agentSessionId: input.agentSessionId, + workerSessionId: input.workerSessionId, + kind: input.kind || "trace", + uri: input.uri || `memory://hwlab/evidence/${evidenceId}`, + mimeType: input.mimeType, + sha256, + sizeBytes: input.sizeBytes ?? Buffer.byteLength(stableJson(content), "utf8"), + serviceId: input.serviceId || CLOUD_API_SERVICE_ID, + environment: ENVIRONMENT_DEV, + metadata: { + ...normalizeJsonObject(input.metadata), + ...(requestMeta.traceId ? { traceId: requestMeta.traceId } : {}) + }, + createdAt: input.createdAt || now + }); + + assertProtocolRecord("evidenceRecord", evidenceRecord); + this.evidenceRecords.set(evidenceRecord.evidenceId, evidenceRecord); + + return { + written: true, + evidenceRecord, + persistence: this.summary() + }; + } + + queryEvidenceRecords(params = {}) { + const records = [...this.evidenceRecords.values()].filter((record) => matchesQuery(record, params, [ + "evidenceId", + "projectId", + "operationId", + "agentSessionId", + "workerSessionId", + "kind", + "serviceId" + ])); + + return { + records: limitResults(records, params.limit), + count: records.length, + persistence: this.summary() + }; + } + + writeAgentTraceEvent(params = {}, requestMeta = {}) { + const traceEvent = normalizeAgentTraceEvent(params.traceEvent ?? params.event ?? params, requestMeta, this.now()); + this.agentTraceEvents.set(traceEvent.id, traceEvent); + return { + written: true, + traceEvent, + persistence: this.summary() + }; + } + + queryAgentTraceEvents(params = {}) { + const records = [...this.agentTraceEvents.values()].filter((record) => matchesQuery(record, params, [ + "id", + "traceId", + "agentSessionId", + "workerSessionId", + "level" + ])); + records.sort(compareTraceEventRecords); + return { + events: limitResults(records.map((record) => record.event), params.limit), + count: records.length, + persistence: this.summary() + }; + } + + writeWorkbenchProjectionState(params = {}, requestMeta = {}) { + const state = normalizeWorkbenchProjectionState(params.projectionState ?? params.state ?? params, requestMeta, this.now()); + this.workbenchProjectionStates.set(state.traceId, state); + return { + written: true, + projectionState: state, + persistence: this.summary() + }; + } + + getWorkbenchProjectionState(params = {}) { + const traceId = textOr(params.traceId, ""); + const state = traceId ? this.workbenchProjectionStates.get(traceId) ?? null : null; + return { + projectionState: state, + found: Boolean(state), + persistence: this.summary() + }; + } + + allocateWorkbenchProjectedSeq(params = {}, requestMeta = {}) { + const allocation = normalizeWorkbenchProjectionAllocation(params, requestMeta, this.now()); + const existing = [...this.workbenchTraceEvents.values()].find((record) => + record.traceId === allocation.traceId && record.sourceEventId === allocation.sourceEventId + ); + if (existing) { + return { + ...allocation, + projectedSeq: nonNegativeInteger(existing.projectedSeq), + reused: true, + persistence: this.summary(), + valuesPrinted: false + }; + } + const eventMax = [...this.workbenchTraceEvents.values()] + .filter((record) => record.traceId === allocation.traceId) + .reduce((max, record) => Math.max(max, nonNegativeInteger(record.projectedSeq)), 0); + const checkpointMax = nonNegativeInteger(this.workbenchProjectionCheckpoints.get(allocation.traceId)?.projectedSeq); + return { + ...allocation, + projectedSeq: Math.max(eventMax, checkpointMax) + 1, + reused: false, + persistence: this.summary(), + valuesPrinted: false + }; + } + + queryWorkbenchProjectionStates(params = {}) { + const dueAt = textOr(params.dueAt, ""); + const statuses = Array.isArray(params.projectionStatuses) ? new Set(params.projectionStatuses.map((item) => textOr(item, "")).filter(Boolean)) : null; + const records = [...this.workbenchProjectionStates.values()] + .filter((record) => matchesQuery(record, params, ["traceId", "sessionId", "runId", "commandId", "sourceRunId", "sourceCommandId"])) + .filter((record) => !statuses || statuses.has(record.projectionStatus)) + .filter((record) => !dueAt || !record.nextRetryAt || String(record.nextRetryAt) <= dueAt) + .sort(compareProjectionStateRecords); + return { + states: limitResults(records, params.limit), + count: records.length, + persistence: this.summary() + }; + } + + writeWorkbenchFacts(params = {}, requestMeta = {}) { + const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); + const inputFacts = facts.inputs.map((fact) => memoryWorkbenchSessionInputWithSeq(fact, this.workbenchSessionInputs)); + for (const fact of inputFacts) this.workbenchSessionInputs.set(fact.inputId, fact); + for (const fact of facts.sessions) this.workbenchSessions.set(fact.sessionId, fact); + for (const fact of facts.messages) this.workbenchMessages.set(fact.messageId, fact); + for (const fact of facts.parts) this.workbenchParts.set(fact.partId, fact); + for (const fact of facts.turns) this.workbenchTurns.set(fact.turnId, fact); + for (const fact of facts.traceEvents) this.workbenchTraceEvents.set(fact.id, fact); + for (const fact of facts.checkpoints) { + const previous = this.workbenchProjectionCheckpoints.get(fact.traceId) ?? null; + if (!previous || nonNegativeInteger(fact.projectedSeq) >= nonNegativeInteger(previous.projectedSeq)) { + this.workbenchProjectionCheckpoints.set(fact.traceId, fact); + } + } + return { + written: true, + facts: { ...facts, inputs: inputFacts }, + persistence: this.summary() + }; + } + + queryWorkbenchFacts(params = {}) { + const families = workbenchFactFamilySet(params); + const sessions = [...this.workbenchSessions.values()].filter((record) => matchesWorkbenchSessionFactQuery(record, params)); + const inputs = [...this.workbenchSessionInputs.values()].filter((record) => matchesQuery(record, params, ["inputId", "sessionId", "turnId", "traceId", "messageId", "commandId", "delivery", "status"])); + const messages = [...this.workbenchMessages.values()].filter((record) => matchesQuery(record, params, ["messageId", "sessionId", "turnId", "traceId", "role", "status"])); + const parts = [...this.workbenchParts.values()].filter((record) => matchesQuery(record, params, ["partId", "messageId", "sessionId", "turnId", "traceId", "partType", "status"])); + const turns = [...this.workbenchTurns.values()].filter((record) => matchesQuery(record, params, ["turnId", "sessionId", "traceId", "messageId", "status"])); + const afterProjectedSeq = nonNegativeInteger(params.afterProjectedSeq); + const traceEvents = [...this.workbenchTraceEvents.values()].filter((record) => matchesQuery(record, params, ["id", "traceId", "sessionId", "turnId", "messageId", "eventType"]) && (afterProjectedSeq <= 0 || nonNegativeInteger(record.projectedSeq) > afterProjectedSeq)); + const checkpoints = [...this.workbenchProjectionCheckpoints.values()].filter((record) => matchesQuery(record, params, ["traceId", "sessionId", "turnId", "runId", "commandId", "projectionStatus", "projectionHealth"])); + const facts = { + inputs: families.has("inputs") ? limitResults(sortWorkbenchFactRows(inputs, params, "inputs", compareWorkbenchFactRecords), params.limit) : [], + sessions: families.has("sessions") ? limitResults(sortWorkbenchFactRows(sessions, params, "sessions", compareWorkbenchFactRecords), params.limit) : [], + messages: families.has("messages") ? limitResults(sortWorkbenchFactRows(messages, params, "messages", compareWorkbenchFactRecords), params.limit) : [], + parts: families.has("parts") ? limitResults(sortWorkbenchFactRows(parts, params, "parts", compareWorkbenchPartFactRecords), params.limit) : [], + turns: families.has("turns") ? limitResults(sortWorkbenchFactRows(turns, params, "turns", compareWorkbenchFactRecords), params.limit) : [], + traceEvents: families.has("traceEvents") ? limitResults(sortWorkbenchFactRows(traceEvents, params, "traceEvents", compareWorkbenchTraceEventFacts), params.limit) : [], + checkpoints: families.has("checkpoints") ? limitResults(sortWorkbenchFactRows(checkpoints, params, "checkpoints", compareWorkbenchFactRecords), params.limit) : [] + }; + return { + facts, + count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0), + persistence: this.summary() + }; + } + + recordAuditEvent(input) { + return this.writeAuditEvent(this.buildAuditEventFromInput(input, input.requestMeta), input.requestMeta).auditEvent; + } + + buildAuditEventFromInput(input = {}, requestMeta = {}) { + const actor = deriveProtocolActorFromMeta(requestMeta, input.params?.audit ?? input); + return createProtocolAuditEvent({ + auditId: input.auditId, + traceId: requestMeta.traceId || input.traceId, + actorType: actor.actorType, + actorId: actor.actorId, + action: input.action, + targetType: input.targetType, + targetId: input.targetId, + projectId: input.projectId, + gatewaySessionId: input.gatewaySessionId, + workerSessionId: input.workerSessionId, + operationId: input.operationId, + serviceId: CLOUD_API_SERVICE_ID, + outcome: input.outcome, + reason: input.reason || input.params?.audit?.reason, + metadata: input.metadata, + occurredAt: input.occurredAt + }); + } + + requireOperationRefs(params, requestMeta = {}) { + const projectId = requireProtocolId(params, "projectId"); + const gatewaySession = this.requireGatewaySession(requireProtocolId(params, "gatewaySessionId")); + const resource = this.requireBoxResource(requireProtocolId(params, "resourceId")); + const capability = this.requireBoxCapability(requireProtocolId(params, "capabilityId")); + + if (gatewaySession.projectId !== projectId || resource.projectId !== projectId || capability.projectId !== projectId) { + throw new HwlabProtocolError("hardware operation references must belong to one project", { + code: ERROR_CODES.invalidParams, + data: { + projectId, + gatewaySessionId: gatewaySession.gatewaySessionId, + resourceId: resource.resourceId, + capabilityId: capability.capabilityId + } + }); + } + if (resource.gatewaySessionId !== gatewaySession.gatewaySessionId) { + throw new HwlabProtocolError("resource is not registered under the requested gateway session", { + code: ERROR_CODES.resourceLocked, + data: { + gatewaySessionId: gatewaySession.gatewaySessionId, + resourceGatewaySessionId: resource.gatewaySessionId, + resourceId: resource.resourceId + } + }); + } + if (capability.resourceId !== resource.resourceId) { + throw new HwlabProtocolError("capability is not registered on the requested resource", { + code: ERROR_CODES.capabilityUnavailable, + data: { + resourceId: resource.resourceId, + capabilityId: capability.capabilityId + } + }); + } + + return { + projectId, + gatewaySession, + resource, + capability, + actor: deriveProtocolActorFromMeta(requestMeta, params.audit ?? {}) + }; + } + + requireGatewaySession(gatewaySessionId) { + const gatewaySession = this.gatewaySessions.get(gatewaySessionId); + if (!gatewaySession) { + throw new HwlabProtocolError(`gateway session ${gatewaySessionId} is not registered`, { + code: ERROR_CODES.sessionNotFound, + data: { gatewaySessionId } + }); + } + return gatewaySession; + } + + requireBoxResource(resourceId) { + const resource = this.boxResources.get(resourceId); + if (!resource) { + throw new HwlabProtocolError(`box resource ${resourceId} is not registered`, { + code: ERROR_CODES.capabilityUnavailable, + data: { resourceId } + }); + } + return resource; + } + + requireBoxCapability(capabilityId) { + const capability = this.boxCapabilities.get(capabilityId); + if (!capability) { + throw new HwlabProtocolError(`capability ${capabilityId} is not reported`, { + code: ERROR_CODES.capabilityUnavailable, + data: { capabilityId } + }); + } + return capability; + } +} diff --git a/internal/db/runtime-store-postgres.ts b/internal/db/runtime-store-postgres.ts new file mode 100644 index 00000000..8bb5bebc --- /dev/null +++ b/internal/db/runtime-store-postgres.ts @@ -0,0 +1,1537 @@ +/* + * PostgreSQL Cloud runtime store implementation. + * Public construction remains in runtime-store.ts. + */ +import { createHash, randomUUID } from "node:crypto"; + +import { + ENVIRONMENT_DEV, + ERROR_CODES, + HwlabProtocolError, + assertProtocolRecord +} from "../protocol/index.mjs"; +import { + CLOUD_API_SERVICE_ID, + createProtocolAuditEvent, + deriveProtocolActorFromMeta +} from "../audit/index.mjs"; +import { + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS +} from "./schema.ts"; + + +import { CloudRuntimeStore } from "./runtime-store-memory.ts"; +import * as runtimeCore from "./runtime-store-core.ts"; + +const { + RUNTIME_STORE_KIND, + RUNTIME_STORE_KIND_POSTGRES, + RUNTIME_DURABLE_ADAPTER_MISSING, + RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, + RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, + RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + RUNTIME_DURABILITY_REQUIRED_EVIDENCE, + RUNTIME_ADAPTER_ENV, + RUNTIME_DURABLE_ENV, + RUNTIME_DB_POOL_MAX_ENV, + requiredPostgresSchema, + postgresCountTables, + postgresRuntimeReadIndexes, + buildPostgresPoolConfig, + RUNTIME_DB_QUERY_TIMEOUT_MS_ENV, + RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV, + RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV, + RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV, + RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV, + DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS, + DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS, + DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS, + DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS, + normalizePositiveInteger, + delayRuntimeDbQueryRetry, + isReadyForDurableRuntimeRead, + normalizeCapabilityInputs, + normalizeShellInput, + normalizeAgentTraceEvent, + normalizeWorkbenchProjectionState, + normalizeWorkbenchProjectionAllocation, + normalizeWorkbenchFacts, + normalizeWorkbenchAggregateEventsForFacts, + appendWorkbenchFactEvent, + workbenchAggregateForFact, + normalizeWorkbenchAggregateEvent, + workbenchAggregateEventType, + workbenchInputDelivery, + workbenchInputStatus, + indexWorkbenchAggregateEvents, + normalizeWorkbenchSessionInputFact, + memoryWorkbenchSessionInputWithSeq, + inputFactWithAggregateSeq, + workbenchSessionInputWithSeq, + workbenchSessionInputPromotedSeq, + normalizeWorkbenchSessionFact, + normalizeWorkbenchMessageFact, + normalizeWorkbenchPartFact, + normalizeWorkbenchTurnFact, + assertWorkbenchTurnFinalResponseInvariant, + normalizeWorkbenchFactStatus, + workbenchFactFinalResponseText, + normalizeWorkbenchTraceEventFact, + normalizeWorkbenchProjectionCheckpoint, + arrayInput, + stableWorkbenchFactId, + requiredWorkbenchFactError, + compareTraceEventRecords, + compareProjectionStateRecords, + compareWorkbenchFactRecords, + compareWorkbenchInputFactRecords, + compareWorkbenchPartFactRecords, + compareWorkbenchTraceEventFacts, + sortWorkbenchFactRows, + WORKBENCH_FACT_FAMILIES, + workbenchFactFamilySet, + workbenchFactFamilyForTable, + workbenchSessionSummarySelectClause, + workbenchSessionSummaryFactFromRow, + workbenchFactOrder, + traceEventLevel, + positiveIntegerOrNull, + nonNegativeInteger, + nonNegativeIntegerOrNull, + projectionStatusOr, + projectionHealthOr, + resultSyncStateOr, + timestampOr, + textOr, + textList, + asObject, + requireString, + requireProtocolId, + normalizeJsonObject, + pruneUndefined, + makeId, + stableJson, + sha256Hex, + sortJson, + matchesQuery, + matchesWorkbenchSessionFactQuery, + limitResults, + normalizeRuntimeAdapter, + normalizePostgresSslMode, + postgresSslModeFromConnectionString, + postgresSslOption, + normalizePostgresConnectionStringForSslMode, + postgresAdapterContract, + addRuntimeDurabilityContract, + durabilityBlockedLayer, + runtimeSafety, + classifyRuntimeDbError, + isRuntimeDbConnectTimeout, + durableRuntimeReadBlockedError, + durableRuntimeReadBlockedData, + isRuntimeDbSslError, + summarizeRuntimeSchema, + notCheckedRuntimeMigration, + blockedRuntimeMigration, + runtimeGates, + defaultRuntimeGates, + defaultSchemaGate, + defaultMigrationGate, + readyGate, + blockedGate, + notCheckedGate, + gateFromReadiness, + snapshotMemory, + newRecords, + changedRecords, + withPersistence, + parseJsonColumn, + postgresPoolStats, + toCountKey, + normalizeRuntimeTimeoutMs, + normalizeRuntimePoolMax +} = runtimeCore; + +export class PostgresCloudRuntimeStore { + constructor({ + env = process.env, + dbUrl, + sslMode = "require", + now = () => new Date().toISOString(), + queryClient, + pgModuleLoader, + logger = console + } = {}) { + this.env = env; + this.dbUrl = dbUrl; + this.sslMode = sslMode; + this.now = now; + this.queryClient = queryClient; + this.pgModuleLoader = pgModuleLoader; + this.logger = logger; + this.pool = null; + this.postgresPoolResetInFlight = null; + this.postgresPoolResetLastAtMs = 0; + this.runtimeReadIndexesReady = false; + this.runtimeReadIndexesReadyPromise = null; + this.memory = new CloudRuntimeStore({ now }); + this.lastReadiness = this.blockedSummary({ + blocker: dbUrl || queryClient ? RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED : RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, + reason: dbUrl || queryClient + ? "Postgres runtime adapter is configured but has not completed a live schema query" + : "Postgres runtime adapter requires HWLAB_CLOUD_DB_URL or an injected query client" + }); + } + + summary() { + return { + ...this.lastReadiness, + counts: this.lastReadiness.counts ?? this.memory.summary().counts + }; + } + + async readiness() { + if (!this.dbUrl && !this.queryClient) { + this.lastReadiness = this.blockedSummary({ + blocker: RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, + reason: "Postgres runtime adapter is selected but HWLAB_CLOUD_DB_URL is not injected" + }); + return this.summary(); + } + + let rows; + try { + const result = await this.query( + "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ANY($1::text[])", + [Object.keys(requiredPostgresSchema)] + ); + rows = Array.isArray(result?.rows) ? result.rows : []; + } catch (error) { + this.lastReadiness = this.blockedSummary(classifyRuntimeDbError(error)); + return this.summary(); + } + + const schema = summarizeRuntimeSchema(rows); + if (!schema.ready) { + this.lastReadiness = this.blockedSummary({ + blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + reason: "Postgres runtime adapter connected, but required runtime tables or columns are missing", + schema, + connection: { + queryAttempted: true, + queryResult: "schema_blocked" + }, + gates: runtimeGates({ + ssl: readyGate(), + auth: readyGate(), + schema: blockedGate({ checked: true, blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }), + migration: notCheckedGate(), + durability: blockedGate({ checked: false, blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }) + }) + }); + return this.summary(); + } + + let migration; + try { + migration = await this.readMigrationReadiness(); + } catch (error) { + const classified = classifyRuntimeDbError(error); + const migrationReadBlocker = [ + RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, + RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ].includes(classified.blocker) + ? classified.blocker + : RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED; + this.lastReadiness = this.blockedSummary({ + blocker: migrationReadBlocker, + reason: migrationReadBlocker === classified.blocker + ? classified.reason + : "Postgres runtime schema is present, but the durable runtime migration ledger could not be proven", + schema, + migration: blockedRuntimeMigration({ errorCode: classified.connection?.errorCode }), + connection: { + queryAttempted: true, + queryResult: migrationReadBlocker === classified.blocker + ? classified.connection?.queryResult + : "migration_blocked", + errorCode: classified.connection?.errorCode ?? null + }, + gates: runtimeGates({ + ssl: classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : readyGate(), + auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED + ? notCheckedGate() + : readyGate(), + schema: readyGate(), + migration: blockedGate({ + checked: true, + blocker: migrationReadBlocker + }), + durability: blockedGate({ checked: false, blocker: migrationReadBlocker }) + }) + }); + return this.summary(); + } + + if (!migration.ready) { + this.lastReadiness = this.blockedSummary({ + blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + reason: "Postgres runtime schema is present, but the required source migration is not recorded", + schema, + migration, + connection: { + queryAttempted: true, + queryResult: "migration_blocked" + }, + gates: runtimeGates({ + ssl: readyGate(), + auth: readyGate(), + schema: readyGate(), + migration: blockedGate({ checked: true, blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }), + durability: blockedGate({ checked: false, blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }) + }) + }); + return this.summary(); + } + + try { + await this.ensureRuntimeReadIndexes(); + } catch (error) { + const classified = classifyRuntimeDbError(error); + this.lastReadiness = this.blockedSummary({ + ...classified, + schema, + migration, + reason: "Postgres runtime schema is present, but Workbench trace read indexes could not be ensured", + connection: { + queryAttempted: true, + queryResult: classified.connection?.queryResult ?? "index_blocked", + errorCode: classified.connection?.errorCode ?? null + }, + gates: runtimeGates({ + ssl: classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : readyGate(), + auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED + ? notCheckedGate() + : readyGate(), + schema: readyGate(), + migration: readyGate(), + durability: blockedGate({ checked: true, blocker: classified.blocker }) + }) + }); + return this.summary(); + } + + let counts; + try { + counts = await this.readCounts(); + } catch (error) { + const classified = classifyRuntimeDbError(error); + this.lastReadiness = this.blockedSummary({ + ...classified, + schema, + migration, + connection: { + queryAttempted: true, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? null + }, + gates: runtimeGates({ + ssl: classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : readyGate(), + auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED + ? notCheckedGate() + : readyGate(), + schema: readyGate(), + migration: readyGate(), + durability: blockedGate({ checked: true, blocker: classified.blocker }) + }) + }); + return this.summary(); + } + + this.lastReadiness = addRuntimeDurabilityContract({ + adapter: RUNTIME_STORE_KIND_POSTGRES, + durable: true, + durableRequested: true, + durableCapable: true, + ready: true, + status: "ready", + blocker: null, + reason: "Postgres durable runtime adapter completed live schema, migration, and read readiness queries", + liveRuntimeEvidence: true, + fixtureEvidence: false, + connection: { + queryAttempted: true, + queryResult: "durable_readiness_ready", + endpointRedacted: true, + valueRedacted: true + }, + schema, + migration, + gates: runtimeGates({ + ssl: readyGate(), + auth: readyGate(), + schema: readyGate(), + migration: readyGate(), + durability: readyGate() + }), + safety: runtimeSafety(), + adapterContract: postgresAdapterContract(), + counts + }); + return this.summary(); + } + + async registerGatewaySession(params = {}, requestMeta = {}) { + await this.assertReadyForWrites(); + const before = snapshotMemory(this.memory); + const result = this.memory.registerGatewaySession(params, requestMeta); + await this.persistChanges(before); + return withPersistence(result, this.summary()); + } + + async registerBoxResource(params = {}, requestMeta = {}) { + await this.assertReadyForWrites(); + const input = asObject(params.resource ?? params.boxResource ?? params, "boxResource"); + await this.hydrateGatewaySession(input.gatewaySessionId); + const before = snapshotMemory(this.memory); + const result = this.memory.registerBoxResource(params, requestMeta); + await this.persistChanges(before); + return withPersistence(result, this.summary()); + } + + async reportBoxCapabilities(params = {}, requestMeta = {}) { + await this.assertReadyForWrites(); + const inputs = normalizeCapabilityInputs(params); + for (const input of inputs) { + await this.hydrateBoxResource(input.resourceId); + } + const before = snapshotMemory(this.memory); + const result = this.memory.reportBoxCapabilities(params, requestMeta); + await this.persistChanges(before); + return withPersistence(result, this.summary()); + } + + async requestHardwareOperation(params = {}, requestMeta = {}) { + await this.assertReadyForWrites(); + await this.hydrateOperationRefs(params); + const before = snapshotMemory(this.memory); + const result = this.memory.requestHardwareOperation(params, requestMeta); + await this.persistChanges(before); + return withPersistence(result, this.summary()); + } + + async invokeHardwareShell(params = {}, requestMeta = {}) { + await this.assertReadyForWrites(); + await this.hydrateOperationRefs(params); + const before = snapshotMemory(this.memory); + const result = this.memory.invokeHardwareShell(params, requestMeta); + await this.persistChanges(before); + return withPersistence(result, this.summary()); + } + + async writeAuditEvent(params = {}, requestMeta = {}) { + await this.assertReadyForWrites(); + const before = snapshotMemory(this.memory); + const result = this.memory.writeAuditEvent(params, requestMeta); + await this.persistChanges(before); + return withPersistence(result, this.summary()); + } + + async queryAuditEvents(params = {}) { + await this.assertReadyForDurableReads("audit.event.query"); + const result = await this.queryDurableReadRows( + "audit.event.query", + "SELECT event_json FROM audit_events ORDER BY timestamp ASC", + [] + ); + const events = result.rows + .map((row) => parseJsonColumn(row.event_json, null)) + .filter(Boolean) + .filter((event) => matchesQuery(event, params, [ + "auditId", + "traceId", + "projectId", + "gatewaySessionId", + "operationId", + "action", + "targetId" + ])); + + return { + events: limitResults(events, params.limit), + count: events.length, + persistence: this.summary() + }; + } + + async writeEvidenceRecord(params = {}, requestMeta = {}) { + await this.assertReadyForWrites(); + const before = snapshotMemory(this.memory); + const result = this.memory.writeEvidenceRecord(params, requestMeta); + await this.persistChanges(before); + return withPersistence(result, this.summary()); + } + + async queryEvidenceRecords(params = {}) { + await this.assertReadyForDurableReads("evidence.record.query"); + const result = await this.queryDurableReadRows( + "evidence.record.query", + "SELECT metadata_json FROM evidence_records ORDER BY created_at ASC", + [] + ); + const records = result.rows + .map((row) => parseJsonColumn(row.metadata_json, null)) + .filter(Boolean) + .filter((record) => matchesQuery(record, params, [ + "evidenceId", + "projectId", + "operationId", + "agentSessionId", + "workerSessionId", + "kind", + "serviceId" + ])); + + return { + records: limitResults(records, params.limit), + count: records.length, + persistence: this.summary() + }; + } + + async writeAgentTraceEvent(params = {}, requestMeta = {}) { + const readiness = this.summary(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + await this.assertReadyForWrites(); + } + const result = this.memory.writeAgentTraceEvent(params, requestMeta); + await this.persistAgentTraceEvent(result.traceEvent); + return withPersistence(result, this.summary()); + } + + async queryAgentTraceEvents(params = {}) { + await this.assertReadyForDurableReads("agent.trace-events.query"); + const traceId = textOr(params.traceId, ""); + const sessionId = textOr(params.sessionId, ""); + const workerSessionId = textOr(params.workerSessionId, ""); + const level = textOr(params.level, ""); + const clauses = []; + const queryParams = []; + if (traceId) { + queryParams.push(traceId); + clauses.push(`trace_id = $${queryParams.length}`); + } + if (sessionId) { + queryParams.push(sessionId); + clauses.push(`agent_session_id = $${queryParams.length}`); + } + if (workerSessionId) { + queryParams.push(workerSessionId); + clauses.push(`worker_session_id = $${queryParams.length}`); + } + if (level) { + queryParams.push(level); + clauses.push(`level = $${queryParams.length}`); + } + const sql = `SELECT event_json FROM agent_trace_events${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY occurred_at ASC, id ASC`; + const result = await this.queryDurableReadRows("agent.trace-events.query", sql, queryParams); + const events = result.rows + .map((row) => parseJsonColumn(row.event_json, null)) + .filter(Boolean) + .filter((event) => matchesQuery(event, params, ["traceId", "sessionId", "workerSessionId", "level"])); + return { + events: limitResults(events, params.limit), + count: events.length, + persistence: this.summary() + }; + } + + async writeWorkbenchProjectionState(params = {}, requestMeta = {}) { + const readiness = this.summary(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + await this.assertReadyForWrites(); + } + const result = this.memory.writeWorkbenchProjectionState(params, requestMeta); + await this.persistWorkbenchProjectionState(result.projectionState); + return withPersistence(result, this.summary()); + } + + async getWorkbenchProjectionState(params = {}) { + const traceId = textOr(params.traceId, ""); + if (!traceId) return withPersistence({ projectionState: null, found: false }, this.summary()); + const result = await this.queryWorkbenchProjectionStates({ traceId, limit: 1 }); + const projectionState = result.states[0] ?? null; + return withPersistence({ projectionState, found: Boolean(projectionState) }, result.persistence); + } + + async queryWorkbenchProjectionStates(params = {}) { + await this.assertReadyForDurableReads("workbench.projection-state.query"); + const clauses = []; + const queryParams = []; + const addTextClause = (field, column) => { + const value = textOr(params[field], ""); + if (!value) return; + queryParams.push(value); + clauses.push(`${column} = $${queryParams.length}`); + }; + addTextClause("traceId", "trace_id"); + addTextClause("sessionId", "session_id"); + addTextClause("runId", "run_id"); + addTextClause("commandId", "command_id"); + const statuses = Array.isArray(params.projectionStatuses) ? params.projectionStatuses.map((item) => textOr(item, "")).filter(Boolean) : []; + if (statuses.length > 0) { + queryParams.push(statuses); + clauses.push(`projection_status = ANY($${queryParams.length})`); + } + const dueAt = textOr(params.dueAt, ""); + if (dueAt) { + queryParams.push(dueAt); + clauses.push(`(next_retry_at IS NULL OR next_retry_at <= $${queryParams.length})`); + } + const limit = positiveIntegerOrNull(params.limit); + const offset = positiveIntegerOrNull(params.offset); + let sql = `SELECT projection_json FROM workbench_projection_state${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY updated_at ASC, trace_id ASC`; + if (limit) { + queryParams.push(limit); + sql += ` LIMIT $${queryParams.length}`; + } + if (offset) { + queryParams.push(offset); + sql += ` OFFSET $${queryParams.length}`; + } + const result = await this.queryDurableReadRows("workbench.projection-state.query", sql, queryParams); + const states = result.rows.map((row) => parseJsonColumn(row.projection_json, null)).filter(Boolean); + return { + states, + count: states.length, + persistence: this.summary() + }; + } + + async allocateWorkbenchProjectedSeq(params = {}, requestMeta = {}) { + const readiness = this.summary(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + await this.assertReadyForWrites(); + } + const allocation = normalizeWorkbenchProjectionAllocation(params, requestMeta, this.now()); + const result = await this.withDurableTransaction("workbench.projected-seq.allocate", async (client) => { + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${allocation.traceId}`]); + const existing = await client.query( + "SELECT projected_seq FROM workbench_trace_events WHERE trace_id = $1 AND source_event_id = $2 ORDER BY projected_seq ASC LIMIT 1", + [allocation.traceId, allocation.sourceEventId] + ); + const existingSeq = nonNegativeInteger(existing.rows?.[0]?.projected_seq); + if (existingSeq > 0) return { projectedSeq: existingSeq, reused: true }; + const maxResult = await client.query( + "SELECT GREATEST(COALESCE((SELECT MAX(projected_seq) FROM workbench_trace_events WHERE trace_id = $1), 0), COALESCE((SELECT projected_seq FROM workbench_projection_checkpoints WHERE trace_id = $1), 0))::int AS max_projected_seq", + [allocation.traceId] + ); + const proposedSeq = nonNegativeInteger(maxResult.rows?.[0]?.max_projected_seq) + 1; + const reserve = await client.query( + "INSERT INTO workbench_projection_checkpoints (trace_id, session_id, turn_id, run_id, command_id, projected_seq, source_seq, source_event_id, projection_status, projection_health, terminal, sealed, diagnostic_json, checkpoint_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (trace_id) DO UPDATE SET projected_seq = GREATEST(workbench_projection_checkpoints.projected_seq + 1, EXCLUDED.projected_seq), updated_at = EXCLUDED.updated_at RETURNING projected_seq", + [allocation.traceId, allocation.sessionId, allocation.turnId, allocation.runId, allocation.commandId, proposedSeq, allocation.sourceSeq, allocation.sourceEventId, "projecting", "healthy", false, false, stableJson({ allocator: "workbench-projected-seq", valuesRedacted: true }), stableJson({ ...allocation, projectedSeq: proposedSeq, projectionStatus: "projecting", projectionHealth: "healthy", valuesPrinted: false }), allocation.createdAt, allocation.updatedAt] + ); + return { projectedSeq: nonNegativeInteger(reserve.rows?.[0]?.projected_seq) || proposedSeq, reused: false }; + }); + return { + ...allocation, + projectedSeq: result.projectedSeq, + reused: result.reused, + persistence: this.summary(), + valuesPrinted: false + }; + } + + async writeWorkbenchFacts(params = {}, requestMeta = {}) { + const readiness = this.summary(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + await this.assertReadyForWrites(); + } + const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); + const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, this.now()); + const txResult = await this.withDurableTransaction("workbench.facts.write", async (client) => { + const persistedEvents = []; + for (const event of aggregateEvents) persistedEvents.push(await this.persistWorkbenchAggregateEvent(event, client)); + const eventIndex = indexWorkbenchAggregateEvents(persistedEvents); + const inputFacts = facts.inputs.map((fact) => inputFactWithAggregateSeq(fact, eventIndex)); + for (const fact of inputFacts) await this.persistWorkbenchSessionInputFact(fact, client); + for (const fact of facts.sessions) await this.persistWorkbenchSessionFact(fact, client); + for (const fact of facts.messages) await this.persistWorkbenchMessageFact(fact, client); + for (const fact of facts.parts) await this.persistWorkbenchPartFact(fact, client); + for (const fact of facts.turns) await this.persistWorkbenchTurnFact(fact, client); + for (const fact of facts.traceEvents) await this.persistWorkbenchTraceEventFact(fact, client); + for (const fact of facts.checkpoints) await this.persistWorkbenchProjectionCheckpoint(fact, client); + for (const fact of facts.messages) { + const event = eventIndex.get(`message:${fact.messageId}`) ?? eventIndex.get(`messages:${fact.messageId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await this.persistWorkbenchProjectionOutbox({ + eventSeq: event?.eventSeq ?? null, + aggregateId: event?.aggregateId ?? null, + aggregateSeq: event?.aggregateSeq ?? 0, + projectionRevision: event?.projectionRevision ?? fact.projectedSeq, + traceId: fact.traceId, + sessionId: fact.sessionId, + turnId: fact.turnId, + messageId: fact.messageId, + projectedSeq: fact.projectedSeq, + sourceSeq: fact.sourceSeq, + sourceEventId: fact.sourceEventId, + commitType: "message", + terminal: Boolean(fact.terminal), + sealed: Boolean(fact.sealed), + payload: { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + createdAt: fact.updatedAt ?? fact.createdAt ?? this.now() + }, client); + } + for (const fact of facts.traceEvents) { + const event = eventIndex.get(`traceEvent:${fact.id}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await this.persistWorkbenchProjectionOutbox({ + eventSeq: event?.eventSeq ?? null, + aggregateId: event?.aggregateId ?? null, + aggregateSeq: event?.aggregateSeq ?? 0, + projectionRevision: event?.projectionRevision ?? fact.projectedSeq, + traceId: fact.traceId, + sessionId: fact.sessionId, + turnId: fact.turnId, + messageId: fact.messageId, + projectedSeq: fact.projectedSeq, + sourceSeq: fact.sourceSeq, + sourceEventId: fact.sourceEventId, + commitType: fact.terminal ? "terminal" : "event", + terminal: Boolean(fact.terminal), + sealed: Boolean(fact.sealed), + payload: { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + createdAt: fact.occurredAt ?? fact.updatedAt ?? this.now() + }, client); + } + for (const fact of facts.turns) { + if (fact.terminal || fact.sealed) { + const event = eventIndex.get(`turn:${fact.turnId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await this.persistWorkbenchProjectionOutbox({ + eventSeq: event?.eventSeq ?? null, + aggregateId: event?.aggregateId ?? null, + aggregateSeq: event?.aggregateSeq ?? 0, + projectionRevision: event?.projectionRevision ?? fact.projectedSeq, + traceId: fact.traceId, + sessionId: fact.sessionId, + turnId: fact.turnId, + messageId: fact.messageId, + projectedSeq: fact.projectedSeq, + sourceSeq: fact.sourceSeq, + sourceEventId: fact.sourceEventId, + commitType: "terminal", + terminal: true, + sealed: true, + payload: { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, valuesRedacted: true }, + createdAt: fact.updatedAt ?? this.now() + }, client); + } + } + return { events: persistedEvents, inputFacts }; + }); + const persistedFacts = { ...facts, inputs: txResult.inputFacts }; + this.memory.writeWorkbenchFacts({ facts: persistedFacts }, requestMeta); + return withPersistence({ written: true, facts: persistedFacts, events: txResult.events }, this.summary()); + } + + async queryWorkbenchFacts(params = {}) { + await this.assertReadyForDurableReads("workbench.facts.query"); + const families = workbenchFactFamilySet(params); + // A single Workbench read can request several fact families. Running those + // SQL reads in parallel lets one HTTP request occupy the whole Postgres + // pool, which makes the control+observer pages amplify into 503s under + // periodic refresh. Keep this read model bounded to one connection per + // request; the page-level latency is preferable to pool starvation. + const entries = []; + entries.push(["inputs", families.has("inputs") ? await this.queryWorkbenchFactRows("workbench.session-inputs.query", "workbench_session_inputs", "input_json", params, { inputId: "input_id", sessionId: "session_id", turnId: "turn_id", traceId: "trace_id", messageId: "message_id", commandId: "command_id", delivery: "delivery", status: "status" }) : []]); + entries.push(["sessions", families.has("sessions") ? await this.queryWorkbenchFactRows("workbench.sessions.query", "workbench_sessions", "session_json", params, { sessionId: "session_id", ownerUserId: "owner_user_id", projectId: "project_id", conversationId: "conversation_id", threadId: "thread_id", traceId: "last_trace_id", lastTraceId: "last_trace_id", status: "status" }) : []]); + entries.push(["messages", families.has("messages") ? await this.queryWorkbenchFactRows("workbench.messages.query", "workbench_messages", "message_json", params, { messageId: "message_id", sessionId: "session_id", turnId: "turn_id", traceId: "trace_id", role: "role", status: "status" }) : []]); + entries.push(["parts", families.has("parts") ? await this.queryWorkbenchFactRows("workbench.parts.query", "workbench_parts", "part_json", params, { partId: "part_id", messageId: "message_id", sessionId: "session_id", turnId: "turn_id", traceId: "trace_id", partType: "part_type", status: "status" }) : []]); + entries.push(["turns", families.has("turns") ? await this.queryWorkbenchFactRows("workbench.turns.query", "workbench_turns", "turn_json", params, { turnId: "turn_id", sessionId: "session_id", traceId: "trace_id", messageId: "message_id", status: "status" }) : []]); + entries.push(["traceEvents", families.has("traceEvents") ? await this.queryWorkbenchFactRows("workbench.trace-events.query", "workbench_trace_events", "event_json", params, { id: "id", traceId: "trace_id", sessionId: "session_id", turnId: "turn_id", messageId: "message_id", eventType: "event_type" }) : []]); + entries.push(["checkpoints", families.has("checkpoints") ? await this.queryWorkbenchFactRows("workbench.projection-checkpoints.query", "workbench_projection_checkpoints", "checkpoint_json", params, { traceId: "trace_id", sessionId: "session_id", turnId: "turn_id", runId: "run_id", commandId: "command_id", projectionStatus: "projection_status", projectionHealth: "projection_health" }) : []]); + const facts = Object.fromEntries(entries); + return withPersistence({ facts, count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0) }, this.summary()); + } + + async queryWorkbenchFactRows(method, table, jsonColumn, params = {}, columnMap = {}) { + const family = workbenchFactFamilyForTable(table); + const queryParams = []; + const clauses = []; + for (const [field, column] of Object.entries(columnMap)) { + const value = textOr(params[field], ""); + if (value) { + queryParams.push(value); + clauses.push(`${column} = $${queryParams.length}`); + } + const values = textList(params[`${field}s`]); + if (values.length > 0) { + queryParams.push(values); + clauses.push(`${column} = ANY($${queryParams.length})`); + } + } + const limit = positiveIntegerOrNull(params.limit); + if (family === "traceEvents") { + const afterProjectedSeq = nonNegativeInteger(params.afterProjectedSeq); + if (afterProjectedSeq > 0) { + queryParams.push(afterProjectedSeq); + clauses.push(`projected_seq > $${queryParams.length}`); + } + let sql = `SELECT ${jsonColumn} FROM ${table}${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY projected_seq ASC`; + if (limit) { + queryParams.push(limit); + sql += ` LIMIT $${queryParams.length}`; + } + const result = await this.queryDurableReadRows(method, sql, queryParams); + return result.rows.map((row) => parseJsonColumn(row[jsonColumn], null)).filter(Boolean); + } + const orderDirection = workbenchFactOrder(params, family) === "updated_desc" ? "DESC" : "ASC"; + const useSessionSummaryProjection = table === "workbench_sessions" && params.sessionProjection === "summary"; + const orderClause = family === "inputs" && orderDirection === "ASC" ? "admitted_seq ASC, input_id ASC" : `updated_at ${orderDirection}`; + let sql = `SELECT ${useSessionSummaryProjection ? workbenchSessionSummarySelectClause() : jsonColumn} FROM ${table}${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY ${orderClause}`; + if (limit) { + queryParams.push(limit); + sql += ` LIMIT $${queryParams.length}`; + } + const result = await this.queryDurableReadRows(method, sql, queryParams); + if (useSessionSummaryProjection) return result.rows.map(workbenchSessionSummaryFactFromRow).filter(Boolean); + return result.rows.map((row) => parseJsonColumn(row[jsonColumn], null)).filter(Boolean); + } + + async hydrateOperationRefs(params = {}) { + await this.hydrateGatewaySession(params.gatewaySessionId); + await this.hydrateBoxResource(params.resourceId); + await this.hydrateBoxCapability(params.capabilityId); + } + + async hydrateGatewaySession(gatewaySessionId) { + if (!gatewaySessionId || this.memory.gatewaySessions.has(gatewaySessionId)) return; + const result = await this.query("SELECT gateway_session_json FROM gateway_sessions WHERE id = $1 LIMIT 1", [ + gatewaySessionId + ]); + const record = parseJsonColumn(result.rows?.[0]?.gateway_session_json, null); + if (record) this.memory.gatewaySessions.set(record.gatewaySessionId, record); + } + + async hydrateBoxResource(resourceId) { + if (!resourceId || this.memory.boxResources.has(resourceId)) return; + const result = await this.query("SELECT resource_json FROM box_resources WHERE id = $1 LIMIT 1", [ + resourceId + ]); + const record = parseJsonColumn(result.rows?.[0]?.resource_json, null); + if (record) { + await this.hydrateGatewaySession(record.gatewaySessionId); + this.memory.boxResources.set(record.resourceId, record); + } + } + + async hydrateBoxCapability(capabilityId) { + if (!capabilityId || this.memory.boxCapabilities.has(capabilityId)) return; + const result = await this.query("SELECT capability_json FROM box_capabilities WHERE id = $1 LIMIT 1", [ + capabilityId + ]); + const record = parseJsonColumn(result.rows?.[0]?.capability_json, null); + if (record) { + await this.hydrateBoxResource(record.resourceId); + this.memory.boxCapabilities.set(record.capabilityId, record); + } + } + + async persistChanges(before) { + for (const record of newRecords(this.memory.gatewaySessions, before.gatewaySessions)) { + await this.persistGatewaySession(record); + } + for (const record of newRecords(this.memory.boxResources, before.boxResources)) { + await this.persistBoxResource(record); + } + for (const record of newRecords(this.memory.boxCapabilities, before.boxCapabilities)) { + await this.persistBoxCapability(record); + } + for (const record of changedRecords(this.memory.hardwareOperations, before.hardwareOperations)) { + await this.persistHardwareOperation(record); + } + for (const record of newRecords(this.memory.auditEvents, before.auditEvents)) { + await this.persistAuditEvent(record); + } + for (const record of newRecords(this.memory.evidenceRecords, before.evidenceRecords)) { + await this.persistEvidenceRecord(record); + } + for (const record of newRecords(this.memory.agentTraceEvents, before.agentTraceEvents)) { + await this.persistAgentTraceEvent(record); + } + for (const record of changedRecords(this.memory.workbenchProjectionStates, before.workbenchProjectionStates)) { + await this.persistWorkbenchProjectionState(record); + } + for (const record of changedRecords(this.memory.workbenchSessionInputs, before.workbenchSessionInputs)) { + await this.persistWorkbenchSessionInputFact(record); + } + for (const record of changedRecords(this.memory.workbenchSessions, before.workbenchSessions)) { + await this.persistWorkbenchSessionFact(record); + } + for (const record of changedRecords(this.memory.workbenchMessages, before.workbenchMessages)) { + await this.persistWorkbenchMessageFact(record); + } + for (const record of changedRecords(this.memory.workbenchParts, before.workbenchParts)) { + await this.persistWorkbenchPartFact(record); + } + for (const record of changedRecords(this.memory.workbenchTurns, before.workbenchTurns)) { + await this.persistWorkbenchTurnFact(record); + } + for (const record of changedRecords(this.memory.workbenchTraceEvents, before.workbenchTraceEvents)) { + await this.persistWorkbenchTraceEventFact(record); + } + for (const record of changedRecords(this.memory.workbenchProjectionCheckpoints, before.workbenchProjectionCheckpoints)) { + await this.persistWorkbenchProjectionCheckpoint(record); + } + } + + async assertReadyForWrites() { + if (!this.dbUrl && !this.queryClient) { + const readiness = this.blockedSummary({ + blocker: RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, + reason: "Postgres runtime adapter is selected but HWLAB_CLOUD_DB_URL is not injected" + }); + this.lastReadiness = readiness; + throw new HwlabProtocolError("Postgres durable runtime adapter is not ready for writes", { + code: ERROR_CODES.internalError, + data: { + adapter: RUNTIME_STORE_KIND_POSTGRES, + blocker: readiness.blocker, + status: readiness.status, + schemaReady: Boolean(readiness.schema?.ready) + }, + context: { + result: "failed" + } + }); + } + } + + async assertReadyForDurableReads(method) { + const current = this.summary(); + if (isReadyForDurableRuntimeRead(current)) return; + const readiness = await this.readiness(); + if (isReadyForDurableRuntimeRead(readiness)) return; + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + throw durableRuntimeReadBlockedError(method, readiness); + } + } + + async queryDurableReadRows(method, sql, params = []) { + try { + return await this.query(sql, params); + } catch (error) { + this.lastReadiness = this.blockedSummary({ + ...classifyRuntimeDbError(error), + reason: `Postgres durable runtime adapter could not complete ${method} read query` + }); + throw durableRuntimeReadBlockedError(method, this.summary()); + } + } + + async persistGatewaySession(record) { + await this.query( + "INSERT INTO gateway_sessions (id, project_id, gateway_service_id, status, started_at, ended_at, gateway_session_json) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, gateway_service_id = EXCLUDED.gateway_service_id, status = EXCLUDED.status, started_at = EXCLUDED.started_at, ended_at = EXCLUDED.ended_at, gateway_session_json = EXCLUDED.gateway_session_json", + [ + record.gatewaySessionId, + record.projectId, + record.serviceId, + record.status, + record.startedAt, + record.stoppedAt ?? null, + stableJson(record) + ] + ); + } + + async persistBoxResource(record) { + await this.query( + "INSERT INTO box_resources (id, project_id, gateway_session_id, resource_state, labels_json, resource_json, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, gateway_session_id = EXCLUDED.gateway_session_id, resource_state = EXCLUDED.resource_state, labels_json = EXCLUDED.labels_json, resource_json = EXCLUDED.resource_json, updated_at = EXCLUDED.updated_at", + [ + record.resourceId, + record.projectId, + record.gatewaySessionId, + record.state, + stableJson(record.metadata ?? {}), + stableJson(record), + record.updatedAt + ] + ); + } + + async persistBoxCapability(record) { + await this.query( + "INSERT INTO box_capabilities (id, box_resource_id, capability_type, capability_json, updated_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (id) DO UPDATE SET box_resource_id = EXCLUDED.box_resource_id, capability_type = EXCLUDED.capability_type, capability_json = EXCLUDED.capability_json, updated_at = EXCLUDED.updated_at", + [ + record.capabilityId, + record.resourceId, + record.name, + stableJson(record), + record.updatedAt + ] + ); + } + + async persistHardwareOperation(record) { + await this.query( + "INSERT INTO hardware_operations (id, project_id, requested_by, operation_type, operation_json, status, requested_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, requested_by = EXCLUDED.requested_by, operation_type = EXCLUDED.operation_type, operation_json = EXCLUDED.operation_json, status = EXCLUDED.status, requested_at = EXCLUDED.requested_at, updated_at = EXCLUDED.updated_at", + [ + record.operationId, + record.projectId, + record.requestedBy, + record.input?.shell ? "hardware.invoke.shell" : "hardware.operation.request", + stableJson(record), + record.status, + record.requestedAt, + record.updatedAt + ] + ); + } + + async persistAuditEvent(record) { + await this.query( + "INSERT INTO audit_events (id, request_id, actor, source, operation, target, result, timestamp, event_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (id) DO UPDATE SET request_id = EXCLUDED.request_id, actor = EXCLUDED.actor, source = EXCLUDED.source, operation = EXCLUDED.operation, target = EXCLUDED.target, result = EXCLUDED.result, timestamp = EXCLUDED.timestamp, event_json = EXCLUDED.event_json", + [ + record.auditId, + record.traceId, + stableJson({ type: record.actorType, id: record.actorId }), + stableJson({ serviceId: record.serviceId, environment: record.environment }), + record.action, + stableJson({ type: record.targetType, id: record.targetId }), + record.outcome ?? "accepted", + record.occurredAt, + stableJson(record) + ] + ); + } + + async persistEvidenceRecord(record) { + await this.query( + "INSERT INTO evidence_records (id, project_id, operation_id, evidence_type, uri, metadata_json, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, operation_id = EXCLUDED.operation_id, evidence_type = EXCLUDED.evidence_type, uri = EXCLUDED.uri, metadata_json = EXCLUDED.metadata_json, created_at = EXCLUDED.created_at", + [ + record.evidenceId, + record.projectId, + record.operationId, + record.kind, + record.uri, + stableJson(record), + record.createdAt + ] + ); + } + + async persistAgentTraceEvent(record) { + await this.query( + "INSERT INTO agent_trace_events (id, trace_id, agent_session_id, worker_session_id, level, message, event_json, occurred_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET trace_id = EXCLUDED.trace_id, agent_session_id = EXCLUDED.agent_session_id, worker_session_id = EXCLUDED.worker_session_id, level = EXCLUDED.level, message = EXCLUDED.message, event_json = EXCLUDED.event_json, occurred_at = EXCLUDED.occurred_at", + [ + record.id, + record.traceId, + record.agentSessionId, + record.workerSessionId, + record.level, + record.message, + stableJson(record.event), + record.occurredAt + ] + ); + } + + async persistWorkbenchProjectionState(record) { + await this.query( + "INSERT INTO workbench_projection_state (trace_id, session_id, conversation_id, thread_id, run_id, command_id, last_agentrun_seq, last_projected_seq, upstream_latest_seq, projection_status, projection_health, result_sync_state, last_projected_at, last_result_sync_at, last_error_code, last_error_message, failure_count, next_retry_at, projection_json, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) ON CONFLICT (trace_id) DO UPDATE SET session_id = EXCLUDED.session_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, run_id = EXCLUDED.run_id, command_id = EXCLUDED.command_id, last_agentrun_seq = EXCLUDED.last_agentrun_seq, last_projected_seq = EXCLUDED.last_projected_seq, upstream_latest_seq = EXCLUDED.upstream_latest_seq, projection_status = EXCLUDED.projection_status, projection_health = EXCLUDED.projection_health, result_sync_state = EXCLUDED.result_sync_state, last_projected_at = EXCLUDED.last_projected_at, last_result_sync_at = EXCLUDED.last_result_sync_at, last_error_code = EXCLUDED.last_error_code, last_error_message = EXCLUDED.last_error_message, failure_count = EXCLUDED.failure_count, next_retry_at = EXCLUDED.next_retry_at, projection_json = EXCLUDED.projection_json, updated_at = EXCLUDED.updated_at", + [ + record.traceId, + record.sessionId, + record.conversationId, + record.threadId, + record.runId, + record.commandId, + record.lastAgentRunSeq, + record.lastProjectedSeq, + record.upstreamLatestSeq, + record.projectionStatus, + record.projectionHealth, + record.resultSyncState, + record.lastProjectedAt, + record.lastResultSyncAt, + record.lastErrorCode, + record.lastErrorMessage, + record.failureCount, + record.nextRetryAt, + stableJson(record), + record.createdAt, + record.updatedAt + ] + ); + } + + async persistWorkbenchAggregateEvent(record, client = this) { + const existing = await client.query( + "SELECT event_seq, aggregate_seq, projection_revision FROM workbench_events WHERE event_id = $1 OR ($2::text IS NOT NULL AND aggregate_id = $3 AND source_event_id = $2) ORDER BY event_seq ASC LIMIT 1", + [record.eventId, record.sourceEventId, record.aggregateId] + ); + const existingRow = existing.rows?.[0] ?? null; + if (existingRow) { + return { + ...record, + eventSeq: Number(existingRow.event_seq), + aggregateSeq: Number(existingRow.aggregate_seq), + projectionRevision: Number(existingRow.projection_revision) + }; + } + await client.query( + "INSERT INTO workbench_event_sequences (aggregate_id, aggregate_type, session_id, turn_id, trace_id, last_seq, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,0,$6,$7) ON CONFLICT (aggregate_id) DO UPDATE SET aggregate_type = EXCLUDED.aggregate_type, session_id = COALESCE(workbench_event_sequences.session_id, EXCLUDED.session_id), turn_id = COALESCE(workbench_event_sequences.turn_id, EXCLUDED.turn_id), trace_id = COALESCE(workbench_event_sequences.trace_id, EXCLUDED.trace_id), updated_at = EXCLUDED.updated_at", + [record.aggregateId, record.aggregateType, record.sessionId, record.turnId, record.traceId, record.committedAt, record.committedAt] + ); + const reserved = await client.query( + "UPDATE workbench_event_sequences SET last_seq = last_seq + 1, updated_at = $2 WHERE aggregate_id = $1 RETURNING last_seq", + [record.aggregateId, record.committedAt] + ); + const aggregateSeq = nonNegativeInteger(reserved.rows?.[0]?.last_seq); + const projectionRevision = nonNegativeInteger(record.projectionRevision) || aggregateSeq; + const inserted = await client.query( + "INSERT INTO workbench_events (event_id, aggregate_id, aggregate_type, aggregate_seq, session_id, turn_id, trace_id, message_id, source_run_id, source_command_id, source_seq, source_event_id, event_type, projection_revision, terminal, sealed, payload_json, occurred_at, committed_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT (event_id) DO UPDATE SET projection_revision = GREATEST(workbench_events.projection_revision, EXCLUDED.projection_revision), terminal = workbench_events.terminal OR EXCLUDED.terminal, sealed = workbench_events.sealed OR EXCLUDED.sealed, payload_json = EXCLUDED.payload_json, committed_at = EXCLUDED.committed_at RETURNING event_seq, aggregate_seq, projection_revision", + [record.eventId, record.aggregateId, record.aggregateType, aggregateSeq, record.sessionId, record.turnId, record.traceId, record.messageId, record.sourceRunId, record.sourceCommandId, record.sourceSeq, record.sourceEventId, record.eventType, projectionRevision, Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.occurredAt, record.committedAt] + ); + const row = inserted.rows?.[0] ?? {}; + return { + ...record, + eventSeq: Number(row.event_seq), + aggregateSeq: Number(row.aggregate_seq) || aggregateSeq, + projectionRevision: Number(row.projection_revision) || projectionRevision + }; + } + + async persistWorkbenchSessionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = EXCLUDED.owner_user_id, project_id = EXCLUDED.project_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at", + [record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + + async persistWorkbenchSessionInputFact(record, client = this) { + await client.query( + "INSERT INTO workbench_session_inputs (input_id, session_id, turn_id, trace_id, message_id, command_id, delivery, admitted_seq, promoted_seq, status, error_code, source_event_id, input_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (input_id) DO UPDATE SET session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, message_id = EXCLUDED.message_id, command_id = COALESCE(EXCLUDED.command_id, workbench_session_inputs.command_id), delivery = EXCLUDED.delivery, admitted_seq = CASE WHEN workbench_session_inputs.admitted_seq > 0 THEN workbench_session_inputs.admitted_seq ELSE EXCLUDED.admitted_seq END, promoted_seq = COALESCE(EXCLUDED.promoted_seq, workbench_session_inputs.promoted_seq), status = EXCLUDED.status, error_code = EXCLUDED.error_code, source_event_id = COALESCE(EXCLUDED.source_event_id, workbench_session_inputs.source_event_id), input_json = EXCLUDED.input_json, updated_at = EXCLUDED.updated_at", + [record.inputId, record.sessionId, record.turnId, record.traceId, record.messageId, record.commandId, record.delivery, record.admittedSeq, record.promotedSeq, record.status, record.errorCode, record.sourceEventId, stableJson(record), record.createdAt, record.updatedAt] + ); + } + + async persistWorkbenchMessageFact(record, client = this) { + await client.query( + "INSERT INTO workbench_messages (message_id, session_id, turn_id, trace_id, role, status, projected_seq, source_seq, source_event_id, terminal, sealed, message_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (message_id) DO UPDATE SET session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, role = EXCLUDED.role, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, message_json = EXCLUDED.message_json, updated_at = EXCLUDED.updated_at", + [record.messageId, record.sessionId, record.turnId, record.traceId, record.role, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + + async persistWorkbenchPartFact(record, client = this) { + await client.query( + "INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, part_json = EXCLUDED.part_json, updated_at = EXCLUDED.updated_at", + [record.partId, record.messageId, record.sessionId, record.turnId, record.traceId, record.partIndex, record.partType, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + + async persistWorkbenchTurnFact(record, client = this) { + await client.query( + "INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = EXCLUDED.message_id, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, final_response_json = EXCLUDED.final_response_json, diagnostic_json = EXCLUDED.diagnostic_json, turn_json = EXCLUDED.turn_json, updated_at = EXCLUDED.updated_at", + [record.turnId, record.sessionId, record.traceId, record.messageId, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record.finalResponse), stableJson(record.diagnostic), stableJson(record), record.createdAt, record.updatedAt] + ); + } + + async persistWorkbenchTraceEventFact(record, client = this) { + await client.query( + "INSERT INTO workbench_trace_events (id, trace_id, session_id, turn_id, message_id, source_seq, source_event_id, projected_seq, event_type, terminal, sealed, event_json, occurred_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (id) DO UPDATE SET trace_id = EXCLUDED.trace_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, message_id = EXCLUDED.message_id, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, projected_seq = EXCLUDED.projected_seq, event_type = EXCLUDED.event_type, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, event_json = EXCLUDED.event_json, occurred_at = EXCLUDED.occurred_at, updated_at = EXCLUDED.updated_at", + [record.id, record.traceId, record.sessionId, record.turnId, record.messageId, record.sourceSeq, record.sourceEventId, record.projectedSeq, record.eventType, record.terminal, record.sealed, stableJson(record), record.occurredAt, record.updatedAt] + ); + } + + async persistWorkbenchProjectionOutbox(record, client = this) { + await client.query( + "INSERT INTO workbench_projection_outbox (event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)", + [record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()] + ); + } + + async readWorkbenchProjectionOutbox({ afterSeq = 0, limit = 100, traceId = null, sessionId = null } = {}) { + const params = []; + let where = "outbox_seq > $1"; + params.push(Number(afterSeq) || 0); + let paramIdx = 2; + if (traceId) { + where += ` AND trace_id = $${paramIdx}`; + params.push(traceId); + paramIdx += 1; + } + if (sessionId) { + where += ` AND session_id = $${paramIdx}`; + params.push(sessionId); + paramIdx += 1; + } + params.push(Math.min(Math.max(Number(limit) || 100, 1), 500)); + const result = await this.query( + `SELECT outbox_seq, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`, + params + ); + return (result.rows ?? []).map((row) => ({ + outboxSeq: Number(row.outbox_seq), + eventSeq: row.event_seq === null || row.event_seq === undefined ? null : Number(row.event_seq), + aggregateId: row.aggregate_id, + aggregateSeq: Number(row.aggregate_seq ?? 0), + projectionRevision: Number(row.projection_revision ?? 0), + traceId: row.trace_id, + sessionId: row.session_id, + turnId: row.turn_id, + messageId: row.message_id, + projectedSeq: Number(row.projected_seq), + sourceSeq: Number(row.source_seq), + sourceEventId: row.source_event_id, + commitType: row.commit_type, + terminal: Boolean(row.terminal), + sealed: Boolean(row.sealed), + payload: typeof row.payload_json === "string" ? JSON.parse(row.payload_json) : row.payload_json, + createdAt: row.created_at, + valuesPrinted: false + })); + } + + async persistWorkbenchProjectionCheckpoint(record, client = this) { + await client.query( + "INSERT INTO workbench_projection_checkpoints (trace_id, session_id, turn_id, run_id, command_id, projected_seq, source_seq, source_event_id, projection_status, projection_health, terminal, sealed, diagnostic_json, checkpoint_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (trace_id) DO UPDATE SET session_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.session_id ELSE workbench_projection_checkpoints.session_id END, turn_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.turn_id ELSE workbench_projection_checkpoints.turn_id END, run_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.run_id ELSE workbench_projection_checkpoints.run_id END, command_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.command_id ELSE workbench_projection_checkpoints.command_id END, projected_seq = GREATEST(workbench_projection_checkpoints.projected_seq, EXCLUDED.projected_seq), source_seq = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.source_seq ELSE workbench_projection_checkpoints.source_seq END, source_event_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.source_event_id ELSE workbench_projection_checkpoints.source_event_id END, projection_status = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.projection_status ELSE workbench_projection_checkpoints.projection_status END, projection_health = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.projection_health ELSE workbench_projection_checkpoints.projection_health END, terminal = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.terminal ELSE workbench_projection_checkpoints.terminal END, sealed = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.sealed ELSE workbench_projection_checkpoints.sealed END, diagnostic_json = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.diagnostic_json ELSE workbench_projection_checkpoints.diagnostic_json END, checkpoint_json = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.checkpoint_json ELSE workbench_projection_checkpoints.checkpoint_json END, updated_at = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.updated_at ELSE workbench_projection_checkpoints.updated_at END", + [record.traceId, record.sessionId, record.turnId, record.runId, record.commandId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.projectionStatus, record.projectionHealth, record.terminal, record.sealed, stableJson(record.diagnostic), stableJson(record), record.createdAt, record.updatedAt] + ); + } + + async readCounts() { + const counts = {}; + for (const table of postgresCountTables) { + const result = await this.query(`SELECT COUNT(*)::int AS count FROM ${table}`, []); + counts[toCountKey(table)] = Number(result.rows?.[0]?.count ?? 0); + } + return counts; + } + + async readMigrationReadiness() { + const result = await this.query( + `SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`, + [CLOUD_CORE_MIGRATION_ID] + ); + const row = result.rows?.[0] ?? null; + const ready = + row?.id === CLOUD_CORE_MIGRATION_ID && + row?.schema_version === CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION; + return { + checked: true, + ready, + table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + appliedMigrationId: row?.id ?? null, + appliedSchemaVersion: row?.schema_version ?? null, + missing: !row + }; + } + + async ensureRuntimeReadIndexes() { + if (this.runtimeReadIndexesReady) return; + if (!this.runtimeReadIndexesReadyPromise) { + this.runtimeReadIndexesReadyPromise = (async () => { + for (const sql of postgresRuntimeReadIndexes) { + await this.query(sql, []); + } + this.runtimeReadIndexesReady = true; + })(); + } + try { + await this.runtimeReadIndexesReadyPromise; + } finally { + if (!this.runtimeReadIndexesReady) this.runtimeReadIndexesReadyPromise = null; + } + } + + recordRuntimeDbOperationFailure(label, error, retry = {}) { + const classified = classifyRuntimeDbError(error); + this.lastReadiness = this.blockedSummary({ + ...classified, + reason: `Postgres durable runtime adapter could not complete ${label}` + }); + this.logger?.warn?.({ + event: "postgres_runtime_operation_failed", + operation: label, + blocker: classified.blocker, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? "UNKNOWN", + retryable: classified.retryable === true, + transient: classified.transient === true, + retryAfterMs: classified.retryAfterMs ?? null, + retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, + retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, + retrying: retry.retrying === true, + retryDelayMs: Number.isFinite(Number(retry.delayMs)) ? Number(retry.delayMs) : null, + retryLabel: retry.label ?? null, + valuesRedacted: true, + endpointRedacted: true + }); + } + + resetPostgresPoolAfterTransientFailure(classified, retry = {}) { + if (classified?.connection?.queryResult !== "connect_timeout") return false; + const pool = this.pool; + if (!pool) return false; + const now = Date.now(); + const cooldownMs = this.runtimePoolResetCooldownMs(); + if (this.postgresPoolResetInFlight) { + this.logger?.warn?.({ + event: "postgres_runtime_pool_reset_skipped", + reason: "in_flight", + blocker: classified.blocker, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? "UNKNOWN", + retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, + retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, + cooldownMs, + poolStats: postgresPoolStats(pool), + valuesRedacted: true, + endpointRedacted: true + }); + return false; + } + if (now - this.postgresPoolResetLastAtMs < cooldownMs) { + this.logger?.warn?.({ + event: "postgres_runtime_pool_reset_skipped", + reason: "cooldown", + blocker: classified.blocker, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? "UNKNOWN", + retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, + retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, + cooldownMs, + poolResetLastAtMs: this.postgresPoolResetLastAtMs, + poolStats: postgresPoolStats(pool), + valuesRedacted: true, + endpointRedacted: true + }); + return false; + } + this.pool = null; + this.postgresPoolResetLastAtMs = now; + this.logger?.warn?.({ + event: "postgres_runtime_pool_reset_after_transient_failure", + blocker: classified.blocker, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? "UNKNOWN", + retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, + retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, + retrying: retry.retrying === true, + cooldownMs, + poolStats: postgresPoolStats(pool), + valuesRedacted: true, + endpointRedacted: true + }); + if (typeof pool.end === "function") { + this.postgresPoolResetInFlight = Promise.resolve() + .then(() => pool.end()) + .catch((error) => this.logger?.warn?.({ + event: "postgres_runtime_pool_reset_failed", + errorCode: error?.code ?? "UNKNOWN", + valuesRedacted: true, + endpointRedacted: true + })) + .finally(() => { + this.postgresPoolResetInFlight = null; + }); + } + return true; + } + + runtimePoolResetCooldownMs() { + const env = this.env ?? {}; + return normalizePositiveInteger(env[RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV], DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS); + } + + async query(sql, params = []) { + const retry = this.runtimeQueryRetryPolicy(); + for (let attempt = 1; attempt <= retry.maxAttempts; attempt += 1) { + try { + const client = await this.getQueryClient(); + return await client.query(sql, params); + } catch (error) { + const classified = classifyRuntimeDbError(error); + const retrying = classified.retryable === true && classified.transient === true && attempt < retry.maxAttempts; + const delayMs = retrying ? Math.min(retry.maxDelayMs, retry.initialDelayMs * (2 ** Math.max(0, attempt - 1))) : null; + this.recordRuntimeDbOperationFailure("query", error, { + attempt, + maxAttempts: retry.maxAttempts, + retrying, + delayMs, + label: `${attempt}/${retry.maxAttempts}` + }); + this.resetPostgresPoolAfterTransientFailure(classified, { + attempt, + maxAttempts: retry.maxAttempts, + retrying, + delayMs, + label: `${attempt}/${retry.maxAttempts}` + }); + if (!retrying) throw error; + await delayRuntimeDbQueryRetry(delayMs); + } + } + } + + runtimeQueryRetryPolicy() { + const env = this.env ?? {}; + return { + maxAttempts: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS), + initialDelayMs: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS), + maxDelayMs: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS) + }; + } + + async withDurableTransaction(label, fn) { + try { + const client = await this.getQueryClient(); + if (typeof client.connect === "function") { + const tx = await client.connect(); + try { + await tx.query("BEGIN"); + const result = await fn(tx); + await tx.query("COMMIT"); + return result; + } catch (error) { + try { await tx.query("ROLLBACK"); } catch {} + throw error; + } finally { + tx.release?.(); + } + } + await client.query("BEGIN"); + try { + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (error) { + try { await client.query("ROLLBACK"); } catch {} + throw error; + } + } catch (error) { + this.recordRuntimeDbOperationFailure(label || "transaction", error); + throw error; + } + } + + async getQueryClient() { + if (this.queryClient) { + return this.queryClient; + } + + if (this.pool) { + return this.pool; + } + + let pg; + try { + pg = await (this.pgModuleLoader ? this.pgModuleLoader() : import("pg")); + } catch (error) { + if (error?.code === "ERR_MODULE_NOT_FOUND") { + const driverError = new Error("Postgres runtime adapter requires the pg package"); + driverError.code = "HWLAB_PG_DRIVER_MISSING"; + throw driverError; + } + throw error; + } + + const Pool = pg.Pool ?? pg.default?.Pool; + if (typeof Pool !== "function") { + const driverError = new Error("Postgres runtime adapter could not load pg.Pool"); + driverError.code = "HWLAB_PG_DRIVER_MISSING"; + throw driverError; + } + + this.pool = new Pool(buildPostgresPoolConfig({ + dbUrl: this.dbUrl, + sslMode: this.sslMode, + timeoutMs: this.env?.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS, + queryTimeoutMs: this.env?.[RUNTIME_DB_QUERY_TIMEOUT_MS_ENV], + poolMax: this.env?.[RUNTIME_DB_POOL_MAX_ENV] + })); + if (typeof this.pool.on === "function") { + this.pool.on("error", (error) => this.handlePostgresPoolError(error)); + } + return this.pool; + } + + handlePostgresPoolError(error) { + const classified = classifyRuntimeDbError(error); + this.lastReadiness = this.blockedSummary(classified); + this.logger?.warn?.({ + event: "postgres_runtime_pool_error", + blocker: classified.blocker, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? "UNKNOWN", + retryable: classified.retryable === true, + transient: classified.transient === true, + retryAfterMs: classified.retryAfterMs ?? null, + valuesRedacted: true, + endpointRedacted: true + }); + } + + blockedSummary({ blocker, reason, schema, migration, connection, gates, retryable, transient, retryAfterMs }) { + const blockedSchema = schema ?? { + ready: false, + checked: false, + missingTables: [], + missingColumns: [] + }; + const blockedMigration = migration ?? notCheckedRuntimeMigration(); + return addRuntimeDurabilityContract({ + adapter: RUNTIME_STORE_KIND_POSTGRES, + durable: false, + durableRequested: true, + durableCapable: false, + ready: false, + status: "blocked", + blocker, + reason, + retryable: retryable === true ? true : undefined, + transient: transient === true ? true : undefined, + retryAfterMs: Number.isFinite(Number(retryAfterMs)) ? Math.max(0, Math.trunc(Number(retryAfterMs))) : undefined, + liveRuntimeEvidence: false, + fixtureEvidence: false, + connection: { + queryAttempted: Boolean(connection?.queryAttempted), + queryResult: connection?.queryResult ?? "not_ready", + endpointRedacted: true, + valueRedacted: true, + errorCode: connection?.errorCode ?? null + }, + schema: blockedSchema, + migration: blockedMigration, + gates: gates ?? runtimeGates({ + ...defaultRuntimeGates({ blocker, connection, schema: blockedSchema, migration: blockedMigration }), + schema: defaultSchemaGate({ blocker, schema: blockedSchema }), + migration: defaultMigrationGate({ blocker, migration: blockedMigration }), + durability: blockedGate({ checked: false, blocker }) + }), + safety: runtimeSafety(), + adapterContract: postgresAdapterContract() + }); + } +} diff --git a/internal/db/runtime-store.ts b/internal/db/runtime-store.ts index 2434241d..34e0c4a3 100644 --- a/internal/db/runtime-store.ts +++ b/internal/db/runtime-store.ts @@ -1,100 +1,37 @@ /* * SPEC: PJ2026-010403 API契约 draft-2026-06-17-r0; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p1-zero-split-durable-realtime; draft-2026-06-24-p0-aggregate-event-stream; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract. - * 职责: Cloud runtime durable store。Code Agent trace/session/projection cursor 投影事实必须从同一持久化适配器写入和恢复。 + * 职责: Cloud runtime store public factories and compatibility exports. */ -import { createHash, randomUUID } from "node:crypto"; import { - ENVIRONMENT_DEV, - ERROR_CODES, - HwlabProtocolError, - assertProtocolRecord -} from "../protocol/index.mjs"; -import { - CLOUD_API_SERVICE_ID, - createProtocolAuditEvent, - deriveProtocolActorFromMeta -} from "../audit/index.mjs"; -import { - CLOUD_CORE_MIGRATION_ID, - CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, - CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS -} from "./schema.ts"; + RUNTIME_ADAPTER_ENV, + RUNTIME_DURABLE_ENV, + RUNTIME_STORE_KIND_POSTGRES, + normalizeRuntimeAdapter +} from "./runtime-store-core.ts"; +import { CloudRuntimeStore } from "./runtime-store-memory.ts"; +import { PostgresCloudRuntimeStore } from "./runtime-store-postgres.ts"; -export const RUNTIME_STORE_KIND = "memory"; -export const RUNTIME_STORE_KIND_POSTGRES = "postgres"; -export const RUNTIME_DURABLE_ADAPTER_MISSING = "runtime_durable_adapter_missing"; -export const RUNTIME_DURABLE_ADAPTER_UNCONFIGURED = "runtime_durable_adapter_unconfigured"; -export const RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING = "runtime_durable_adapter_driver_missing"; -export const RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED = "runtime_durable_adapter_ssl_blocked"; -export const RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED = "runtime_durable_adapter_auth_blocked"; -export const RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED = "runtime_durable_adapter_schema_blocked"; -export const RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED = "runtime_durable_adapter_migration_blocked"; -export const RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED = "runtime_durable_adapter_query_blocked"; -export const RUNTIME_DURABILITY_REQUIRED_EVIDENCE = "runtime_adapter_schema_migration_read_query"; - -export const RUNTIME_ADAPTER_ENV = "HWLAB_CLOUD_RUNTIME_ADAPTER"; -export const RUNTIME_DURABLE_ENV = "HWLAB_CLOUD_RUNTIME_DURABLE"; -export const RUNTIME_DB_POOL_MAX_ENV = "HWLAB_CLOUD_DB_POOL_MAX"; - -const requiredPostgresSchema = CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS; - -const postgresCountTables = Object.freeze([ - "users", - "user_sessions", - "gateway_sessions", - "box_resources", - "box_capabilities", - "hardware_operations", - "audit_events", - "evidence_records", - "agent_sessions", - "account_workspaces", - "worker_sessions", - "agent_trace_events", - "workbench_projection_state", - "workbench_sessions", - "workbench_messages", - "workbench_parts", - "workbench_turns", - "workbench_trace_events", - "workbench_event_sequences", - "workbench_events", - "workbench_session_inputs", - "workbench_projection_checkpoints" -]); - -const postgresRuntimeReadIndexes = Object.freeze([ - "CREATE INDEX IF NOT EXISTS idx_agent_trace_events_trace_order ON agent_trace_events(trace_id, occurred_at, id)", - "CREATE INDEX IF NOT EXISTS idx_agent_trace_events_session_order ON agent_trace_events(agent_session_id, occurred_at, id)", - "CREATE INDEX IF NOT EXISTS idx_agent_trace_events_worker_order ON agent_trace_events(worker_session_id, occurred_at, id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_projection_state_status_retry_updated ON workbench_projection_state(projection_status, next_retry_at, updated_at)", - "CREATE INDEX IF NOT EXISTS idx_workbench_projection_state_run_command ON workbench_projection_state(run_id, command_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_projection_state_session_updated ON workbench_projection_state(session_id, updated_at DESC)", - "CREATE INDEX IF NOT EXISTS idx_workbench_sessions_updated ON workbench_sessions(updated_at DESC, session_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_sessions_owner_updated ON workbench_sessions(owner_user_id, updated_at DESC, session_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_sessions_status_updated ON workbench_sessions(status, updated_at DESC, session_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_messages_session_updated ON workbench_messages(session_id, updated_at ASC, message_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_messages_trace ON workbench_messages(trace_id, message_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_messages_turn ON workbench_messages(turn_id, message_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_parts_message_index ON workbench_parts(message_id, part_index, part_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_parts_session_trace ON workbench_parts(session_id, trace_id, part_index)", - "CREATE INDEX IF NOT EXISTS idx_workbench_turns_session_updated ON workbench_turns(session_id, updated_at DESC, turn_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_turns_trace ON workbench_turns(trace_id, turn_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_trace_events_trace_projected_seq ON workbench_trace_events(trace_id, projected_seq)", - "CREATE INDEX IF NOT EXISTS idx_workbench_trace_events_session_projected_seq ON workbench_trace_events(session_id, projected_seq)", - "CREATE INDEX IF NOT EXISTS idx_workbench_event_sequences_session ON workbench_event_sequences(session_id, updated_at DESC, aggregate_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_event_sequences_trace ON workbench_event_sequences(trace_id, updated_at DESC, aggregate_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_events_trace_seq ON workbench_events(trace_id, event_seq)", - "CREATE INDEX IF NOT EXISTS idx_workbench_events_session_seq ON workbench_events(session_id, event_seq)", - "CREATE INDEX IF NOT EXISTS idx_workbench_events_type_seq ON workbench_events(event_type, event_seq)", - "CREATE INDEX IF NOT EXISTS idx_workbench_session_inputs_session_seq ON workbench_session_inputs(session_id, admitted_seq, input_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_session_inputs_trace ON workbench_session_inputs(trace_id, input_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_session_inputs_command ON workbench_session_inputs(command_id, input_id) WHERE command_id IS NOT NULL", - "CREATE INDEX IF NOT EXISTS idx_workbench_projection_checkpoints_status_updated ON workbench_projection_checkpoints(projection_status, updated_at DESC, trace_id)", - "CREATE INDEX IF NOT EXISTS idx_workbench_projection_checkpoints_session_updated ON workbench_projection_checkpoints(session_id, updated_at DESC, trace_id)" -]); +export { + RUNTIME_ADAPTER_ENV, + RUNTIME_DB_POOL_MAX_ENV, + RUNTIME_DURABILITY_REQUIRED_EVIDENCE, + RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + RUNTIME_DURABLE_ADAPTER_MISSING, + RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, + RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, + RUNTIME_DURABLE_ENV, + RUNTIME_STORE_KIND, + RUNTIME_STORE_KIND_POSTGRES, + buildPostgresPoolConfig, + classifyRuntimeDbError +} from "./runtime-store-core.ts"; +export { CloudRuntimeStore } from "./runtime-store-memory.ts"; +export { PostgresCloudRuntimeStore } from "./runtime-store-postgres.ts"; export function createCloudRuntimeStore(options = {}) { return new CloudRuntimeStore(options); @@ -119,3497 +56,3 @@ export function createConfiguredCloudRuntimeStore(options = {}) { return createCloudRuntimeStore(options); } - -export function buildPostgresPoolConfig({ dbUrl, sslMode, timeoutMs, poolMax, queryTimeoutMs } = {}) { - const normalizedSslMode = normalizePostgresSslMode(sslMode ?? postgresSslModeFromConnectionString(dbUrl)); - const normalizedPoolMax = normalizeRuntimePoolMax(poolMax); - const normalizedQueryTimeoutMs = queryTimeoutMs === undefined || queryTimeoutMs === null || String(queryTimeoutMs).trim() === "" - ? null - : normalizeRuntimeTimeoutMs(queryTimeoutMs); - return { - connectionString: normalizePostgresConnectionStringForSslMode(dbUrl, normalizedSslMode), - ssl: postgresSslOption(normalizedSslMode), - connectionTimeoutMillis: normalizeRuntimeTimeoutMs(timeoutMs), - ...(normalizedQueryTimeoutMs !== null ? { query_timeout: normalizedQueryTimeoutMs, statement_timeout: normalizedQueryTimeoutMs } : {}), - ...(normalizedPoolMax !== null ? { max: normalizedPoolMax } : {}) - }; -} - -const RUNTIME_DB_QUERY_TIMEOUT_MS_ENV = "HWLAB_CLOUD_DB_QUERY_TIMEOUT_MS"; -const RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS"; -const RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS"; -const RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS"; -const RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV = "HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS"; -const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS = 5; -const DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS = 250; -const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS = 5_000; -const DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS = 10_000; - -function normalizePositiveInteger(value, fallback) { - const number = Number(value); - return Number.isFinite(number) && number > 0 ? Math.trunc(number) : fallback; -} - -function delayRuntimeDbQueryRetry(ms) { - return new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.trunc(Number(ms) || 0)))); -} - -export class CloudRuntimeStore { - constructor({ now = () => new Date().toISOString() } = {}) { - this.now = now; - this.gatewaySessions = new Map(); - this.boxResources = new Map(); - this.boxCapabilities = new Map(); - this.hardwareOperations = new Map(); - this.auditEvents = new Map(); - this.evidenceRecords = new Map(); - this.agentTraceEvents = new Map(); - this.workbenchProjectionStates = new Map(); - this.workbenchSessions = new Map(); - this.workbenchMessages = new Map(); - this.workbenchParts = new Map(); - this.workbenchTurns = new Map(); - this.workbenchTraceEvents = new Map(); - this.workbenchEventSequences = new Map(); - this.workbenchEvents = new Map(); - this.workbenchSessionInputs = new Map(); - this.workbenchProjectionCheckpoints = new Map(); - } - - summary() { - return addRuntimeDurabilityContract({ - adapter: RUNTIME_STORE_KIND, - durable: false, - status: "degraded", - blocker: RUNTIME_DURABLE_ADAPTER_MISSING, - reason: "L1 runtime writes are process-local only; no DB-backed durable runtime adapter is configured", - adapterContract: { - required: "DB-backed durable runtime adapter using HWLAB_CLOUD_DB_URL", - sourceIssue: "pikasTech/HWLAB#164", - secretMaterialRequiredInSource: false - }, - counts: { - gatewaySessions: this.gatewaySessions.size, - boxResources: this.boxResources.size, - boxCapabilities: this.boxCapabilities.size, - hardwareOperations: this.hardwareOperations.size, - auditEvents: this.auditEvents.size, - evidenceRecords: this.evidenceRecords.size, - agentTraceEvents: this.agentTraceEvents.size, - workbenchProjectionStates: this.workbenchProjectionStates.size, - workbenchSessions: this.workbenchSessions.size, - workbenchMessages: this.workbenchMessages.size, - workbenchParts: this.workbenchParts.size, - workbenchTurns: this.workbenchTurns.size, - workbenchTraceEvents: this.workbenchTraceEvents.size, - workbenchEventSequences: this.workbenchEventSequences.size, - workbenchEvents: this.workbenchEvents.size, - workbenchSessionInputs: this.workbenchSessionInputs.size, - workbenchProjectionCheckpoints: this.workbenchProjectionCheckpoints.size - } - }); - } - - registerGatewaySession(params = {}, requestMeta = {}) { - const now = this.now(); - const input = asObject(params.gatewaySession ?? params, "gatewaySession"); - const gatewaySession = pruneUndefined({ - gatewaySessionId: input.gatewaySessionId || makeId("gws"), - projectId: requireProtocolId(input, "projectId"), - serviceId: input.serviceId || "hwlab-gateway", - gatewayId: input.gatewayId || makeId("gtw"), - endpoint: input.endpoint, - status: input.status || "connected", - environment: ENVIRONMENT_DEV, - startedAt: input.startedAt || now, - lastSeenAt: input.lastSeenAt || now, - stoppedAt: input.stoppedAt, - labels: normalizeJsonObject(input.labels) - }); - - assertProtocolRecord("gatewaySession", gatewaySession); - this.gatewaySessions.set(gatewaySession.gatewaySessionId, gatewaySession); - - const auditEvent = this.recordAuditEvent({ - requestMeta, - params, - action: "gateway.session.register", - targetType: "gateway_session", - targetId: gatewaySession.gatewaySessionId, - projectId: gatewaySession.projectId, - gatewaySessionId: gatewaySession.gatewaySessionId, - outcome: "accepted", - metadata: { - gatewayId: gatewaySession.gatewayId, - serviceId: gatewaySession.serviceId - } - }); - - return { - registered: true, - gatewaySession, - auditEvent, - persistence: this.summary() - }; - } - - registerBoxResource(params = {}, requestMeta = {}) { - const now = this.now(); - const input = asObject(params.resource ?? params.boxResource ?? params, "boxResource"); - const gatewaySessionId = requireProtocolId(input, "gatewaySessionId"); - const gatewaySession = this.requireGatewaySession(gatewaySessionId); - const projectId = input.projectId || gatewaySession.projectId; - const resource = pruneUndefined({ - resourceId: input.resourceId || makeId("res"), - projectId, - gatewaySessionId, - boxId: input.boxId || makeId("box"), - resourceType: input.resourceType || "simulator_endpoint", - name: input.name, - state: input.state || "available", - environment: ENVIRONMENT_DEV, - metadata: normalizeJsonObject(input.metadata), - createdAt: input.createdAt || now, - updatedAt: input.updatedAt || now - }); - - assertProtocolRecord("boxResource", resource); - this.boxResources.set(resource.resourceId, resource); - - const auditEvent = this.recordAuditEvent({ - requestMeta, - params, - action: "box.resource.register", - targetType: "box_resource", - targetId: resource.resourceId, - projectId: resource.projectId, - gatewaySessionId: resource.gatewaySessionId, - outcome: "accepted", - metadata: { - boxId: resource.boxId, - resourceType: resource.resourceType - } - }); - - return { - registered: true, - resource, - auditEvent, - persistence: this.summary() - }; - } - - reportBoxCapabilities(params = {}, requestMeta = {}) { - const now = this.now(); - const inputs = normalizeCapabilityInputs(params); - const capabilities = []; - - for (const input of inputs) { - const resourceId = requireProtocolId(input, "resourceId"); - const resource = this.requireBoxResource(resourceId); - const capability = pruneUndefined({ - capabilityId: input.capabilityId || makeId("cap"), - resourceId, - projectId: input.projectId || resource.projectId, - name: requireString(input, "name"), - description: input.description, - direction: input.direction || "bidirectional", - valueType: input.valueType || "object", - unit: input.unit, - constraints: normalizeJsonObject(input.constraints), - mutatesState: input.mutatesState ?? true, - createdAt: input.createdAt || now, - updatedAt: input.updatedAt || now - }); - - assertProtocolRecord("boxCapability", capability); - this.boxCapabilities.set(capability.capabilityId, capability); - capabilities.push(capability); - } - - const first = capabilities[0]; - const auditEvent = this.recordAuditEvent({ - requestMeta, - params, - action: "box.capability.report", - targetType: "box_capability", - targetId: first.capabilityId, - projectId: first.projectId, - gatewaySessionId: this.boxResources.get(first.resourceId)?.gatewaySessionId, - outcome: "accepted", - metadata: { - capabilityIds: capabilities.map((capability) => capability.capabilityId) - } - }); - - return { - reported: true, - capabilities, - auditEvent, - persistence: this.summary() - }; - } - - requestHardwareOperation(params = {}, requestMeta = {}) { - const refs = this.requireOperationRefs(params, requestMeta); - const now = this.now(); - const operation = pruneUndefined({ - operationId: params.operationId || makeId("op"), - projectId: refs.projectId, - gatewaySessionId: refs.gatewaySession.gatewaySessionId, - agentSessionId: params.agentSessionId, - workerSessionId: params.workerSessionId, - resourceId: refs.resource.resourceId, - capabilityId: refs.capability.capabilityId, - requestedBy: refs.actor.actorId, - input: normalizeJsonObject(params.input), - status: "accepted", - environment: ENVIRONMENT_DEV, - requestedAt: params.requestedAt || now, - updatedAt: now - }); - - assertProtocolRecord("hardwareOperation", operation); - this.hardwareOperations.set(operation.operationId, operation); - - const auditEvent = this.recordAuditEvent({ - requestMeta, - params, - action: "hardware.operation.request", - targetType: "hardware_operation", - targetId: operation.operationId, - projectId: operation.projectId, - gatewaySessionId: operation.gatewaySessionId, - workerSessionId: operation.workerSessionId, - operationId: operation.operationId, - outcome: "accepted", - metadata: { - resourceId: operation.resourceId, - capabilityId: operation.capabilityId - } - }); - - return { - accepted: true, - status: operation.status, - operationId: operation.operationId, - operation, - auditEvent, - persistence: this.summary() - }; - } - - invokeHardwareShell(params = {}, requestMeta = {}) { - const shellInput = normalizeShellInput(params); - const operationResult = this.requestHardwareOperation( - { - ...params, - input: { - ...normalizeJsonObject(params.input), - shell: shellInput - } - }, - requestMeta - ); - const operation = { - ...operationResult.operation, - output: { - shellExecuted: false, - dispatchStatus: "not_connected", - message: "cloud-api recorded the shell invoke request; no gateway shell adapter is connected in L1" - }, - updatedAt: this.now() - }; - assertProtocolRecord("hardwareOperation", operation); - this.hardwareOperations.set(operation.operationId, operation); - - const evidenceRecord = this.writeEvidenceRecord( - { - projectId: operation.projectId, - operationId: operation.operationId, - kind: "trace", - mimeType: "application/json", - serviceId: CLOUD_API_SERVICE_ID, - metadata: { - traceId: requestMeta.traceId, - gatewaySessionId: operation.gatewaySessionId, - resourceId: operation.resourceId, - capabilityId: operation.capabilityId, - shellExecuted: false, - dispatchStatus: "not_connected" - }, - content: { - operationId: operation.operationId, - shell: shellInput, - dispatchStatus: "not_connected" - } - }, - requestMeta - ).evidenceRecord; - - const auditEvent = this.recordAuditEvent({ - requestMeta, - params, - action: "hardware.invoke.shell", - targetType: "hardware_operation", - targetId: operation.operationId, - projectId: operation.projectId, - gatewaySessionId: operation.gatewaySessionId, - workerSessionId: operation.workerSessionId, - operationId: operation.operationId, - outcome: "accepted", - metadata: { - evidenceId: evidenceRecord.evidenceId, - shellExecuted: false, - dispatchStatus: "not_connected" - } - }); - - return { - accepted: true, - status: operation.status, - operationId: operation.operationId, - operation, - auditEvent, - evidenceRecord, - dispatch: operation.output, - persistence: this.summary() - }; - } - - writeAuditEvent(params = {}, requestMeta = {}) { - const eventInput = params.event ?? params.auditEvent ?? params; - const auditEvent = eventInput.auditId - ? pruneUndefined({ - ...eventInput, - environment: ENVIRONMENT_DEV - }) - : this.buildAuditEventFromInput(eventInput, requestMeta); - - assertProtocolRecord("auditEvent", auditEvent); - this.auditEvents.set(auditEvent.auditId, auditEvent); - - return { - written: true, - auditEvent, - persistence: this.summary() - }; - } - - queryAuditEvents(params = {}) { - const events = [...this.auditEvents.values()].filter((event) => matchesQuery(event, params, [ - "auditId", - "traceId", - "projectId", - "gatewaySessionId", - "operationId", - "action", - "targetId" - ])); - - return { - events: limitResults(events, params.limit), - count: events.length, - persistence: this.summary() - }; - } - - writeEvidenceRecord(params = {}, requestMeta = {}) { - const now = this.now(); - const input = asObject(params.record ?? params.evidenceRecord ?? params, "evidenceRecord"); - const content = input.content ?? input.metadata ?? {}; - const sha256 = input.sha256 || sha256Hex(content); - const evidenceId = input.evidenceId || makeId("evd"); - const evidenceRecord = pruneUndefined({ - evidenceId, - projectId: requireProtocolId(input, "projectId"), - operationId: requireProtocolId(input, "operationId"), - agentSessionId: input.agentSessionId, - workerSessionId: input.workerSessionId, - kind: input.kind || "trace", - uri: input.uri || `memory://hwlab/evidence/${evidenceId}`, - mimeType: input.mimeType, - sha256, - sizeBytes: input.sizeBytes ?? Buffer.byteLength(stableJson(content), "utf8"), - serviceId: input.serviceId || CLOUD_API_SERVICE_ID, - environment: ENVIRONMENT_DEV, - metadata: { - ...normalizeJsonObject(input.metadata), - ...(requestMeta.traceId ? { traceId: requestMeta.traceId } : {}) - }, - createdAt: input.createdAt || now - }); - - assertProtocolRecord("evidenceRecord", evidenceRecord); - this.evidenceRecords.set(evidenceRecord.evidenceId, evidenceRecord); - - return { - written: true, - evidenceRecord, - persistence: this.summary() - }; - } - - queryEvidenceRecords(params = {}) { - const records = [...this.evidenceRecords.values()].filter((record) => matchesQuery(record, params, [ - "evidenceId", - "projectId", - "operationId", - "agentSessionId", - "workerSessionId", - "kind", - "serviceId" - ])); - - return { - records: limitResults(records, params.limit), - count: records.length, - persistence: this.summary() - }; - } - - writeAgentTraceEvent(params = {}, requestMeta = {}) { - const traceEvent = normalizeAgentTraceEvent(params.traceEvent ?? params.event ?? params, requestMeta, this.now()); - this.agentTraceEvents.set(traceEvent.id, traceEvent); - return { - written: true, - traceEvent, - persistence: this.summary() - }; - } - - queryAgentTraceEvents(params = {}) { - const records = [...this.agentTraceEvents.values()].filter((record) => matchesQuery(record, params, [ - "id", - "traceId", - "agentSessionId", - "workerSessionId", - "level" - ])); - records.sort(compareTraceEventRecords); - return { - events: limitResults(records.map((record) => record.event), params.limit), - count: records.length, - persistence: this.summary() - }; - } - - writeWorkbenchProjectionState(params = {}, requestMeta = {}) { - const state = normalizeWorkbenchProjectionState(params.projectionState ?? params.state ?? params, requestMeta, this.now()); - this.workbenchProjectionStates.set(state.traceId, state); - return { - written: true, - projectionState: state, - persistence: this.summary() - }; - } - - getWorkbenchProjectionState(params = {}) { - const traceId = textOr(params.traceId, ""); - const state = traceId ? this.workbenchProjectionStates.get(traceId) ?? null : null; - return { - projectionState: state, - found: Boolean(state), - persistence: this.summary() - }; - } - - allocateWorkbenchProjectedSeq(params = {}, requestMeta = {}) { - const allocation = normalizeWorkbenchProjectionAllocation(params, requestMeta, this.now()); - const existing = [...this.workbenchTraceEvents.values()].find((record) => - record.traceId === allocation.traceId && record.sourceEventId === allocation.sourceEventId - ); - if (existing) { - return { - ...allocation, - projectedSeq: nonNegativeInteger(existing.projectedSeq), - reused: true, - persistence: this.summary(), - valuesPrinted: false - }; - } - const eventMax = [...this.workbenchTraceEvents.values()] - .filter((record) => record.traceId === allocation.traceId) - .reduce((max, record) => Math.max(max, nonNegativeInteger(record.projectedSeq)), 0); - const checkpointMax = nonNegativeInteger(this.workbenchProjectionCheckpoints.get(allocation.traceId)?.projectedSeq); - return { - ...allocation, - projectedSeq: Math.max(eventMax, checkpointMax) + 1, - reused: false, - persistence: this.summary(), - valuesPrinted: false - }; - } - - queryWorkbenchProjectionStates(params = {}) { - const dueAt = textOr(params.dueAt, ""); - const statuses = Array.isArray(params.projectionStatuses) ? new Set(params.projectionStatuses.map((item) => textOr(item, "")).filter(Boolean)) : null; - const records = [...this.workbenchProjectionStates.values()] - .filter((record) => matchesQuery(record, params, ["traceId", "sessionId", "runId", "commandId", "sourceRunId", "sourceCommandId"])) - .filter((record) => !statuses || statuses.has(record.projectionStatus)) - .filter((record) => !dueAt || !record.nextRetryAt || String(record.nextRetryAt) <= dueAt) - .sort(compareProjectionStateRecords); - return { - states: limitResults(records, params.limit), - count: records.length, - persistence: this.summary() - }; - } - - writeWorkbenchFacts(params = {}, requestMeta = {}) { - const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); - const inputFacts = facts.inputs.map((fact) => memoryWorkbenchSessionInputWithSeq(fact, this.workbenchSessionInputs)); - for (const fact of inputFacts) this.workbenchSessionInputs.set(fact.inputId, fact); - for (const fact of facts.sessions) this.workbenchSessions.set(fact.sessionId, fact); - for (const fact of facts.messages) this.workbenchMessages.set(fact.messageId, fact); - for (const fact of facts.parts) this.workbenchParts.set(fact.partId, fact); - for (const fact of facts.turns) this.workbenchTurns.set(fact.turnId, fact); - for (const fact of facts.traceEvents) this.workbenchTraceEvents.set(fact.id, fact); - for (const fact of facts.checkpoints) { - const previous = this.workbenchProjectionCheckpoints.get(fact.traceId) ?? null; - if (!previous || nonNegativeInteger(fact.projectedSeq) >= nonNegativeInteger(previous.projectedSeq)) { - this.workbenchProjectionCheckpoints.set(fact.traceId, fact); - } - } - return { - written: true, - facts: { ...facts, inputs: inputFacts }, - persistence: this.summary() - }; - } - - queryWorkbenchFacts(params = {}) { - const families = workbenchFactFamilySet(params); - const sessions = [...this.workbenchSessions.values()].filter((record) => matchesWorkbenchSessionFactQuery(record, params)); - const inputs = [...this.workbenchSessionInputs.values()].filter((record) => matchesQuery(record, params, ["inputId", "sessionId", "turnId", "traceId", "messageId", "commandId", "delivery", "status"])); - const messages = [...this.workbenchMessages.values()].filter((record) => matchesQuery(record, params, ["messageId", "sessionId", "turnId", "traceId", "role", "status"])); - const parts = [...this.workbenchParts.values()].filter((record) => matchesQuery(record, params, ["partId", "messageId", "sessionId", "turnId", "traceId", "partType", "status"])); - const turns = [...this.workbenchTurns.values()].filter((record) => matchesQuery(record, params, ["turnId", "sessionId", "traceId", "messageId", "status"])); - const afterProjectedSeq = nonNegativeInteger(params.afterProjectedSeq); - const traceEvents = [...this.workbenchTraceEvents.values()].filter((record) => matchesQuery(record, params, ["id", "traceId", "sessionId", "turnId", "messageId", "eventType"]) && (afterProjectedSeq <= 0 || nonNegativeInteger(record.projectedSeq) > afterProjectedSeq)); - const checkpoints = [...this.workbenchProjectionCheckpoints.values()].filter((record) => matchesQuery(record, params, ["traceId", "sessionId", "turnId", "runId", "commandId", "projectionStatus", "projectionHealth"])); - const facts = { - inputs: families.has("inputs") ? limitResults(sortWorkbenchFactRows(inputs, params, "inputs", compareWorkbenchFactRecords), params.limit) : [], - sessions: families.has("sessions") ? limitResults(sortWorkbenchFactRows(sessions, params, "sessions", compareWorkbenchFactRecords), params.limit) : [], - messages: families.has("messages") ? limitResults(sortWorkbenchFactRows(messages, params, "messages", compareWorkbenchFactRecords), params.limit) : [], - parts: families.has("parts") ? limitResults(sortWorkbenchFactRows(parts, params, "parts", compareWorkbenchPartFactRecords), params.limit) : [], - turns: families.has("turns") ? limitResults(sortWorkbenchFactRows(turns, params, "turns", compareWorkbenchFactRecords), params.limit) : [], - traceEvents: families.has("traceEvents") ? limitResults(sortWorkbenchFactRows(traceEvents, params, "traceEvents", compareWorkbenchTraceEventFacts), params.limit) : [], - checkpoints: families.has("checkpoints") ? limitResults(sortWorkbenchFactRows(checkpoints, params, "checkpoints", compareWorkbenchFactRecords), params.limit) : [] - }; - return { - facts, - count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0), - persistence: this.summary() - }; - } - - recordAuditEvent(input) { - return this.writeAuditEvent(this.buildAuditEventFromInput(input, input.requestMeta), input.requestMeta).auditEvent; - } - - buildAuditEventFromInput(input = {}, requestMeta = {}) { - const actor = deriveProtocolActorFromMeta(requestMeta, input.params?.audit ?? input); - return createProtocolAuditEvent({ - auditId: input.auditId, - traceId: requestMeta.traceId || input.traceId, - actorType: actor.actorType, - actorId: actor.actorId, - action: input.action, - targetType: input.targetType, - targetId: input.targetId, - projectId: input.projectId, - gatewaySessionId: input.gatewaySessionId, - workerSessionId: input.workerSessionId, - operationId: input.operationId, - serviceId: CLOUD_API_SERVICE_ID, - outcome: input.outcome, - reason: input.reason || input.params?.audit?.reason, - metadata: input.metadata, - occurredAt: input.occurredAt - }); - } - - requireOperationRefs(params, requestMeta = {}) { - const projectId = requireProtocolId(params, "projectId"); - const gatewaySession = this.requireGatewaySession(requireProtocolId(params, "gatewaySessionId")); - const resource = this.requireBoxResource(requireProtocolId(params, "resourceId")); - const capability = this.requireBoxCapability(requireProtocolId(params, "capabilityId")); - - if (gatewaySession.projectId !== projectId || resource.projectId !== projectId || capability.projectId !== projectId) { - throw new HwlabProtocolError("hardware operation references must belong to one project", { - code: ERROR_CODES.invalidParams, - data: { - projectId, - gatewaySessionId: gatewaySession.gatewaySessionId, - resourceId: resource.resourceId, - capabilityId: capability.capabilityId - } - }); - } - if (resource.gatewaySessionId !== gatewaySession.gatewaySessionId) { - throw new HwlabProtocolError("resource is not registered under the requested gateway session", { - code: ERROR_CODES.resourceLocked, - data: { - gatewaySessionId: gatewaySession.gatewaySessionId, - resourceGatewaySessionId: resource.gatewaySessionId, - resourceId: resource.resourceId - } - }); - } - if (capability.resourceId !== resource.resourceId) { - throw new HwlabProtocolError("capability is not registered on the requested resource", { - code: ERROR_CODES.capabilityUnavailable, - data: { - resourceId: resource.resourceId, - capabilityId: capability.capabilityId - } - }); - } - - return { - projectId, - gatewaySession, - resource, - capability, - actor: deriveProtocolActorFromMeta(requestMeta, params.audit ?? {}) - }; - } - - requireGatewaySession(gatewaySessionId) { - const gatewaySession = this.gatewaySessions.get(gatewaySessionId); - if (!gatewaySession) { - throw new HwlabProtocolError(`gateway session ${gatewaySessionId} is not registered`, { - code: ERROR_CODES.sessionNotFound, - data: { gatewaySessionId } - }); - } - return gatewaySession; - } - - requireBoxResource(resourceId) { - const resource = this.boxResources.get(resourceId); - if (!resource) { - throw new HwlabProtocolError(`box resource ${resourceId} is not registered`, { - code: ERROR_CODES.capabilityUnavailable, - data: { resourceId } - }); - } - return resource; - } - - requireBoxCapability(capabilityId) { - const capability = this.boxCapabilities.get(capabilityId); - if (!capability) { - throw new HwlabProtocolError(`capability ${capabilityId} is not reported`, { - code: ERROR_CODES.capabilityUnavailable, - data: { capabilityId } - }); - } - return capability; - } -} - -export class PostgresCloudRuntimeStore { - constructor({ - env = process.env, - dbUrl, - sslMode = "require", - now = () => new Date().toISOString(), - queryClient, - pgModuleLoader, - logger = console - } = {}) { - this.env = env; - this.dbUrl = dbUrl; - this.sslMode = sslMode; - this.now = now; - this.queryClient = queryClient; - this.pgModuleLoader = pgModuleLoader; - this.logger = logger; - this.pool = null; - this.postgresPoolResetInFlight = null; - this.postgresPoolResetLastAtMs = 0; - this.runtimeReadIndexesReady = false; - this.runtimeReadIndexesReadyPromise = null; - this.memory = new CloudRuntimeStore({ now }); - this.lastReadiness = this.blockedSummary({ - blocker: dbUrl || queryClient ? RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED : RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, - reason: dbUrl || queryClient - ? "Postgres runtime adapter is configured but has not completed a live schema query" - : "Postgres runtime adapter requires HWLAB_CLOUD_DB_URL or an injected query client" - }); - } - - summary() { - return { - ...this.lastReadiness, - counts: this.lastReadiness.counts ?? this.memory.summary().counts - }; - } - - async readiness() { - if (!this.dbUrl && !this.queryClient) { - this.lastReadiness = this.blockedSummary({ - blocker: RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, - reason: "Postgres runtime adapter is selected but HWLAB_CLOUD_DB_URL is not injected" - }); - return this.summary(); - } - - let rows; - try { - const result = await this.query( - "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ANY($1::text[])", - [Object.keys(requiredPostgresSchema)] - ); - rows = Array.isArray(result?.rows) ? result.rows : []; - } catch (error) { - this.lastReadiness = this.blockedSummary(classifyRuntimeDbError(error)); - return this.summary(); - } - - const schema = summarizeRuntimeSchema(rows); - if (!schema.ready) { - this.lastReadiness = this.blockedSummary({ - blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, - reason: "Postgres runtime adapter connected, but required runtime tables or columns are missing", - schema, - connection: { - queryAttempted: true, - queryResult: "schema_blocked" - }, - gates: runtimeGates({ - ssl: readyGate(), - auth: readyGate(), - schema: blockedGate({ checked: true, blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }), - migration: notCheckedGate(), - durability: blockedGate({ checked: false, blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }) - }) - }); - return this.summary(); - } - - let migration; - try { - migration = await this.readMigrationReadiness(); - } catch (error) { - const classified = classifyRuntimeDbError(error); - const migrationReadBlocker = [ - RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, - RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED - ].includes(classified.blocker) - ? classified.blocker - : RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED; - this.lastReadiness = this.blockedSummary({ - blocker: migrationReadBlocker, - reason: migrationReadBlocker === classified.blocker - ? classified.reason - : "Postgres runtime schema is present, but the durable runtime migration ledger could not be proven", - schema, - migration: blockedRuntimeMigration({ errorCode: classified.connection?.errorCode }), - connection: { - queryAttempted: true, - queryResult: migrationReadBlocker === classified.blocker - ? classified.connection?.queryResult - : "migration_blocked", - errorCode: classified.connection?.errorCode ?? null - }, - gates: runtimeGates({ - ssl: classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED - ? blockedGate({ checked: true, blocker: classified.blocker }) - : readyGate(), - auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED - ? blockedGate({ checked: true, blocker: classified.blocker }) - : classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED - ? notCheckedGate() - : readyGate(), - schema: readyGate(), - migration: blockedGate({ - checked: true, - blocker: migrationReadBlocker - }), - durability: blockedGate({ checked: false, blocker: migrationReadBlocker }) - }) - }); - return this.summary(); - } - - if (!migration.ready) { - this.lastReadiness = this.blockedSummary({ - blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, - reason: "Postgres runtime schema is present, but the required source migration is not recorded", - schema, - migration, - connection: { - queryAttempted: true, - queryResult: "migration_blocked" - }, - gates: runtimeGates({ - ssl: readyGate(), - auth: readyGate(), - schema: readyGate(), - migration: blockedGate({ checked: true, blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }), - durability: blockedGate({ checked: false, blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }) - }) - }); - return this.summary(); - } - - try { - await this.ensureRuntimeReadIndexes(); - } catch (error) { - const classified = classifyRuntimeDbError(error); - this.lastReadiness = this.blockedSummary({ - ...classified, - schema, - migration, - reason: "Postgres runtime schema is present, but Workbench trace read indexes could not be ensured", - connection: { - queryAttempted: true, - queryResult: classified.connection?.queryResult ?? "index_blocked", - errorCode: classified.connection?.errorCode ?? null - }, - gates: runtimeGates({ - ssl: classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED - ? blockedGate({ checked: true, blocker: classified.blocker }) - : readyGate(), - auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED - ? blockedGate({ checked: true, blocker: classified.blocker }) - : classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED - ? notCheckedGate() - : readyGate(), - schema: readyGate(), - migration: readyGate(), - durability: blockedGate({ checked: true, blocker: classified.blocker }) - }) - }); - return this.summary(); - } - - let counts; - try { - counts = await this.readCounts(); - } catch (error) { - const classified = classifyRuntimeDbError(error); - this.lastReadiness = this.blockedSummary({ - ...classified, - schema, - migration, - connection: { - queryAttempted: true, - queryResult: classified.connection?.queryResult ?? "query_blocked", - errorCode: classified.connection?.errorCode ?? null - }, - gates: runtimeGates({ - ssl: classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED - ? blockedGate({ checked: true, blocker: classified.blocker }) - : readyGate(), - auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED - ? blockedGate({ checked: true, blocker: classified.blocker }) - : classified.blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED - ? notCheckedGate() - : readyGate(), - schema: readyGate(), - migration: readyGate(), - durability: blockedGate({ checked: true, blocker: classified.blocker }) - }) - }); - return this.summary(); - } - - this.lastReadiness = addRuntimeDurabilityContract({ - adapter: RUNTIME_STORE_KIND_POSTGRES, - durable: true, - durableRequested: true, - durableCapable: true, - ready: true, - status: "ready", - blocker: null, - reason: "Postgres durable runtime adapter completed live schema, migration, and read readiness queries", - liveRuntimeEvidence: true, - fixtureEvidence: false, - connection: { - queryAttempted: true, - queryResult: "durable_readiness_ready", - endpointRedacted: true, - valueRedacted: true - }, - schema, - migration, - gates: runtimeGates({ - ssl: readyGate(), - auth: readyGate(), - schema: readyGate(), - migration: readyGate(), - durability: readyGate() - }), - safety: runtimeSafety(), - adapterContract: postgresAdapterContract(), - counts - }); - return this.summary(); - } - - async registerGatewaySession(params = {}, requestMeta = {}) { - await this.assertReadyForWrites(); - const before = snapshotMemory(this.memory); - const result = this.memory.registerGatewaySession(params, requestMeta); - await this.persistChanges(before); - return withPersistence(result, this.summary()); - } - - async registerBoxResource(params = {}, requestMeta = {}) { - await this.assertReadyForWrites(); - const input = asObject(params.resource ?? params.boxResource ?? params, "boxResource"); - await this.hydrateGatewaySession(input.gatewaySessionId); - const before = snapshotMemory(this.memory); - const result = this.memory.registerBoxResource(params, requestMeta); - await this.persistChanges(before); - return withPersistence(result, this.summary()); - } - - async reportBoxCapabilities(params = {}, requestMeta = {}) { - await this.assertReadyForWrites(); - const inputs = normalizeCapabilityInputs(params); - for (const input of inputs) { - await this.hydrateBoxResource(input.resourceId); - } - const before = snapshotMemory(this.memory); - const result = this.memory.reportBoxCapabilities(params, requestMeta); - await this.persistChanges(before); - return withPersistence(result, this.summary()); - } - - async requestHardwareOperation(params = {}, requestMeta = {}) { - await this.assertReadyForWrites(); - await this.hydrateOperationRefs(params); - const before = snapshotMemory(this.memory); - const result = this.memory.requestHardwareOperation(params, requestMeta); - await this.persistChanges(before); - return withPersistence(result, this.summary()); - } - - async invokeHardwareShell(params = {}, requestMeta = {}) { - await this.assertReadyForWrites(); - await this.hydrateOperationRefs(params); - const before = snapshotMemory(this.memory); - const result = this.memory.invokeHardwareShell(params, requestMeta); - await this.persistChanges(before); - return withPersistence(result, this.summary()); - } - - async writeAuditEvent(params = {}, requestMeta = {}) { - await this.assertReadyForWrites(); - const before = snapshotMemory(this.memory); - const result = this.memory.writeAuditEvent(params, requestMeta); - await this.persistChanges(before); - return withPersistence(result, this.summary()); - } - - async queryAuditEvents(params = {}) { - await this.assertReadyForDurableReads("audit.event.query"); - const result = await this.queryDurableReadRows( - "audit.event.query", - "SELECT event_json FROM audit_events ORDER BY timestamp ASC", - [] - ); - const events = result.rows - .map((row) => parseJsonColumn(row.event_json, null)) - .filter(Boolean) - .filter((event) => matchesQuery(event, params, [ - "auditId", - "traceId", - "projectId", - "gatewaySessionId", - "operationId", - "action", - "targetId" - ])); - - return { - events: limitResults(events, params.limit), - count: events.length, - persistence: this.summary() - }; - } - - async writeEvidenceRecord(params = {}, requestMeta = {}) { - await this.assertReadyForWrites(); - const before = snapshotMemory(this.memory); - const result = this.memory.writeEvidenceRecord(params, requestMeta); - await this.persistChanges(before); - return withPersistence(result, this.summary()); - } - - async queryEvidenceRecords(params = {}) { - await this.assertReadyForDurableReads("evidence.record.query"); - const result = await this.queryDurableReadRows( - "evidence.record.query", - "SELECT metadata_json FROM evidence_records ORDER BY created_at ASC", - [] - ); - const records = result.rows - .map((row) => parseJsonColumn(row.metadata_json, null)) - .filter(Boolean) - .filter((record) => matchesQuery(record, params, [ - "evidenceId", - "projectId", - "operationId", - "agentSessionId", - "workerSessionId", - "kind", - "serviceId" - ])); - - return { - records: limitResults(records, params.limit), - count: records.length, - persistence: this.summary() - }; - } - - async writeAgentTraceEvent(params = {}, requestMeta = {}) { - const readiness = this.summary(); - if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { - await this.assertReadyForWrites(); - } - const result = this.memory.writeAgentTraceEvent(params, requestMeta); - await this.persistAgentTraceEvent(result.traceEvent); - return withPersistence(result, this.summary()); - } - - async queryAgentTraceEvents(params = {}) { - await this.assertReadyForDurableReads("agent.trace-events.query"); - const traceId = textOr(params.traceId, ""); - const sessionId = textOr(params.sessionId, ""); - const workerSessionId = textOr(params.workerSessionId, ""); - const level = textOr(params.level, ""); - const clauses = []; - const queryParams = []; - if (traceId) { - queryParams.push(traceId); - clauses.push(`trace_id = $${queryParams.length}`); - } - if (sessionId) { - queryParams.push(sessionId); - clauses.push(`agent_session_id = $${queryParams.length}`); - } - if (workerSessionId) { - queryParams.push(workerSessionId); - clauses.push(`worker_session_id = $${queryParams.length}`); - } - if (level) { - queryParams.push(level); - clauses.push(`level = $${queryParams.length}`); - } - const sql = `SELECT event_json FROM agent_trace_events${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY occurred_at ASC, id ASC`; - const result = await this.queryDurableReadRows("agent.trace-events.query", sql, queryParams); - const events = result.rows - .map((row) => parseJsonColumn(row.event_json, null)) - .filter(Boolean) - .filter((event) => matchesQuery(event, params, ["traceId", "sessionId", "workerSessionId", "level"])); - return { - events: limitResults(events, params.limit), - count: events.length, - persistence: this.summary() - }; - } - - async writeWorkbenchProjectionState(params = {}, requestMeta = {}) { - const readiness = this.summary(); - if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { - await this.assertReadyForWrites(); - } - const result = this.memory.writeWorkbenchProjectionState(params, requestMeta); - await this.persistWorkbenchProjectionState(result.projectionState); - return withPersistence(result, this.summary()); - } - - async getWorkbenchProjectionState(params = {}) { - const traceId = textOr(params.traceId, ""); - if (!traceId) return withPersistence({ projectionState: null, found: false }, this.summary()); - const result = await this.queryWorkbenchProjectionStates({ traceId, limit: 1 }); - const projectionState = result.states[0] ?? null; - return withPersistence({ projectionState, found: Boolean(projectionState) }, result.persistence); - } - - async queryWorkbenchProjectionStates(params = {}) { - await this.assertReadyForDurableReads("workbench.projection-state.query"); - const clauses = []; - const queryParams = []; - const addTextClause = (field, column) => { - const value = textOr(params[field], ""); - if (!value) return; - queryParams.push(value); - clauses.push(`${column} = $${queryParams.length}`); - }; - addTextClause("traceId", "trace_id"); - addTextClause("sessionId", "session_id"); - addTextClause("runId", "run_id"); - addTextClause("commandId", "command_id"); - const statuses = Array.isArray(params.projectionStatuses) ? params.projectionStatuses.map((item) => textOr(item, "")).filter(Boolean) : []; - if (statuses.length > 0) { - queryParams.push(statuses); - clauses.push(`projection_status = ANY($${queryParams.length})`); - } - const dueAt = textOr(params.dueAt, ""); - if (dueAt) { - queryParams.push(dueAt); - clauses.push(`(next_retry_at IS NULL OR next_retry_at <= $${queryParams.length})`); - } - const limit = positiveIntegerOrNull(params.limit); - const offset = positiveIntegerOrNull(params.offset); - let sql = `SELECT projection_json FROM workbench_projection_state${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY updated_at ASC, trace_id ASC`; - if (limit) { - queryParams.push(limit); - sql += ` LIMIT $${queryParams.length}`; - } - if (offset) { - queryParams.push(offset); - sql += ` OFFSET $${queryParams.length}`; - } - const result = await this.queryDurableReadRows("workbench.projection-state.query", sql, queryParams); - const states = result.rows.map((row) => parseJsonColumn(row.projection_json, null)).filter(Boolean); - return { - states, - count: states.length, - persistence: this.summary() - }; - } - - async allocateWorkbenchProjectedSeq(params = {}, requestMeta = {}) { - const readiness = this.summary(); - if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { - await this.assertReadyForWrites(); - } - const allocation = normalizeWorkbenchProjectionAllocation(params, requestMeta, this.now()); - const result = await this.withDurableTransaction("workbench.projected-seq.allocate", async (client) => { - await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${allocation.traceId}`]); - const existing = await client.query( - "SELECT projected_seq FROM workbench_trace_events WHERE trace_id = $1 AND source_event_id = $2 ORDER BY projected_seq ASC LIMIT 1", - [allocation.traceId, allocation.sourceEventId] - ); - const existingSeq = nonNegativeInteger(existing.rows?.[0]?.projected_seq); - if (existingSeq > 0) return { projectedSeq: existingSeq, reused: true }; - const maxResult = await client.query( - "SELECT GREATEST(COALESCE((SELECT MAX(projected_seq) FROM workbench_trace_events WHERE trace_id = $1), 0), COALESCE((SELECT projected_seq FROM workbench_projection_checkpoints WHERE trace_id = $1), 0))::int AS max_projected_seq", - [allocation.traceId] - ); - const proposedSeq = nonNegativeInteger(maxResult.rows?.[0]?.max_projected_seq) + 1; - const reserve = await client.query( - "INSERT INTO workbench_projection_checkpoints (trace_id, session_id, turn_id, run_id, command_id, projected_seq, source_seq, source_event_id, projection_status, projection_health, terminal, sealed, diagnostic_json, checkpoint_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (trace_id) DO UPDATE SET projected_seq = GREATEST(workbench_projection_checkpoints.projected_seq + 1, EXCLUDED.projected_seq), updated_at = EXCLUDED.updated_at RETURNING projected_seq", - [allocation.traceId, allocation.sessionId, allocation.turnId, allocation.runId, allocation.commandId, proposedSeq, allocation.sourceSeq, allocation.sourceEventId, "projecting", "healthy", false, false, stableJson({ allocator: "workbench-projected-seq", valuesRedacted: true }), stableJson({ ...allocation, projectedSeq: proposedSeq, projectionStatus: "projecting", projectionHealth: "healthy", valuesPrinted: false }), allocation.createdAt, allocation.updatedAt] - ); - return { projectedSeq: nonNegativeInteger(reserve.rows?.[0]?.projected_seq) || proposedSeq, reused: false }; - }); - return { - ...allocation, - projectedSeq: result.projectedSeq, - reused: result.reused, - persistence: this.summary(), - valuesPrinted: false - }; - } - - async writeWorkbenchFacts(params = {}, requestMeta = {}) { - const readiness = this.summary(); - if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { - await this.assertReadyForWrites(); - } - const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); - const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, this.now()); - const txResult = await this.withDurableTransaction("workbench.facts.write", async (client) => { - const persistedEvents = []; - for (const event of aggregateEvents) persistedEvents.push(await this.persistWorkbenchAggregateEvent(event, client)); - const eventIndex = indexWorkbenchAggregateEvents(persistedEvents); - const inputFacts = facts.inputs.map((fact) => inputFactWithAggregateSeq(fact, eventIndex)); - for (const fact of inputFacts) await this.persistWorkbenchSessionInputFact(fact, client); - for (const fact of facts.sessions) await this.persistWorkbenchSessionFact(fact, client); - for (const fact of facts.messages) await this.persistWorkbenchMessageFact(fact, client); - for (const fact of facts.parts) await this.persistWorkbenchPartFact(fact, client); - for (const fact of facts.turns) await this.persistWorkbenchTurnFact(fact, client); - for (const fact of facts.traceEvents) await this.persistWorkbenchTraceEventFact(fact, client); - for (const fact of facts.checkpoints) await this.persistWorkbenchProjectionCheckpoint(fact, client); - for (const fact of facts.messages) { - const event = eventIndex.get(`message:${fact.messageId}`) ?? eventIndex.get(`messages:${fact.messageId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; - await this.persistWorkbenchProjectionOutbox({ - eventSeq: event?.eventSeq ?? null, - aggregateId: event?.aggregateId ?? null, - aggregateSeq: event?.aggregateSeq ?? 0, - projectionRevision: event?.projectionRevision ?? fact.projectedSeq, - traceId: fact.traceId, - sessionId: fact.sessionId, - turnId: fact.turnId, - messageId: fact.messageId, - projectedSeq: fact.projectedSeq, - sourceSeq: fact.sourceSeq, - sourceEventId: fact.sourceEventId, - commitType: "message", - terminal: Boolean(fact.terminal), - sealed: Boolean(fact.sealed), - payload: { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, - createdAt: fact.updatedAt ?? fact.createdAt ?? this.now() - }, client); - } - for (const fact of facts.traceEvents) { - const event = eventIndex.get(`traceEvent:${fact.id}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; - await this.persistWorkbenchProjectionOutbox({ - eventSeq: event?.eventSeq ?? null, - aggregateId: event?.aggregateId ?? null, - aggregateSeq: event?.aggregateSeq ?? 0, - projectionRevision: event?.projectionRevision ?? fact.projectedSeq, - traceId: fact.traceId, - sessionId: fact.sessionId, - turnId: fact.turnId, - messageId: fact.messageId, - projectedSeq: fact.projectedSeq, - sourceSeq: fact.sourceSeq, - sourceEventId: fact.sourceEventId, - commitType: fact.terminal ? "terminal" : "event", - terminal: Boolean(fact.terminal), - sealed: Boolean(fact.sealed), - payload: { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, - createdAt: fact.occurredAt ?? fact.updatedAt ?? this.now() - }, client); - } - for (const fact of facts.turns) { - if (fact.terminal || fact.sealed) { - const event = eventIndex.get(`turn:${fact.turnId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; - await this.persistWorkbenchProjectionOutbox({ - eventSeq: event?.eventSeq ?? null, - aggregateId: event?.aggregateId ?? null, - aggregateSeq: event?.aggregateSeq ?? 0, - projectionRevision: event?.projectionRevision ?? fact.projectedSeq, - traceId: fact.traceId, - sessionId: fact.sessionId, - turnId: fact.turnId, - messageId: fact.messageId, - projectedSeq: fact.projectedSeq, - sourceSeq: fact.sourceSeq, - sourceEventId: fact.sourceEventId, - commitType: "terminal", - terminal: true, - sealed: true, - payload: { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, valuesRedacted: true }, - createdAt: fact.updatedAt ?? this.now() - }, client); - } - } - return { events: persistedEvents, inputFacts }; - }); - const persistedFacts = { ...facts, inputs: txResult.inputFacts }; - this.memory.writeWorkbenchFacts({ facts: persistedFacts }, requestMeta); - return withPersistence({ written: true, facts: persistedFacts, events: txResult.events }, this.summary()); - } - - async queryWorkbenchFacts(params = {}) { - await this.assertReadyForDurableReads("workbench.facts.query"); - const families = workbenchFactFamilySet(params); - // A single Workbench read can request several fact families. Running those - // SQL reads in parallel lets one HTTP request occupy the whole Postgres - // pool, which makes the control+observer pages amplify into 503s under - // periodic refresh. Keep this read model bounded to one connection per - // request; the page-level latency is preferable to pool starvation. - const entries = []; - entries.push(["inputs", families.has("inputs") ? await this.queryWorkbenchFactRows("workbench.session-inputs.query", "workbench_session_inputs", "input_json", params, { inputId: "input_id", sessionId: "session_id", turnId: "turn_id", traceId: "trace_id", messageId: "message_id", commandId: "command_id", delivery: "delivery", status: "status" }) : []]); - entries.push(["sessions", families.has("sessions") ? await this.queryWorkbenchFactRows("workbench.sessions.query", "workbench_sessions", "session_json", params, { sessionId: "session_id", ownerUserId: "owner_user_id", projectId: "project_id", conversationId: "conversation_id", threadId: "thread_id", traceId: "last_trace_id", lastTraceId: "last_trace_id", status: "status" }) : []]); - entries.push(["messages", families.has("messages") ? await this.queryWorkbenchFactRows("workbench.messages.query", "workbench_messages", "message_json", params, { messageId: "message_id", sessionId: "session_id", turnId: "turn_id", traceId: "trace_id", role: "role", status: "status" }) : []]); - entries.push(["parts", families.has("parts") ? await this.queryWorkbenchFactRows("workbench.parts.query", "workbench_parts", "part_json", params, { partId: "part_id", messageId: "message_id", sessionId: "session_id", turnId: "turn_id", traceId: "trace_id", partType: "part_type", status: "status" }) : []]); - entries.push(["turns", families.has("turns") ? await this.queryWorkbenchFactRows("workbench.turns.query", "workbench_turns", "turn_json", params, { turnId: "turn_id", sessionId: "session_id", traceId: "trace_id", messageId: "message_id", status: "status" }) : []]); - entries.push(["traceEvents", families.has("traceEvents") ? await this.queryWorkbenchFactRows("workbench.trace-events.query", "workbench_trace_events", "event_json", params, { id: "id", traceId: "trace_id", sessionId: "session_id", turnId: "turn_id", messageId: "message_id", eventType: "event_type" }) : []]); - entries.push(["checkpoints", families.has("checkpoints") ? await this.queryWorkbenchFactRows("workbench.projection-checkpoints.query", "workbench_projection_checkpoints", "checkpoint_json", params, { traceId: "trace_id", sessionId: "session_id", turnId: "turn_id", runId: "run_id", commandId: "command_id", projectionStatus: "projection_status", projectionHealth: "projection_health" }) : []]); - const facts = Object.fromEntries(entries); - return withPersistence({ facts, count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0) }, this.summary()); - } - - async queryWorkbenchFactRows(method, table, jsonColumn, params = {}, columnMap = {}) { - const family = workbenchFactFamilyForTable(table); - const queryParams = []; - const clauses = []; - for (const [field, column] of Object.entries(columnMap)) { - const value = textOr(params[field], ""); - if (value) { - queryParams.push(value); - clauses.push(`${column} = $${queryParams.length}`); - } - const values = textList(params[`${field}s`]); - if (values.length > 0) { - queryParams.push(values); - clauses.push(`${column} = ANY($${queryParams.length})`); - } - } - const limit = positiveIntegerOrNull(params.limit); - if (family === "traceEvents") { - const afterProjectedSeq = nonNegativeInteger(params.afterProjectedSeq); - if (afterProjectedSeq > 0) { - queryParams.push(afterProjectedSeq); - clauses.push(`projected_seq > $${queryParams.length}`); - } - let sql = `SELECT ${jsonColumn} FROM ${table}${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY projected_seq ASC`; - if (limit) { - queryParams.push(limit); - sql += ` LIMIT $${queryParams.length}`; - } - const result = await this.queryDurableReadRows(method, sql, queryParams); - return result.rows.map((row) => parseJsonColumn(row[jsonColumn], null)).filter(Boolean); - } - const orderDirection = workbenchFactOrder(params, family) === "updated_desc" ? "DESC" : "ASC"; - const useSessionSummaryProjection = table === "workbench_sessions" && params.sessionProjection === "summary"; - const orderClause = family === "inputs" && orderDirection === "ASC" ? "admitted_seq ASC, input_id ASC" : `updated_at ${orderDirection}`; - let sql = `SELECT ${useSessionSummaryProjection ? workbenchSessionSummarySelectClause() : jsonColumn} FROM ${table}${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY ${orderClause}`; - if (limit) { - queryParams.push(limit); - sql += ` LIMIT $${queryParams.length}`; - } - const result = await this.queryDurableReadRows(method, sql, queryParams); - if (useSessionSummaryProjection) return result.rows.map(workbenchSessionSummaryFactFromRow).filter(Boolean); - return result.rows.map((row) => parseJsonColumn(row[jsonColumn], null)).filter(Boolean); - } - - async hydrateOperationRefs(params = {}) { - await this.hydrateGatewaySession(params.gatewaySessionId); - await this.hydrateBoxResource(params.resourceId); - await this.hydrateBoxCapability(params.capabilityId); - } - - async hydrateGatewaySession(gatewaySessionId) { - if (!gatewaySessionId || this.memory.gatewaySessions.has(gatewaySessionId)) return; - const result = await this.query("SELECT gateway_session_json FROM gateway_sessions WHERE id = $1 LIMIT 1", [ - gatewaySessionId - ]); - const record = parseJsonColumn(result.rows?.[0]?.gateway_session_json, null); - if (record) this.memory.gatewaySessions.set(record.gatewaySessionId, record); - } - - async hydrateBoxResource(resourceId) { - if (!resourceId || this.memory.boxResources.has(resourceId)) return; - const result = await this.query("SELECT resource_json FROM box_resources WHERE id = $1 LIMIT 1", [ - resourceId - ]); - const record = parseJsonColumn(result.rows?.[0]?.resource_json, null); - if (record) { - await this.hydrateGatewaySession(record.gatewaySessionId); - this.memory.boxResources.set(record.resourceId, record); - } - } - - async hydrateBoxCapability(capabilityId) { - if (!capabilityId || this.memory.boxCapabilities.has(capabilityId)) return; - const result = await this.query("SELECT capability_json FROM box_capabilities WHERE id = $1 LIMIT 1", [ - capabilityId - ]); - const record = parseJsonColumn(result.rows?.[0]?.capability_json, null); - if (record) { - await this.hydrateBoxResource(record.resourceId); - this.memory.boxCapabilities.set(record.capabilityId, record); - } - } - - async persistChanges(before) { - for (const record of newRecords(this.memory.gatewaySessions, before.gatewaySessions)) { - await this.persistGatewaySession(record); - } - for (const record of newRecords(this.memory.boxResources, before.boxResources)) { - await this.persistBoxResource(record); - } - for (const record of newRecords(this.memory.boxCapabilities, before.boxCapabilities)) { - await this.persistBoxCapability(record); - } - for (const record of changedRecords(this.memory.hardwareOperations, before.hardwareOperations)) { - await this.persistHardwareOperation(record); - } - for (const record of newRecords(this.memory.auditEvents, before.auditEvents)) { - await this.persistAuditEvent(record); - } - for (const record of newRecords(this.memory.evidenceRecords, before.evidenceRecords)) { - await this.persistEvidenceRecord(record); - } - for (const record of newRecords(this.memory.agentTraceEvents, before.agentTraceEvents)) { - await this.persistAgentTraceEvent(record); - } - for (const record of changedRecords(this.memory.workbenchProjectionStates, before.workbenchProjectionStates)) { - await this.persistWorkbenchProjectionState(record); - } - for (const record of changedRecords(this.memory.workbenchSessionInputs, before.workbenchSessionInputs)) { - await this.persistWorkbenchSessionInputFact(record); - } - for (const record of changedRecords(this.memory.workbenchSessions, before.workbenchSessions)) { - await this.persistWorkbenchSessionFact(record); - } - for (const record of changedRecords(this.memory.workbenchMessages, before.workbenchMessages)) { - await this.persistWorkbenchMessageFact(record); - } - for (const record of changedRecords(this.memory.workbenchParts, before.workbenchParts)) { - await this.persistWorkbenchPartFact(record); - } - for (const record of changedRecords(this.memory.workbenchTurns, before.workbenchTurns)) { - await this.persistWorkbenchTurnFact(record); - } - for (const record of changedRecords(this.memory.workbenchTraceEvents, before.workbenchTraceEvents)) { - await this.persistWorkbenchTraceEventFact(record); - } - for (const record of changedRecords(this.memory.workbenchProjectionCheckpoints, before.workbenchProjectionCheckpoints)) { - await this.persistWorkbenchProjectionCheckpoint(record); - } - } - - async assertReadyForWrites() { - if (!this.dbUrl && !this.queryClient) { - const readiness = this.blockedSummary({ - blocker: RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, - reason: "Postgres runtime adapter is selected but HWLAB_CLOUD_DB_URL is not injected" - }); - this.lastReadiness = readiness; - throw new HwlabProtocolError("Postgres durable runtime adapter is not ready for writes", { - code: ERROR_CODES.internalError, - data: { - adapter: RUNTIME_STORE_KIND_POSTGRES, - blocker: readiness.blocker, - status: readiness.status, - schemaReady: Boolean(readiness.schema?.ready) - }, - context: { - result: "failed" - } - }); - } - } - - async assertReadyForDurableReads(method) { - const current = this.summary(); - if (isReadyForDurableRuntimeRead(current)) return; - const readiness = await this.readiness(); - if (isReadyForDurableRuntimeRead(readiness)) return; - if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { - throw durableRuntimeReadBlockedError(method, readiness); - } - } - - async queryDurableReadRows(method, sql, params = []) { - try { - return await this.query(sql, params); - } catch (error) { - this.lastReadiness = this.blockedSummary({ - ...classifyRuntimeDbError(error), - reason: `Postgres durable runtime adapter could not complete ${method} read query` - }); - throw durableRuntimeReadBlockedError(method, this.summary()); - } - } - - async persistGatewaySession(record) { - await this.query( - "INSERT INTO gateway_sessions (id, project_id, gateway_service_id, status, started_at, ended_at, gateway_session_json) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, gateway_service_id = EXCLUDED.gateway_service_id, status = EXCLUDED.status, started_at = EXCLUDED.started_at, ended_at = EXCLUDED.ended_at, gateway_session_json = EXCLUDED.gateway_session_json", - [ - record.gatewaySessionId, - record.projectId, - record.serviceId, - record.status, - record.startedAt, - record.stoppedAt ?? null, - stableJson(record) - ] - ); - } - - async persistBoxResource(record) { - await this.query( - "INSERT INTO box_resources (id, project_id, gateway_session_id, resource_state, labels_json, resource_json, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, gateway_session_id = EXCLUDED.gateway_session_id, resource_state = EXCLUDED.resource_state, labels_json = EXCLUDED.labels_json, resource_json = EXCLUDED.resource_json, updated_at = EXCLUDED.updated_at", - [ - record.resourceId, - record.projectId, - record.gatewaySessionId, - record.state, - stableJson(record.metadata ?? {}), - stableJson(record), - record.updatedAt - ] - ); - } - - async persistBoxCapability(record) { - await this.query( - "INSERT INTO box_capabilities (id, box_resource_id, capability_type, capability_json, updated_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (id) DO UPDATE SET box_resource_id = EXCLUDED.box_resource_id, capability_type = EXCLUDED.capability_type, capability_json = EXCLUDED.capability_json, updated_at = EXCLUDED.updated_at", - [ - record.capabilityId, - record.resourceId, - record.name, - stableJson(record), - record.updatedAt - ] - ); - } - - async persistHardwareOperation(record) { - await this.query( - "INSERT INTO hardware_operations (id, project_id, requested_by, operation_type, operation_json, status, requested_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, requested_by = EXCLUDED.requested_by, operation_type = EXCLUDED.operation_type, operation_json = EXCLUDED.operation_json, status = EXCLUDED.status, requested_at = EXCLUDED.requested_at, updated_at = EXCLUDED.updated_at", - [ - record.operationId, - record.projectId, - record.requestedBy, - record.input?.shell ? "hardware.invoke.shell" : "hardware.operation.request", - stableJson(record), - record.status, - record.requestedAt, - record.updatedAt - ] - ); - } - - async persistAuditEvent(record) { - await this.query( - "INSERT INTO audit_events (id, request_id, actor, source, operation, target, result, timestamp, event_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (id) DO UPDATE SET request_id = EXCLUDED.request_id, actor = EXCLUDED.actor, source = EXCLUDED.source, operation = EXCLUDED.operation, target = EXCLUDED.target, result = EXCLUDED.result, timestamp = EXCLUDED.timestamp, event_json = EXCLUDED.event_json", - [ - record.auditId, - record.traceId, - stableJson({ type: record.actorType, id: record.actorId }), - stableJson({ serviceId: record.serviceId, environment: record.environment }), - record.action, - stableJson({ type: record.targetType, id: record.targetId }), - record.outcome ?? "accepted", - record.occurredAt, - stableJson(record) - ] - ); - } - - async persistEvidenceRecord(record) { - await this.query( - "INSERT INTO evidence_records (id, project_id, operation_id, evidence_type, uri, metadata_json, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, operation_id = EXCLUDED.operation_id, evidence_type = EXCLUDED.evidence_type, uri = EXCLUDED.uri, metadata_json = EXCLUDED.metadata_json, created_at = EXCLUDED.created_at", - [ - record.evidenceId, - record.projectId, - record.operationId, - record.kind, - record.uri, - stableJson(record), - record.createdAt - ] - ); - } - - async persistAgentTraceEvent(record) { - await this.query( - "INSERT INTO agent_trace_events (id, trace_id, agent_session_id, worker_session_id, level, message, event_json, occurred_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET trace_id = EXCLUDED.trace_id, agent_session_id = EXCLUDED.agent_session_id, worker_session_id = EXCLUDED.worker_session_id, level = EXCLUDED.level, message = EXCLUDED.message, event_json = EXCLUDED.event_json, occurred_at = EXCLUDED.occurred_at", - [ - record.id, - record.traceId, - record.agentSessionId, - record.workerSessionId, - record.level, - record.message, - stableJson(record.event), - record.occurredAt - ] - ); - } - - async persistWorkbenchProjectionState(record) { - await this.query( - "INSERT INTO workbench_projection_state (trace_id, session_id, conversation_id, thread_id, run_id, command_id, last_agentrun_seq, last_projected_seq, upstream_latest_seq, projection_status, projection_health, result_sync_state, last_projected_at, last_result_sync_at, last_error_code, last_error_message, failure_count, next_retry_at, projection_json, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) ON CONFLICT (trace_id) DO UPDATE SET session_id = EXCLUDED.session_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, run_id = EXCLUDED.run_id, command_id = EXCLUDED.command_id, last_agentrun_seq = EXCLUDED.last_agentrun_seq, last_projected_seq = EXCLUDED.last_projected_seq, upstream_latest_seq = EXCLUDED.upstream_latest_seq, projection_status = EXCLUDED.projection_status, projection_health = EXCLUDED.projection_health, result_sync_state = EXCLUDED.result_sync_state, last_projected_at = EXCLUDED.last_projected_at, last_result_sync_at = EXCLUDED.last_result_sync_at, last_error_code = EXCLUDED.last_error_code, last_error_message = EXCLUDED.last_error_message, failure_count = EXCLUDED.failure_count, next_retry_at = EXCLUDED.next_retry_at, projection_json = EXCLUDED.projection_json, updated_at = EXCLUDED.updated_at", - [ - record.traceId, - record.sessionId, - record.conversationId, - record.threadId, - record.runId, - record.commandId, - record.lastAgentRunSeq, - record.lastProjectedSeq, - record.upstreamLatestSeq, - record.projectionStatus, - record.projectionHealth, - record.resultSyncState, - record.lastProjectedAt, - record.lastResultSyncAt, - record.lastErrorCode, - record.lastErrorMessage, - record.failureCount, - record.nextRetryAt, - stableJson(record), - record.createdAt, - record.updatedAt - ] - ); - } - - async persistWorkbenchAggregateEvent(record, client = this) { - const existing = await client.query( - "SELECT event_seq, aggregate_seq, projection_revision FROM workbench_events WHERE event_id = $1 OR ($2::text IS NOT NULL AND aggregate_id = $3 AND source_event_id = $2) ORDER BY event_seq ASC LIMIT 1", - [record.eventId, record.sourceEventId, record.aggregateId] - ); - const existingRow = existing.rows?.[0] ?? null; - if (existingRow) { - return { - ...record, - eventSeq: Number(existingRow.event_seq), - aggregateSeq: Number(existingRow.aggregate_seq), - projectionRevision: Number(existingRow.projection_revision) - }; - } - await client.query( - "INSERT INTO workbench_event_sequences (aggregate_id, aggregate_type, session_id, turn_id, trace_id, last_seq, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,0,$6,$7) ON CONFLICT (aggregate_id) DO UPDATE SET aggregate_type = EXCLUDED.aggregate_type, session_id = COALESCE(workbench_event_sequences.session_id, EXCLUDED.session_id), turn_id = COALESCE(workbench_event_sequences.turn_id, EXCLUDED.turn_id), trace_id = COALESCE(workbench_event_sequences.trace_id, EXCLUDED.trace_id), updated_at = EXCLUDED.updated_at", - [record.aggregateId, record.aggregateType, record.sessionId, record.turnId, record.traceId, record.committedAt, record.committedAt] - ); - const reserved = await client.query( - "UPDATE workbench_event_sequences SET last_seq = last_seq + 1, updated_at = $2 WHERE aggregate_id = $1 RETURNING last_seq", - [record.aggregateId, record.committedAt] - ); - const aggregateSeq = nonNegativeInteger(reserved.rows?.[0]?.last_seq); - const projectionRevision = nonNegativeInteger(record.projectionRevision) || aggregateSeq; - const inserted = await client.query( - "INSERT INTO workbench_events (event_id, aggregate_id, aggregate_type, aggregate_seq, session_id, turn_id, trace_id, message_id, source_run_id, source_command_id, source_seq, source_event_id, event_type, projection_revision, terminal, sealed, payload_json, occurred_at, committed_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT (event_id) DO UPDATE SET projection_revision = GREATEST(workbench_events.projection_revision, EXCLUDED.projection_revision), terminal = workbench_events.terminal OR EXCLUDED.terminal, sealed = workbench_events.sealed OR EXCLUDED.sealed, payload_json = EXCLUDED.payload_json, committed_at = EXCLUDED.committed_at RETURNING event_seq, aggregate_seq, projection_revision", - [record.eventId, record.aggregateId, record.aggregateType, aggregateSeq, record.sessionId, record.turnId, record.traceId, record.messageId, record.sourceRunId, record.sourceCommandId, record.sourceSeq, record.sourceEventId, record.eventType, projectionRevision, Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.occurredAt, record.committedAt] - ); - const row = inserted.rows?.[0] ?? {}; - return { - ...record, - eventSeq: Number(row.event_seq), - aggregateSeq: Number(row.aggregate_seq) || aggregateSeq, - projectionRevision: Number(row.projection_revision) || projectionRevision - }; - } - - async persistWorkbenchSessionFact(record, client = this) { - await client.query( - "INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = EXCLUDED.owner_user_id, project_id = EXCLUDED.project_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at", - [record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] - ); - } - - async persistWorkbenchSessionInputFact(record, client = this) { - await client.query( - "INSERT INTO workbench_session_inputs (input_id, session_id, turn_id, trace_id, message_id, command_id, delivery, admitted_seq, promoted_seq, status, error_code, source_event_id, input_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (input_id) DO UPDATE SET session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, message_id = EXCLUDED.message_id, command_id = COALESCE(EXCLUDED.command_id, workbench_session_inputs.command_id), delivery = EXCLUDED.delivery, admitted_seq = CASE WHEN workbench_session_inputs.admitted_seq > 0 THEN workbench_session_inputs.admitted_seq ELSE EXCLUDED.admitted_seq END, promoted_seq = COALESCE(EXCLUDED.promoted_seq, workbench_session_inputs.promoted_seq), status = EXCLUDED.status, error_code = EXCLUDED.error_code, source_event_id = COALESCE(EXCLUDED.source_event_id, workbench_session_inputs.source_event_id), input_json = EXCLUDED.input_json, updated_at = EXCLUDED.updated_at", - [record.inputId, record.sessionId, record.turnId, record.traceId, record.messageId, record.commandId, record.delivery, record.admittedSeq, record.promotedSeq, record.status, record.errorCode, record.sourceEventId, stableJson(record), record.createdAt, record.updatedAt] - ); - } - - async persistWorkbenchMessageFact(record, client = this) { - await client.query( - "INSERT INTO workbench_messages (message_id, session_id, turn_id, trace_id, role, status, projected_seq, source_seq, source_event_id, terminal, sealed, message_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (message_id) DO UPDATE SET session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, role = EXCLUDED.role, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, message_json = EXCLUDED.message_json, updated_at = EXCLUDED.updated_at", - [record.messageId, record.sessionId, record.turnId, record.traceId, record.role, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] - ); - } - - async persistWorkbenchPartFact(record, client = this) { - await client.query( - "INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, part_json = EXCLUDED.part_json, updated_at = EXCLUDED.updated_at", - [record.partId, record.messageId, record.sessionId, record.turnId, record.traceId, record.partIndex, record.partType, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] - ); - } - - async persistWorkbenchTurnFact(record, client = this) { - await client.query( - "INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = EXCLUDED.message_id, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, final_response_json = EXCLUDED.final_response_json, diagnostic_json = EXCLUDED.diagnostic_json, turn_json = EXCLUDED.turn_json, updated_at = EXCLUDED.updated_at", - [record.turnId, record.sessionId, record.traceId, record.messageId, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record.finalResponse), stableJson(record.diagnostic), stableJson(record), record.createdAt, record.updatedAt] - ); - } - - async persistWorkbenchTraceEventFact(record, client = this) { - await client.query( - "INSERT INTO workbench_trace_events (id, trace_id, session_id, turn_id, message_id, source_seq, source_event_id, projected_seq, event_type, terminal, sealed, event_json, occurred_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (id) DO UPDATE SET trace_id = EXCLUDED.trace_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, message_id = EXCLUDED.message_id, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, projected_seq = EXCLUDED.projected_seq, event_type = EXCLUDED.event_type, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, event_json = EXCLUDED.event_json, occurred_at = EXCLUDED.occurred_at, updated_at = EXCLUDED.updated_at", - [record.id, record.traceId, record.sessionId, record.turnId, record.messageId, record.sourceSeq, record.sourceEventId, record.projectedSeq, record.eventType, record.terminal, record.sealed, stableJson(record), record.occurredAt, record.updatedAt] - ); - } - - async persistWorkbenchProjectionOutbox(record, client = this) { - await client.query( - "INSERT INTO workbench_projection_outbox (event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)", - [record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()] - ); - } - - async readWorkbenchProjectionOutbox({ afterSeq = 0, limit = 100, traceId = null, sessionId = null } = {}) { - const params = []; - let where = "outbox_seq > $1"; - params.push(Number(afterSeq) || 0); - let paramIdx = 2; - if (traceId) { - where += ` AND trace_id = $${paramIdx}`; - params.push(traceId); - paramIdx += 1; - } - if (sessionId) { - where += ` AND session_id = $${paramIdx}`; - params.push(sessionId); - paramIdx += 1; - } - params.push(Math.min(Math.max(Number(limit) || 100, 1), 500)); - const result = await this.query( - `SELECT outbox_seq, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`, - params - ); - return (result.rows ?? []).map((row) => ({ - outboxSeq: Number(row.outbox_seq), - eventSeq: row.event_seq === null || row.event_seq === undefined ? null : Number(row.event_seq), - aggregateId: row.aggregate_id, - aggregateSeq: Number(row.aggregate_seq ?? 0), - projectionRevision: Number(row.projection_revision ?? 0), - traceId: row.trace_id, - sessionId: row.session_id, - turnId: row.turn_id, - messageId: row.message_id, - projectedSeq: Number(row.projected_seq), - sourceSeq: Number(row.source_seq), - sourceEventId: row.source_event_id, - commitType: row.commit_type, - terminal: Boolean(row.terminal), - sealed: Boolean(row.sealed), - payload: typeof row.payload_json === "string" ? JSON.parse(row.payload_json) : row.payload_json, - createdAt: row.created_at, - valuesPrinted: false - })); - } - - async persistWorkbenchProjectionCheckpoint(record, client = this) { - await client.query( - "INSERT INTO workbench_projection_checkpoints (trace_id, session_id, turn_id, run_id, command_id, projected_seq, source_seq, source_event_id, projection_status, projection_health, terminal, sealed, diagnostic_json, checkpoint_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (trace_id) DO UPDATE SET session_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.session_id ELSE workbench_projection_checkpoints.session_id END, turn_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.turn_id ELSE workbench_projection_checkpoints.turn_id END, run_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.run_id ELSE workbench_projection_checkpoints.run_id END, command_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.command_id ELSE workbench_projection_checkpoints.command_id END, projected_seq = GREATEST(workbench_projection_checkpoints.projected_seq, EXCLUDED.projected_seq), source_seq = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.source_seq ELSE workbench_projection_checkpoints.source_seq END, source_event_id = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.source_event_id ELSE workbench_projection_checkpoints.source_event_id END, projection_status = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.projection_status ELSE workbench_projection_checkpoints.projection_status END, projection_health = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.projection_health ELSE workbench_projection_checkpoints.projection_health END, terminal = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.terminal ELSE workbench_projection_checkpoints.terminal END, sealed = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.sealed ELSE workbench_projection_checkpoints.sealed END, diagnostic_json = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.diagnostic_json ELSE workbench_projection_checkpoints.diagnostic_json END, checkpoint_json = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.checkpoint_json ELSE workbench_projection_checkpoints.checkpoint_json END, updated_at = CASE WHEN EXCLUDED.projected_seq >= workbench_projection_checkpoints.projected_seq THEN EXCLUDED.updated_at ELSE workbench_projection_checkpoints.updated_at END", - [record.traceId, record.sessionId, record.turnId, record.runId, record.commandId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.projectionStatus, record.projectionHealth, record.terminal, record.sealed, stableJson(record.diagnostic), stableJson(record), record.createdAt, record.updatedAt] - ); - } - - async readCounts() { - const counts = {}; - for (const table of postgresCountTables) { - const result = await this.query(`SELECT COUNT(*)::int AS count FROM ${table}`, []); - counts[toCountKey(table)] = Number(result.rows?.[0]?.count ?? 0); - } - return counts; - } - - async readMigrationReadiness() { - const result = await this.query( - `SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`, - [CLOUD_CORE_MIGRATION_ID] - ); - const row = result.rows?.[0] ?? null; - const ready = - row?.id === CLOUD_CORE_MIGRATION_ID && - row?.schema_version === CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION; - return { - checked: true, - ready, - table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: CLOUD_CORE_MIGRATION_ID, - requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, - appliedMigrationId: row?.id ?? null, - appliedSchemaVersion: row?.schema_version ?? null, - missing: !row - }; - } - - async ensureRuntimeReadIndexes() { - if (this.runtimeReadIndexesReady) return; - if (!this.runtimeReadIndexesReadyPromise) { - this.runtimeReadIndexesReadyPromise = (async () => { - for (const sql of postgresRuntimeReadIndexes) { - await this.query(sql, []); - } - this.runtimeReadIndexesReady = true; - })(); - } - try { - await this.runtimeReadIndexesReadyPromise; - } finally { - if (!this.runtimeReadIndexesReady) this.runtimeReadIndexesReadyPromise = null; - } - } - - recordRuntimeDbOperationFailure(label, error, retry = {}) { - const classified = classifyRuntimeDbError(error); - this.lastReadiness = this.blockedSummary({ - ...classified, - reason: `Postgres durable runtime adapter could not complete ${label}` - }); - this.logger?.warn?.({ - event: "postgres_runtime_operation_failed", - operation: label, - blocker: classified.blocker, - queryResult: classified.connection?.queryResult ?? "query_blocked", - errorCode: classified.connection?.errorCode ?? "UNKNOWN", - retryable: classified.retryable === true, - transient: classified.transient === true, - retryAfterMs: classified.retryAfterMs ?? null, - retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, - retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, - retrying: retry.retrying === true, - retryDelayMs: Number.isFinite(Number(retry.delayMs)) ? Number(retry.delayMs) : null, - retryLabel: retry.label ?? null, - valuesRedacted: true, - endpointRedacted: true - }); - } - - resetPostgresPoolAfterTransientFailure(classified, retry = {}) { - if (classified?.connection?.queryResult !== "connect_timeout") return false; - const pool = this.pool; - if (!pool) return false; - const now = Date.now(); - const cooldownMs = this.runtimePoolResetCooldownMs(); - if (this.postgresPoolResetInFlight) { - this.logger?.warn?.({ - event: "postgres_runtime_pool_reset_skipped", - reason: "in_flight", - blocker: classified.blocker, - queryResult: classified.connection?.queryResult ?? "query_blocked", - errorCode: classified.connection?.errorCode ?? "UNKNOWN", - retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, - retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, - cooldownMs, - poolStats: postgresPoolStats(pool), - valuesRedacted: true, - endpointRedacted: true - }); - return false; - } - if (now - this.postgresPoolResetLastAtMs < cooldownMs) { - this.logger?.warn?.({ - event: "postgres_runtime_pool_reset_skipped", - reason: "cooldown", - blocker: classified.blocker, - queryResult: classified.connection?.queryResult ?? "query_blocked", - errorCode: classified.connection?.errorCode ?? "UNKNOWN", - retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, - retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, - cooldownMs, - poolResetLastAtMs: this.postgresPoolResetLastAtMs, - poolStats: postgresPoolStats(pool), - valuesRedacted: true, - endpointRedacted: true - }); - return false; - } - this.pool = null; - this.postgresPoolResetLastAtMs = now; - this.logger?.warn?.({ - event: "postgres_runtime_pool_reset_after_transient_failure", - blocker: classified.blocker, - queryResult: classified.connection?.queryResult ?? "query_blocked", - errorCode: classified.connection?.errorCode ?? "UNKNOWN", - retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, - retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, - retrying: retry.retrying === true, - cooldownMs, - poolStats: postgresPoolStats(pool), - valuesRedacted: true, - endpointRedacted: true - }); - if (typeof pool.end === "function") { - this.postgresPoolResetInFlight = Promise.resolve() - .then(() => pool.end()) - .catch((error) => this.logger?.warn?.({ - event: "postgres_runtime_pool_reset_failed", - errorCode: error?.code ?? "UNKNOWN", - valuesRedacted: true, - endpointRedacted: true - })) - .finally(() => { - this.postgresPoolResetInFlight = null; - }); - } - return true; - } - - runtimePoolResetCooldownMs() { - const env = this.env ?? {}; - return normalizePositiveInteger(env[RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV], DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS); - } - - async query(sql, params = []) { - const retry = this.runtimeQueryRetryPolicy(); - for (let attempt = 1; attempt <= retry.maxAttempts; attempt += 1) { - try { - const client = await this.getQueryClient(); - return await client.query(sql, params); - } catch (error) { - const classified = classifyRuntimeDbError(error); - const retrying = classified.retryable === true && classified.transient === true && attempt < retry.maxAttempts; - const delayMs = retrying ? Math.min(retry.maxDelayMs, retry.initialDelayMs * (2 ** Math.max(0, attempt - 1))) : null; - this.recordRuntimeDbOperationFailure("query", error, { - attempt, - maxAttempts: retry.maxAttempts, - retrying, - delayMs, - label: `${attempt}/${retry.maxAttempts}` - }); - this.resetPostgresPoolAfterTransientFailure(classified, { - attempt, - maxAttempts: retry.maxAttempts, - retrying, - delayMs, - label: `${attempt}/${retry.maxAttempts}` - }); - if (!retrying) throw error; - await delayRuntimeDbQueryRetry(delayMs); - } - } - } - - runtimeQueryRetryPolicy() { - const env = this.env ?? {}; - return { - maxAttempts: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS), - initialDelayMs: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS), - maxDelayMs: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS) - }; - } - - async withDurableTransaction(label, fn) { - try { - const client = await this.getQueryClient(); - if (typeof client.connect === "function") { - const tx = await client.connect(); - try { - await tx.query("BEGIN"); - const result = await fn(tx); - await tx.query("COMMIT"); - return result; - } catch (error) { - try { await tx.query("ROLLBACK"); } catch {} - throw error; - } finally { - tx.release?.(); - } - } - await client.query("BEGIN"); - try { - const result = await fn(client); - await client.query("COMMIT"); - return result; - } catch (error) { - try { await client.query("ROLLBACK"); } catch {} - throw error; - } - } catch (error) { - this.recordRuntimeDbOperationFailure(label || "transaction", error); - throw error; - } - } - - async getQueryClient() { - if (this.queryClient) { - return this.queryClient; - } - - if (this.pool) { - return this.pool; - } - - let pg; - try { - pg = await (this.pgModuleLoader ? this.pgModuleLoader() : import("pg")); - } catch (error) { - if (error?.code === "ERR_MODULE_NOT_FOUND") { - const driverError = new Error("Postgres runtime adapter requires the pg package"); - driverError.code = "HWLAB_PG_DRIVER_MISSING"; - throw driverError; - } - throw error; - } - - const Pool = pg.Pool ?? pg.default?.Pool; - if (typeof Pool !== "function") { - const driverError = new Error("Postgres runtime adapter could not load pg.Pool"); - driverError.code = "HWLAB_PG_DRIVER_MISSING"; - throw driverError; - } - - this.pool = new Pool(buildPostgresPoolConfig({ - dbUrl: this.dbUrl, - sslMode: this.sslMode, - timeoutMs: this.env?.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS, - queryTimeoutMs: this.env?.[RUNTIME_DB_QUERY_TIMEOUT_MS_ENV], - poolMax: this.env?.[RUNTIME_DB_POOL_MAX_ENV] - })); - if (typeof this.pool.on === "function") { - this.pool.on("error", (error) => this.handlePostgresPoolError(error)); - } - return this.pool; - } - - handlePostgresPoolError(error) { - const classified = classifyRuntimeDbError(error); - this.lastReadiness = this.blockedSummary(classified); - this.logger?.warn?.({ - event: "postgres_runtime_pool_error", - blocker: classified.blocker, - queryResult: classified.connection?.queryResult ?? "query_blocked", - errorCode: classified.connection?.errorCode ?? "UNKNOWN", - retryable: classified.retryable === true, - transient: classified.transient === true, - retryAfterMs: classified.retryAfterMs ?? null, - valuesRedacted: true, - endpointRedacted: true - }); - } - - blockedSummary({ blocker, reason, schema, migration, connection, gates, retryable, transient, retryAfterMs }) { - const blockedSchema = schema ?? { - ready: false, - checked: false, - missingTables: [], - missingColumns: [] - }; - const blockedMigration = migration ?? notCheckedRuntimeMigration(); - return addRuntimeDurabilityContract({ - adapter: RUNTIME_STORE_KIND_POSTGRES, - durable: false, - durableRequested: true, - durableCapable: false, - ready: false, - status: "blocked", - blocker, - reason, - retryable: retryable === true ? true : undefined, - transient: transient === true ? true : undefined, - retryAfterMs: Number.isFinite(Number(retryAfterMs)) ? Math.max(0, Math.trunc(Number(retryAfterMs))) : undefined, - liveRuntimeEvidence: false, - fixtureEvidence: false, - connection: { - queryAttempted: Boolean(connection?.queryAttempted), - queryResult: connection?.queryResult ?? "not_ready", - endpointRedacted: true, - valueRedacted: true, - errorCode: connection?.errorCode ?? null - }, - schema: blockedSchema, - migration: blockedMigration, - gates: gates ?? runtimeGates({ - ...defaultRuntimeGates({ blocker, connection, schema: blockedSchema, migration: blockedMigration }), - schema: defaultSchemaGate({ blocker, schema: blockedSchema }), - migration: defaultMigrationGate({ blocker, migration: blockedMigration }), - durability: blockedGate({ checked: false, blocker }) - }), - safety: runtimeSafety(), - adapterContract: postgresAdapterContract() - }); - } -} - -function isReadyForDurableRuntimeRead(readiness = {}) { - return readiness.ready === true && readiness.durable === true && readiness.liveRuntimeEvidence === true; -} - -function normalizeCapabilityInputs(params) { - const value = params.capabilities ?? params.capability ?? params; - if (Array.isArray(value)) { - if (value.length === 0) { - throw new HwlabProtocolError("capabilities must not be empty", { - code: ERROR_CODES.invalidParams - }); - } - return value.map((item) => asObject(item, "capability")); - } - return [asObject(value, "capability")]; -} - -function normalizeShellInput(params) { - const input = normalizeJsonObject(params.input); - const command = params.command ?? input.command; - if (command === undefined || command === null || command === "") { - throw new HwlabProtocolError("hardware.invoke.shell requires input.command or command", { - code: ERROR_CODES.invalidParams, - data: { - required: ["projectId", "gatewaySessionId", "resourceId", "capabilityId", "input.command"] - } - }); - } - if (Array.isArray(command)) { - return { - command: command.map((item) => String(item)), - cwd: input.cwd, - timeoutMs: input.timeoutMs - }; - } - return { - command: String(command), - cwd: input.cwd, - timeoutMs: input.timeoutMs - }; -} - -function normalizeAgentTraceEvent(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); - if (!traceId) { - throw new HwlabProtocolError("agent trace event requires traceId", { - code: ERROR_CODES.invalidParams, - data: { required: ["traceId"] } - }); - } - const seq = positiveIntegerOrNull(input.seq); - const occurredAt = timestampOr(input.occurredAt ?? input.createdAt ?? input.timestamp, now); - const status = textOr(input.status, "observed"); - const type = textOr(input.type ?? input.kind, "event"); - const level = traceEventLevel(input.level ?? status ?? type); - const message = textOr(input.message ?? input.label ?? type, ""); - const agentSessionId = textOr(input.agentSessionId ?? input.sessionId ?? requestMeta.agentSessionId ?? requestMeta.sessionId, ""); - const workerSessionId = textOr(input.workerSessionId ?? input.workerSession?.id ?? input.workerId, ""); - const id = textOr(input.id ?? input.eventId, "") || `tev_${sha256Hex({ - traceId, - seq, - source: input.source ?? null, - sourceSeq: input.sourceSeq ?? null, - type, - status, - label: input.label ?? null, - message, - occurredAt - }).slice(0, 32)}`; - const event = pruneUndefined({ - ...input, - traceId, - seq: seq ?? undefined, - agentSessionId: agentSessionId || undefined, - workerSessionId: workerSessionId || undefined, - level, - createdAt: timestampOr(input.createdAt ?? occurredAt, occurredAt), - valuesPrinted: false - }); - return { - id, - traceId, - agentSessionId: agentSessionId || null, - workerSessionId: workerSessionId || null, - level, - message, - occurredAt, - event - }; -} - -function normalizeWorkbenchProjectionState(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); - const runId = textOr(input.runId ?? input.sourceRunId, ""); - const commandId = textOr(input.commandId ?? input.sourceCommandId, ""); - if (!traceId || !runId || !commandId) { - throw new HwlabProtocolError("workbench projection state requires traceId, runId, and commandId", { - code: ERROR_CODES.invalidParams, - data: { required: ["traceId", "runId", "commandId"] } - }); - } - const timestamp = timestampOr(input.updatedAt, now); - const lastAgentRunSeq = nonNegativeInteger(input.lastAgentRunSeq ?? input.lastSourceSeq); - const lastProjectedSeq = nonNegativeInteger(input.lastProjectedSeq ?? lastAgentRunSeq); - const upstreamLatestSeq = nonNegativeIntegerOrNull(input.upstreamLatestSeq ?? input.sourceLatestSeq); - return pruneUndefined({ - ...input, - traceId, - sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, - conversationId: textOr(input.conversationId, "") || null, - threadId: textOr(input.threadId, "") || null, - ownerUserId: textOr(input.ownerUserId, "") || null, - ownerRole: textOr(input.ownerRole, "") || null, - sourceRunId: runId, - sourceCommandId: commandId, - runId, - commandId, - lastSourceSeq: lastAgentRunSeq, - lastAgentRunSeq, - lastProjectedSeq, - sourceLatestSeq: upstreamLatestSeq ?? lastAgentRunSeq, - upstreamLatestSeq: upstreamLatestSeq ?? lastAgentRunSeq, - projectionStatus: projectionStatusOr(input.projectionStatus, "projecting"), - projectionHealth: projectionHealthOr(input.projectionHealth, "healthy"), - resultSyncState: resultSyncStateOr(input.resultSyncState, "not_started"), - lastProjectedAt: timestampOr(input.lastProjectedAt, timestamp), - lastResultSyncAt: input.lastResultSyncAt ? timestampOr(input.lastResultSyncAt, timestamp) : null, - lastErrorCode: textOr(input.lastErrorCode, "") || null, - lastErrorMessage: textOr(input.lastErrorMessage, "") || null, - failureCount: nonNegativeInteger(input.failureCount), - nextRetryAt: input.nextRetryAt ? timestampOr(input.nextRetryAt, timestamp) : null, - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false - }); -} - -function normalizeWorkbenchProjectionAllocation(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); - const sourceEventId = textOr(input.sourceEventId ?? input.eventId, ""); - if (!traceId || !sourceEventId) { - throw new HwlabProtocolError("workbench projectedSeq allocation requires traceId and sourceEventId", { - code: ERROR_CODES.invalidParams, - data: { required: ["traceId", "sourceEventId"] } - }); - } - const timestamp = timestampOr(input.updatedAt ?? input.occurredAt ?? input.createdAt, now); - return pruneUndefined({ - ...input, - traceId, - sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, - turnId: textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null, - runId: textOr(input.runId ?? input.sourceRunId, "") || null, - commandId: textOr(input.commandId ?? input.sourceCommandId, "") || null, - sourceSeq: nonNegativeInteger(input.sourceSeq ?? input.lastSourceSeq ?? input.lastAgentRunSeq), - sourceEventId, - eventType: textOr(input.eventType ?? input.type, "event"), - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false - }); -} - -function normalizeWorkbenchFacts(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - return { - inputs: arrayInput(input.inputs ?? input.input).map((item) => normalizeWorkbenchSessionInputFact(item, requestMeta, now)), - sessions: arrayInput(input.sessions ?? input.session).map((item) => normalizeWorkbenchSessionFact(item, requestMeta, now)), - messages: arrayInput(input.messages ?? input.message).map((item) => normalizeWorkbenchMessageFact(item, requestMeta, now)), - parts: arrayInput(input.parts ?? input.part).map((item) => normalizeWorkbenchPartFact(item, requestMeta, now)), - turns: arrayInput(input.turns ?? input.turn).map((item) => normalizeWorkbenchTurnFact(item, requestMeta, now)), - traceEvents: arrayInput(input.traceEvents ?? input.traceEvent).map((item) => normalizeWorkbenchTraceEventFact(item, requestMeta, now)), - checkpoints: arrayInput(input.checkpoints ?? input.checkpoint).map((item) => normalizeWorkbenchProjectionCheckpoint(item, requestMeta, now)) - }; -} - -function normalizeWorkbenchAggregateEventsForFacts(facts = {}, requestMeta = {}, now) { - const events = []; - for (const fact of arrayInput(facts.inputs)) appendWorkbenchFactEvent(events, "inputs", fact, requestMeta, now); - for (const fact of arrayInput(facts.sessions)) appendWorkbenchFactEvent(events, "sessions", fact, requestMeta, now); - for (const fact of arrayInput(facts.messages)) appendWorkbenchFactEvent(events, "messages", fact, requestMeta, now); - for (const fact of arrayInput(facts.parts)) appendWorkbenchFactEvent(events, "parts", fact, requestMeta, now); - for (const fact of arrayInput(facts.traceEvents)) appendWorkbenchFactEvent(events, "traceEvents", fact, requestMeta, now); - for (const fact of arrayInput(facts.checkpoints)) appendWorkbenchFactEvent(events, "checkpoints", fact, requestMeta, now); - for (const fact of arrayInput(facts.turns)) appendWorkbenchFactEvent(events, "turns", fact, requestMeta, now); - return events; -} - -function appendWorkbenchFactEvent(events, factFamily, fact, requestMeta, now) { - const aggregate = workbenchAggregateForFact(fact, requestMeta); - const factId = textOr(fact.id ?? fact.inputId ?? fact.sessionId ?? fact.messageId ?? fact.partId ?? fact.turnId ?? fact.traceId, ""); - const rawSourceEventId = textOr(fact.sourceEventId ?? fact.eventId, "") || factId; - const sourceEventId = rawSourceEventId ? `${factFamily}:${rawSourceEventId}` : null; - const eventType = workbenchAggregateEventType(factFamily, fact); - const projectedSeq = nonNegativeInteger(fact.projectedSeq); - const sourceSeq = nonNegativeInteger(fact.sourceSeq); - const occurredAt = timestampOr(fact.occurredAt ?? fact.updatedAt ?? fact.createdAt, now); - events.push(normalizeWorkbenchAggregateEvent({ - ...aggregate, - factFamily, - factId, - factKey: `${factFamily.slice(0, -1) || factFamily}:${factId}`, - eventId: stableWorkbenchFactId("wbe", { aggregateId: aggregate.aggregateId, factFamily, sourceEventId, eventType, factId }), - messageId: textOr(fact.messageId, "") || null, - sourceRunId: textOr(fact.sourceRunId ?? fact.runId, "") || null, - sourceCommandId: textOr(fact.sourceCommandId ?? fact.commandId, "") || null, - sourceSeq, - sourceEventId, - eventType, - projectionRevision: projectedSeq || sourceSeq, - terminal: Boolean(fact.terminal), - sealed: Boolean(fact.sealed), - payload: { - factFamily, - factId, - status: textOr(fact.status ?? fact.projectionStatus, "") || null, - eventType, - projectedSeq, - sourceSeq, - originalSourceEventId: rawSourceEventId || null, - valuesRedacted: true - }, - occurredAt, - committedAt: timestampOr(fact.updatedAt ?? occurredAt, now) - }, requestMeta, now)); -} - -function workbenchAggregateForFact(fact = {}, requestMeta = {}) { - const sessionId = textOr(fact.sessionId ?? requestMeta.sessionId, "") || null; - const turnId = textOr(fact.turnId ?? requestMeta.turnId, "") || null; - const traceId = textOr(fact.traceId ?? requestMeta.traceId, "") || null; - if (sessionId) return { aggregateId: `session:${sessionId}`, aggregateType: "session", sessionId, turnId, traceId }; - if (turnId) return { aggregateId: `turn:${turnId}`, aggregateType: "turn", sessionId, turnId, traceId }; - if (traceId) return { aggregateId: `trace:${traceId}`, aggregateType: "trace", sessionId, turnId, traceId }; - throw requiredWorkbenchFactError("workbench aggregate event", ["sessionId", "turnId", "traceId"]); -} - -function normalizeWorkbenchAggregateEvent(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const aggregateId = textOr(input.aggregateId, ""); - if (!aggregateId) throw requiredWorkbenchFactError("workbench aggregate event", ["aggregateId"]); - const timestamp = timestampOr(input.committedAt ?? input.occurredAt ?? input.updatedAt ?? input.createdAt, now); - const sourceSeq = nonNegativeInteger(input.sourceSeq); - const projectionRevision = nonNegativeInteger(input.projectionRevision ?? input.projectedSeq ?? sourceSeq); - return pruneUndefined({ - ...input, - eventSeq: nonNegativeInteger(input.eventSeq), - eventId: textOr(input.eventId ?? input.id, "") || stableWorkbenchFactId("wbe", { aggregateId, eventType: input.eventType, sourceEventId: input.sourceEventId, sourceSeq, projectionRevision }), - aggregateId, - aggregateType: textOr(input.aggregateType, "trace"), - aggregateSeq: nonNegativeInteger(input.aggregateSeq), - sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, - turnId: textOr(input.turnId ?? requestMeta.turnId, "") || null, - traceId: textOr(input.traceId ?? requestMeta.traceId, "") || null, - messageId: textOr(input.messageId ?? requestMeta.messageId, "") || null, - sourceRunId: textOr(input.sourceRunId ?? input.runId, "") || null, - sourceCommandId: textOr(input.sourceCommandId ?? input.commandId, "") || null, - sourceSeq, - sourceEventId: textOr(input.sourceEventId, "") || null, - eventType: textOr(input.eventType ?? input.type, "event"), - projectionRevision, - terminal: Boolean(input.terminal), - sealed: Boolean(input.sealed), - payload: normalizeJsonObject(input.payload), - occurredAt: timestampOr(input.occurredAt, timestamp), - committedAt: timestamp, - valuesPrinted: false - }); -} - -function workbenchAggregateEventType(factFamily, fact = {}) { - if (factFamily === "inputs") return `input_${textOr(fact.delivery, "queue")}_${textOr(fact.status, "admitted")}`; - if (factFamily === "sessions") return "session_upsert"; - if (factFamily === "messages") return `${textOr(fact.role, "agent")}_message`; - if (factFamily === "parts") return `part_${textOr(fact.partType ?? fact.type, "text")}`; - if (factFamily === "turns") return fact.terminal || fact.sealed ? "turn_terminal" : "turn_update"; - if (factFamily === "traceEvents") return `trace_${textOr(fact.eventType ?? fact.type, "event")}`; - if (factFamily === "checkpoints") return `checkpoint_${textOr(fact.projectionStatus, "projecting")}`; - return "event"; -} - -function workbenchInputDelivery(value) { - const text = textOr(value, "queue").toLowerCase(); - return ["queue", "steer", "cancel"].includes(text) ? text : "queue"; -} - -function workbenchInputStatus(value) { - const text = textOr(value, "admitted").toLowerCase().replace(/-/gu, "_"); - return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; -} - -function indexWorkbenchAggregateEvents(events = []) { - const index = new Map(); - for (const event of events) { - if (event?.factKey) index.set(event.factKey, event); - if (event?.factFamily && event?.factId) index.set(`${event.factFamily}:${event.factId}`, event); - if (event?.factFamily === "inputs" && event?.factId) index.set(`input:${event.factId}`, event); - if (event?.factFamily === "turns" && event?.turnId) index.set(`turn:${event.turnId}`, event); - if (event?.factFamily === "traceEvents" && event?.factId) index.set(`traceEvent:${event.factId}`, event); - if (event?.sourceEventId) index.set(`sourceEvent:${event.sourceEventId}`, event); - } - return index; -} - -function normalizeWorkbenchSessionInputFact(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); - if (!sessionId) throw requiredWorkbenchFactError("workbench session input fact", ["sessionId"]); - const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); - const turnId = textOr(input.turnId ?? requestMeta.turnId ?? input.traceId ?? requestMeta.traceId, "") || null; - const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; - const messageId = textOr(input.messageId ?? requestMeta.messageId, "") || null; - const commandId = textOr(input.commandId ?? input.sourceCommandId, "") || null; - const delivery = workbenchInputDelivery(input.delivery); - const status = workbenchInputStatus(input.status); - const sourceEventId = textOr(input.sourceEventId ?? input.eventId, "") || `${sessionId}:${turnId ?? "turn-unassigned"}:${delivery}:${traceId ?? commandId ?? messageId ?? "input"}`; - const inputId = textOr(input.inputId ?? input.id, "") || stableWorkbenchFactId("wsi", { sessionId, turnId, traceId, messageId, commandId, delivery, sourceEventId }); - const admittedSeq = nonNegativeInteger(input.admittedSeq ?? input.aggregateSeq ?? input.sourceSeq); - const promotedSeq = workbenchSessionInputPromotedSeq({ ...input, status }, admittedSeq); - return pruneUndefined({ - ...input, - id: inputId, - inputId, - sessionId, - turnId, - traceId, - messageId, - commandId, - delivery, - admittedSeq, - promotedSeq, - sourceSeq: admittedSeq, - projectedSeq: admittedSeq, - status, - errorCode: textOr(input.errorCode ?? input.error?.code, "") || null, - sourceEventId, - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false, - valuesRedacted: true - }); -} - -function memoryWorkbenchSessionInputWithSeq(fact, records) { - const existingFactSeq = nonNegativeInteger(fact.admittedSeq); - if (existingFactSeq > 0) return workbenchSessionInputWithSeq(fact, existingFactSeq); - const existing = records.get(fact.inputId) ?? null; - const existingSeq = nonNegativeInteger(existing?.admittedSeq); - if (existingSeq > 0) return workbenchSessionInputWithSeq(fact, existingSeq); - const nextSeq = [...records.values()] - .filter((record) => record.sessionId === fact.sessionId) - .reduce((max, record) => Math.max(max, nonNegativeInteger(record.admittedSeq)), 0) + 1; - return workbenchSessionInputWithSeq(fact, nextSeq); -} - -function inputFactWithAggregateSeq(fact, eventIndex) { - const existingFactSeq = nonNegativeInteger(fact.admittedSeq); - if (existingFactSeq > 0) return workbenchSessionInputWithSeq(fact, existingFactSeq); - const event = eventIndex.get(`input:${fact.inputId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; - const admittedSeq = nonNegativeInteger(event?.aggregateSeq); - return admittedSeq > 0 ? workbenchSessionInputWithSeq(fact, admittedSeq) : fact; -} - -function workbenchSessionInputWithSeq(fact, admittedSeq) { - return { - ...fact, - admittedSeq, - promotedSeq: workbenchSessionInputPromotedSeq(fact, admittedSeq), - sourceSeq: admittedSeq, - projectedSeq: admittedSeq - }; -} - -function workbenchSessionInputPromotedSeq(fact, admittedSeq) { - const existing = nonNegativeIntegerOrNull(fact.promotedSeq); - if (existing !== null && existing > 0) return existing; - const seq = nonNegativeInteger(admittedSeq); - return textOr(fact.status, "") === "promoted" && seq > 0 ? seq : existing; -} - -function normalizeWorkbenchSessionFact(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const sessionId = textOr(input.sessionId ?? input.id ?? requestMeta.sessionId, ""); - if (!sessionId) throw requiredWorkbenchFactError("workbench session fact", ["sessionId"]); - const sourceSeq = nonNegativeInteger(input.sourceSeq); - const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); - const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); - return pruneUndefined({ - ...input, - id: sessionId, - sessionId, - ownerUserId: textOr(input.ownerUserId ?? requestMeta.ownerUserId, "") || null, - projectId: textOr(input.projectId ?? requestMeta.projectId, "") || null, - conversationId: textOr(input.conversationId ?? requestMeta.conversationId, "") || null, - threadId: textOr(input.threadId ?? requestMeta.threadId, "") || null, - status: textOr(input.status, "unknown"), - lastTraceId: textOr(input.lastTraceId ?? input.traceId ?? requestMeta.traceId, "") || null, - projectedSeq, - sourceSeq, - sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, - terminal: Boolean(input.terminal), - sealed: Boolean(input.sealed), - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false - }); -} - -function normalizeWorkbenchMessageFact(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); - if (!sessionId) throw requiredWorkbenchFactError("workbench message fact", ["sessionId"]); - const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; - const turnId = textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null; - const role = textOr(input.role, "agent"); - const sourceSeq = nonNegativeInteger(input.sourceSeq); - const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); - const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); - const messageId = textOr(input.messageId ?? input.id, "") || stableWorkbenchFactId("msg", { sessionId, turnId, traceId, role, sourceSeq, projectedSeq }); - return pruneUndefined({ - ...input, - id: messageId, - messageId, - sessionId, - turnId, - traceId, - role, - status: textOr(input.status, "unknown"), - projectedSeq, - sourceSeq, - sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, - terminal: Boolean(input.terminal), - sealed: Boolean(input.sealed), - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false - }); -} - -function normalizeWorkbenchPartFact(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); - const messageId = textOr(input.messageId ?? requestMeta.messageId, ""); - if (!sessionId || !messageId) throw requiredWorkbenchFactError("workbench part fact", ["sessionId", "messageId"]); - const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; - const turnId = textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null; - const partIndex = nonNegativeInteger(input.partIndex ?? input.index); - const sourceSeq = nonNegativeInteger(input.sourceSeq); - const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); - const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); - const partType = textOr(input.partType ?? input.type, "text"); - const partId = textOr(input.partId ?? input.id, "") || stableWorkbenchFactId("wpt", { sessionId, messageId, turnId, traceId, partIndex, partType }); - return pruneUndefined({ - ...input, - id: partId, - partId, - messageId, - sessionId, - turnId, - traceId, - partIndex, - partType, - status: textOr(input.status, "unknown"), - projectedSeq, - sourceSeq, - sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, - terminal: Boolean(input.terminal), - sealed: Boolean(input.sealed), - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false - }); -} - -function normalizeWorkbenchTurnFact(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, ""); - const traceId = textOr(input.traceId ?? requestMeta.traceId, "") || null; - const turnId = textOr(input.turnId ?? input.id ?? requestMeta.turnId ?? traceId, ""); - if (!sessionId || !turnId) throw requiredWorkbenchFactError("workbench turn fact", ["sessionId", "turnId"]); - const sourceSeq = nonNegativeInteger(input.sourceSeq); - const projectedSeq = nonNegativeInteger(input.projectedSeq ?? sourceSeq); - const timestamp = timestampOr(input.updatedAt ?? input.createdAt, now); - const status = textOr(input.status, "unknown"); - const terminal = Boolean(input.terminal); - const sealed = Boolean(input.sealed); - const finalResponse = input.finalResponse ?? null; - assertWorkbenchTurnFinalResponseInvariant({ status, terminal, sealed, finalResponse }); - return pruneUndefined({ - ...input, - id: turnId, - turnId, - sessionId, - traceId, - messageId: textOr(input.messageId ?? requestMeta.messageId, "") || null, - status, - projectedSeq, - sourceSeq, - sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, - terminal, - sealed, - finalResponse, - diagnostic: normalizeJsonObject(input.diagnostic ?? input.projection ?? input.blocker), - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false - }); -} - -function assertWorkbenchTurnFinalResponseInvariant({ status, terminal, sealed, finalResponse } = {}) { - if (normalizeWorkbenchFactStatus(status) !== "completed") return; - if (!terminal && !sealed) return; - if (workbenchFactFinalResponseText(finalResponse)) return; - const error = new Error("completed Workbench turn facts must include an authoritative final response before sealing"); - error.code = "workbench_terminal_final_response_missing"; - error.layer = "workbench-read-model"; - error.category = "projection-invariant"; - throw error; -} - -function normalizeWorkbenchFactStatus(value) { - return textOr(value, "").toLowerCase().replace(/_/gu, "-"); -} - -function workbenchFactFinalResponseText(value) { - if (value && typeof value === "object" && !Array.isArray(value)) { - return workbenchFactFinalResponseText(value.text ?? value.content ?? value.message ?? value.summary ?? value.preview); - } - const text = textOr(value, ""); - return text && text !== "[object Object]" ? text : null; -} - -function normalizeWorkbenchTraceEventFact(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); - if (!traceId) throw requiredWorkbenchFactError("workbench trace event fact", ["traceId"]); - const sessionId = textOr(input.sessionId ?? requestMeta.sessionId, "") || null; - const turnId = textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null; - const sourceSeq = nonNegativeInteger(input.sourceSeq); - const projectedSeq = nonNegativeInteger(input.projectedSeq ?? input.seq); - const occurredAt = timestampOr(input.occurredAt ?? input.createdAt ?? input.timestamp, now); - const eventType = textOr(input.eventType ?? input.type ?? input.kind, "event"); - const id = textOr(input.id ?? input.eventId, "") || stableWorkbenchFactId("wte", { traceId, sourceSeq, projectedSeq, eventType, sourceEventId: input.sourceEventId ?? null, occurredAt }); - return pruneUndefined({ - ...input, - id, - traceId, - sessionId, - turnId, - messageId: textOr(input.messageId ?? requestMeta.messageId, "") || null, - sourceSeq, - sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, - projectedSeq, - eventType, - terminal: Boolean(input.terminal), - sealed: Boolean(input.sealed), - occurredAt, - updatedAt: timestampOr(input.updatedAt, occurredAt), - valuesPrinted: false - }); -} - -function normalizeWorkbenchProjectionCheckpoint(value, requestMeta = {}, now) { - const input = normalizeJsonObject(value); - const traceId = textOr(input.traceId ?? requestMeta.traceId, ""); - if (!traceId) throw requiredWorkbenchFactError("workbench projection checkpoint", ["traceId"]); - const sourceSeq = nonNegativeInteger(input.sourceSeq ?? input.lastSourceSeq ?? input.lastAgentRunSeq); - const projectedSeq = nonNegativeInteger(input.projectedSeq ?? input.lastProjectedSeq ?? sourceSeq); - const timestamp = timestampOr(input.updatedAt ?? input.lastProjectedAt, now); - return pruneUndefined({ - ...input, - id: traceId, - traceId, - sessionId: textOr(input.sessionId ?? requestMeta.sessionId, "") || null, - turnId: textOr(input.turnId ?? requestMeta.turnId ?? traceId, "") || null, - runId: textOr(input.runId ?? input.sourceRunId, "") || null, - commandId: textOr(input.commandId ?? input.sourceCommandId, "") || null, - projectedSeq, - sourceSeq, - sourceEventId: textOr(input.sourceEventId ?? input.eventId, "") || null, - projectionStatus: projectionStatusOr(input.projectionStatus, "projecting"), - projectionHealth: projectionHealthOr(input.projectionHealth, "healthy"), - terminal: Boolean(input.terminal), - sealed: Boolean(input.sealed), - diagnostic: normalizeJsonObject(input.diagnostic ?? input.blocker), - createdAt: timestampOr(input.createdAt, timestamp), - updatedAt: timestamp, - valuesPrinted: false - }); -} - -function arrayInput(value) { - if (Array.isArray(value)) return value; - if (value && typeof value === "object") return [value]; - return []; -} - -function stableWorkbenchFactId(prefix, value) { - return `${prefix}_${sha256Hex(value).slice(0, 32)}`; -} - -function requiredWorkbenchFactError(label, fields) { - return new HwlabProtocolError(`${label} requires ${fields.join(", ")}`, { - code: ERROR_CODES.invalidParams, - data: { required: fields } - }); -} - -function compareTraceEventRecords(left, right) { - const leftSeq = positiveIntegerOrNull(left?.event?.seq); - const rightSeq = positiveIntegerOrNull(right?.event?.seq); - if (leftSeq && rightSeq && leftSeq !== rightSeq) return leftSeq - rightSeq; - return String(left?.occurredAt ?? "").localeCompare(String(right?.occurredAt ?? "")) || - String(left?.id ?? "").localeCompare(String(right?.id ?? "")); -} - -function compareProjectionStateRecords(left, right) { - return String(left?.updatedAt ?? "").localeCompare(String(right?.updatedAt ?? "")) || - String(left?.traceId ?? "").localeCompare(String(right?.traceId ?? "")); -} - -function compareWorkbenchFactRecords(left, right) { - return String(left?.updatedAt ?? left?.createdAt ?? "").localeCompare(String(right?.updatedAt ?? right?.createdAt ?? "")) || - String(left?.id ?? left?.sessionId ?? left?.messageId ?? left?.turnId ?? left?.traceId ?? "").localeCompare(String(right?.id ?? right?.sessionId ?? right?.messageId ?? right?.turnId ?? right?.traceId ?? "")); -} - -function compareWorkbenchInputFactRecords(left, right) { - return nonNegativeInteger(left?.admittedSeq) - nonNegativeInteger(right?.admittedSeq) || - String(left?.inputId ?? left?.id ?? "").localeCompare(String(right?.inputId ?? right?.id ?? "")); -} - -function compareWorkbenchPartFactRecords(left, right) { - return String(left?.messageId ?? "").localeCompare(String(right?.messageId ?? "")) || - nonNegativeInteger(left?.partIndex) - nonNegativeInteger(right?.partIndex) || - compareWorkbenchFactRecords(left, right); -} - -function compareWorkbenchTraceEventFacts(left, right) { - return nonNegativeInteger(left?.projectedSeq) - nonNegativeInteger(right?.projectedSeq); -} - -function sortWorkbenchFactRows(records, params = {}, family, fallbackCompare) { - const sorted = [...records]; - if (workbenchFactOrder(params, family) === "updated_desc") { - return sorted.sort((left, right) => String(right?.updatedAt ?? right?.createdAt ?? "").localeCompare(String(left?.updatedAt ?? left?.createdAt ?? "")) || fallbackCompare(left, right)); - } - if (family === "inputs") return sorted.sort(compareWorkbenchInputFactRecords); - return sorted.sort(fallbackCompare); -} - -const WORKBENCH_FACT_FAMILIES = Object.freeze(["inputs", "sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"]); - -function workbenchFactFamilySet(params = {}) { - const raw = params.families ?? params.factFamilies; - if (raw === undefined || raw === null || raw === "") return new Set(WORKBENCH_FACT_FAMILIES); - const values = Array.isArray(raw) ? raw : String(raw).split(","); - const selected = values.map((value) => String(value ?? "").trim()).filter((value) => WORKBENCH_FACT_FAMILIES.includes(value)); - return new Set(selected.length > 0 ? selected : WORKBENCH_FACT_FAMILIES); -} - -function workbenchFactFamilyForTable(table) { - if (table === "workbench_session_inputs") return "inputs"; - if (table === "workbench_sessions") return "sessions"; - if (table === "workbench_messages") return "messages"; - if (table === "workbench_parts") return "parts"; - if (table === "workbench_turns") return "turns"; - if (table === "workbench_trace_events") return "traceEvents"; - if (table === "workbench_projection_checkpoints") return "checkpoints"; - return "unknown"; -} - -function workbenchSessionSummarySelectClause() { - return [ - "session_id", - "owner_user_id", - "project_id", - "conversation_id", - "thread_id", - "status", - "last_trace_id", - "projected_seq", - "source_seq", - "source_event_id", - "terminal", - "sealed", - "created_at", - "updated_at", - "COALESCE(NULLIF(session_json::jsonb ->> 'providerProfile', ''), NULLIF(session_json::jsonb #>> '{sessionJson,providerProfile}', '')) AS provider_profile" - ].join(", "); -} - -function workbenchSessionSummaryFactFromRow(row = {}) { - const sessionId = textOr(row.session_id, ""); - if (!sessionId) return null; - const providerProfile = textOr(row.provider_profile, ""); - return { - id: sessionId, - sessionId, - ownerUserId: textOr(row.owner_user_id, null), - projectId: textOr(row.project_id, null), - conversationId: textOr(row.conversation_id, null), - threadId: textOr(row.thread_id, null), - status: textOr(row.status, "unknown"), - lastTraceId: textOr(row.last_trace_id, null), - projectedSeq: nonNegativeInteger(row.projected_seq), - sourceSeq: nonNegativeInteger(row.source_seq), - sourceEventId: textOr(row.source_event_id, null), - terminal: row.terminal === true || row.terminal === "t", - sealed: row.sealed === true || row.sealed === "t", - ...(providerProfile ? { - providerProfile, - sessionJson: { - providerProfile, - valuesRedacted: true, - secretMaterialStored: false - } - } : {}), - createdAt: textOr(row.created_at, null), - updatedAt: textOr(row.updated_at, row.created_at ?? null), - valuesPrinted: false, - valuesRedacted: true - }; -} - -function workbenchFactOrder(params = {}, family = "") { - const value = params[`${family}Order`] ?? params.order; - return value === "updated_desc" ? "updated_desc" : "updated_asc"; -} - -function traceEventLevel(value) { - const text = textOr(value, "info").toLowerCase(); - if (["failed", "failure", "error", "timeout"].includes(text)) return "error"; - if (["warn", "warning", "blocked", "canceled", "cancelled"].includes(text)) return "warn"; - if (["debug", "trace"].includes(text)) return "debug"; - return "info"; -} - -function positiveIntegerOrNull(value) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed > 0 ? parsed : null; -} - -function nonNegativeInteger(value) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; -} - -function nonNegativeIntegerOrNull(value) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed >= 0 ? parsed : null; -} - -function projectionStatusOr(value, fallback) { - const text = textOr(value, fallback).toLowerCase(); - return ["projecting", "caught_up", "terminal", "degraded", "blocked", "stalled"].includes(text) ? text : fallback; -} - -function projectionHealthOr(value, fallback) { - const text = textOr(value, fallback).toLowerCase(); - return ["healthy", "degraded", "unavailable", "stalled"].includes(text) ? text : fallback; -} - -function resultSyncStateOr(value, fallback) { - const text = textOr(value, fallback).toLowerCase(); - return ["not_started", "pending", "synced", "failed", "timed_out"].includes(text) ? text : fallback; -} - -function timestampOr(value, fallback) { - const text = textOr(value, ""); - if (text) { - const ms = Date.parse(text); - if (Number.isFinite(ms)) return new Date(ms).toISOString(); - } - return textOr(fallback, "") || new Date().toISOString(); -} - -function textOr(value, fallback) { - const text = String(value ?? "").trim(); - return text || fallback; -} - -function textList(value) { - const raw = Array.isArray(value) ? value : []; - return [...new Set(raw.map((item) => textOr(item, "")).filter(Boolean))]; -} - -function asObject(value, label) { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new HwlabProtocolError(`${label} must be a JSON object`, { - code: ERROR_CODES.invalidParams - }); - } - return value; -} - -function requireString(input, field) { - if (typeof input[field] !== "string" || input[field].trim() === "") { - throw new HwlabProtocolError(`${field} is required`, { - code: ERROR_CODES.invalidParams, - data: { field } - }); - } - return input[field]; -} - -function requireProtocolId(input, field) { - const value = requireString(input, field); - return value; -} - -function normalizeJsonObject(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return {}; - } - return { ...value }; -} - -function pruneUndefined(input) { - return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)); -} - -function makeId(prefix) { - return `${prefix}_${randomUUID()}`; -} - -function stableJson(value) { - return JSON.stringify(sortJson(value)); -} - -function sha256Hex(value) { - return createHash("sha256").update(stableJson(value)).digest("hex"); -} - -function sortJson(value) { - if (Array.isArray(value)) { - return value.map(sortJson); - } - if (value && typeof value === "object") { - return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sortJson(value[key])])); - } - return value; -} - -function matchesQuery(record, params, fields) { - for (const field of fields) { - if (params[field] !== undefined && params[field] !== null && record[field] !== params[field]) { - return false; - } - const values = textList(params[`${field}s`]); - if (values.length > 0 && !values.includes(textOr(record[field], ""))) { - return false; - } - } - return true; -} - -function matchesWorkbenchSessionFactQuery(record, params) { - if (!matchesQuery(record, params, ["sessionId", "ownerUserId", "projectId", "conversationId", "threadId", "status"])) return false; - const traceId = textOr(params.traceId ?? params.lastTraceId, ""); - if (traceId && record.lastTraceId !== traceId && record.traceId !== traceId) return false; - return true; -} - -function limitResults(records, limit) { - const parsed = Number.parseInt(limit ?? "", 10); - if (!Number.isInteger(parsed) || parsed <= 0) { - return records; - } - return records.slice(0, parsed); -} - -function normalizeRuntimeAdapter(value) { - if (typeof value !== "string" || value.trim() === "") { - return RUNTIME_STORE_KIND; - } - const normalized = value.trim().toLowerCase(); - if (["postgres", "postgresql", "pg", "db"].includes(normalized)) { - return RUNTIME_STORE_KIND_POSTGRES; - } - return RUNTIME_STORE_KIND; -} - -function normalizePostgresSslMode(value) { - const normalized = String(value ?? "").trim().toLowerCase(); - if (["disable", "false", "0", "off", "no"].includes(normalized)) { - return "disable"; - } - return "require"; -} - -function postgresSslModeFromConnectionString(dbUrl) { - if (typeof dbUrl !== "string" || dbUrl.trim() === "") return undefined; - try { - const url = new URL(dbUrl); - if (!["postgres:", "postgresql:"].includes(url.protocol)) return undefined; - return url.searchParams.get("sslmode") ?? url.searchParams.get("ssl") ?? undefined; - } catch { - return undefined; - } -} - -function postgresSslOption(sslMode) { - return sslMode === "disable" ? false : { rejectUnauthorized: false }; -} - -function normalizePostgresConnectionStringForSslMode(dbUrl, sslMode) { - if (typeof dbUrl !== "string" || dbUrl.trim() === "") { - return dbUrl; - } - - let url; - try { - url = new URL(dbUrl); - } catch { - return dbUrl; - } - - if (!["postgres:", "postgresql:"].includes(url.protocol)) { - return dbUrl; - } - - for (const key of ["ssl", "sslmode", "sslcert", "sslkey", "sslrootcert"]) { - url.searchParams.delete(key); - } - if (sslMode !== "disable") { - url.searchParams.set("sslmode", "require"); - url.searchParams.set("uselibpqcompat", "true"); - } - return url.toString(); -} - -function postgresAdapterContract() { - return { - required: "Postgres-backed durable runtime adapter using HWLAB_CLOUD_DB_URL", - adapterEnv: RUNTIME_ADAPTER_ENV, - adapterEnvValue: RUNTIME_STORE_KIND_POSTGRES, - sourceIssue: "pikasTech/HWLAB#164", - secretMaterialRequiredInSource: false, - fixtureEvidenceAllowed: false - }; -} - -function addRuntimeDurabilityContract(summary) { - return { - ...summary, - durabilityContract: { - ready: summary.durable === true && summary.ready === true && summary.liveRuntimeEvidence === true, - status: summary.durable === true && summary.ready === true && summary.liveRuntimeEvidence === true ? "ready" : "blocked", - requiredEvidence: RUNTIME_DURABILITY_REQUIRED_EVIDENCE, - dbLiveEvidenceIsDurabilityEvidence: false, - liveRuntimeEvidence: Boolean(summary.liveRuntimeEvidence), - adapterQueryRequired: true, - blockedLayer: summary.durable === true && summary.ready === true && summary.liveRuntimeEvidence === true - ? null - : durabilityBlockedLayer(summary), - blocker: summary.blocker ?? null, - secretMaterialRead: false - } - }; -} - -function durabilityBlockedLayer(summary = {}) { - const blocker = summary.blocker; - if ( - blocker === RUNTIME_DURABLE_ADAPTER_MISSING || - blocker === RUNTIME_DURABLE_ADAPTER_UNCONFIGURED || - blocker === RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING - ) { - return "adapter"; - } - if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) { - return "ssl"; - } - if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) { - return "auth"; - } - if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) { - return "schema"; - } - if (blocker === RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED) { - return "migration"; - } - if (summary.connection?.queryAttempted || blocker === RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED) { - return "durability_query"; - } - return "adapter"; -} - -function runtimeSafety() { - return { - devOnly: true, - secretsRead: false, - secretMaterialRead: false, - valuesRedacted: true, - endpointRedacted: true - }; -} - -export function classifyRuntimeDbError(error) { - const code = typeof error?.code === "string" ? error.code : "UNKNOWN"; - if (isRuntimeDbConnectTimeout(error)) { - return { - blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, - reason: "Postgres runtime adapter connection attempt timed out before completing the runtime query", - retryable: true, - transient: true, - retryAfterMs: 5000, - connection: { - queryAttempted: true, - queryResult: "connect_timeout", - errorCode: code - } - }; - } - if (code === "HWLAB_PG_DRIVER_MISSING") { - return { - blocker: RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, - reason: "Postgres runtime adapter is selected, but the pg package is not installed in the runtime image", - connection: { - queryAttempted: false, - queryResult: "driver_missing", - errorCode: code - } - }; - } - if (isRuntimeDbSslError(error)) { - return { - blocker: RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, - reason: "Postgres runtime adapter reached the DB driver but SSL negotiation or SSL mode compatibility blocked the schema query", - connection: { - queryAttempted: true, - queryResult: "ssl_negotiation_blocked", - errorCode: code - } - }; - } - if (["28P01", "28000", "42501"].includes(code)) { - return { - blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, - reason: "Postgres runtime adapter reached the DB driver but authentication or authorization blocked the schema query", - connection: { - queryAttempted: true, - queryResult: "auth_blocked", - errorCode: code - } - }; - } - if (["42P01", "42703", "3F000"].includes(code)) { - return { - blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, - reason: "Postgres runtime adapter reached the database, but the runtime schema is missing or incompatible", - connection: { - queryAttempted: true, - queryResult: "schema_blocked", - errorCode: code - } - }; - } - if (code === "53300") { - return { - blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, - reason: "Postgres runtime adapter reached the database, but the server has exhausted available connection slots", - retryable: true, - transient: true, - retryAfterMs: 2000, - connection: { - queryAttempted: true, - queryResult: "too_many_connections", - errorCode: code - } - }; - } - return { - blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, - reason: "Postgres runtime adapter could not complete a live runtime schema query", - connection: { - queryAttempted: true, - queryResult: "query_blocked", - errorCode: code - } - }; -} - -function isRuntimeDbConnectTimeout(error) { - const message = String(error?.message ?? error ?? "").toLowerCase(); - const pgConnectTimeout = message.includes("timeout exceeded when trying to connect"); - const pgConnectionTimeoutTermination = message.includes("connection terminated due to connection timeout"); - if (!pgConnectTimeout && !pgConnectionTimeoutTermination) return false; - if (pgConnectionTimeoutTermination) return true; - const stack = String(error?.stack ?? "").toLowerCase(); - return !stack || stack.includes("pg-pool") || stack.includes("node_modules/pg"); -} - -function durableRuntimeReadBlockedError(method, readiness = {}) { - return new HwlabProtocolError("Postgres durable runtime adapter is blocked for runtime evidence queries", { - code: ERROR_CODES.internalError, - data: durableRuntimeReadBlockedData(method, readiness), - context: { - result: "failed" - } - }); -} - -function durableRuntimeReadBlockedData(method, readiness = {}) { - const durability = readiness.durabilityContract ?? {}; - const connection = readiness.connection ?? {}; - const blockedLayer = durability.blockedLayer ?? durabilityBlockedLayer(readiness); - const queryResult = connection.queryResult ?? (blockedLayer === "durability_query" ? "query_blocked" : "not_ready"); - - return pruneUndefined({ - method, - adapter: readiness.adapter ?? RUNTIME_STORE_KIND_POSTGRES, - status: "blocked", - blocker: readiness.blocker ?? RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, - durable: false, - durableRequested: true, - liveRuntimeEvidence: false, - queryResult, - blockedLayer, - requiredEvidence: durability.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE, - dbLiveEvidenceIsDurabilityEvidence: false, - retryable: readiness.retryable === true || connection.queryResult === "connect_timeout", - transient: readiness.transient === true || connection.queryResult === "connect_timeout", - retryAfterMs: Number.isFinite(Number(readiness.retryAfterMs)) ? Math.max(0, Math.trunc(Number(readiness.retryAfterMs))) : undefined, - secretMaterialRead: false, - valueRedacted: true, - endpointRedacted: true, - errorCode: connection.errorCode ?? undefined, - redaction: { - valueRedacted: true, - valuesRedacted: true, - endpointRedacted: true, - secretMaterialRead: false - } - }); -} - -function isRuntimeDbSslError(error) { - const code = typeof error?.code === "string" ? error.code : ""; - if (code.startsWith("ERR_SSL") || code.startsWith("ERR_TLS")) { - return true; - } - if ([ - "DEPTH_ZERO_SELF_SIGNED_CERT", - "SELF_SIGNED_CERT_IN_CHAIN", - "UNABLE_TO_VERIFY_LEAF_SIGNATURE", - "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", - "CERT_HAS_EXPIRED" - ].includes(code)) { - return true; - } - const message = typeof error?.message === "string" ? error.message.toLowerCase() : ""; - return [ - "does not support ssl", - "ssl is not enabled", - "ssl negotiation", - "ssl off", - "ssl required", - "requires ssl", - "no ssl encryption", - "no encryption", - "hostssl", - "server requires ssl", - "before secure tls connection", - "tls handshake" - ].some((pattern) => message.includes(pattern)); -} - -function summarizeRuntimeSchema(rows) { - const columnsByTable = new Map(); - for (const row of rows) { - const table = row.table_name; - const column = row.column_name; - if (!table || !column) continue; - if (!columnsByTable.has(table)) { - columnsByTable.set(table, new Set()); - } - columnsByTable.get(table).add(column); - } - - const missingTables = []; - const missingColumns = []; - for (const [table, columns] of Object.entries(requiredPostgresSchema)) { - const observed = columnsByTable.get(table); - if (!observed) { - missingTables.push(table); - missingColumns.push(...columns.map((column) => `${table}.${column}`)); - continue; - } - for (const column of columns) { - if (!observed.has(column)) { - missingColumns.push(`${table}.${column}`); - } - } - } - - return { - ready: missingTables.length === 0 && missingColumns.length === 0, - checked: true, - requiredTables: Object.keys(requiredPostgresSchema), - missingTables, - missingColumns - }; -} - -function notCheckedRuntimeMigration() { - return { - checked: false, - ready: false, - table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: CLOUD_CORE_MIGRATION_ID, - requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, - appliedMigrationId: null, - appliedSchemaVersion: null, - missing: true - }; -} - -function blockedRuntimeMigration({ errorCode = null } = {}) { - return { - ...notCheckedRuntimeMigration(), - checked: true, - errorCode - }; -} - -function runtimeGates({ - ssl = notCheckedGate(), - auth = notCheckedGate(), - schema = notCheckedGate(), - migration = notCheckedGate(), - durability = notCheckedGate() -} = {}) { - return { - ssl, - auth, - schema, - migration, - durability - }; -} - -function defaultRuntimeGates({ blocker, connection, schema, migration } = {}) { - if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) { - return { - ssl: blockedGate({ checked: true, blocker }), - auth: notCheckedGate() - }; - } - if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) { - return { - ssl: readyGate(), - auth: blockedGate({ checked: true, blocker }) - }; - } - if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) { - return { - ssl: connection?.queryAttempted ? readyGate() : notCheckedGate(), - auth: connection?.queryAttempted ? readyGate() : notCheckedGate() - }; - } - if (blocker === RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED && schema?.checked && migration?.checked) { - return { - ssl: readyGate(), - auth: readyGate() - }; - } - return { - ssl: notCheckedGate(), - auth: notCheckedGate() - }; -} - -function defaultSchemaGate({ blocker, schema } = {}) { - if (schema?.checked) { - return gateFromReadiness(schema.ready, { blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }); - } - if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) { - return blockedGate({ checked: true, blocker }); - } - return notCheckedGate(); -} - -function defaultMigrationGate({ blocker, migration } = {}) { - if (migration?.checked) { - return gateFromReadiness(migration.ready, { blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }); - } - if (blocker === RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED) { - return blockedGate({ checked: true, blocker }); - } - return notCheckedGate(); -} - -function readyGate() { - return { - checked: true, - ready: true, - status: "ready", - blocker: null - }; -} - -function blockedGate({ checked = true, blocker = "runtime_durable_adapter_blocked" } = {}) { - return { - checked, - ready: false, - status: checked ? "blocked" : "not_checked", - blocker - }; -} - -function notCheckedGate() { - return { - checked: false, - ready: false, - status: "not_checked", - blocker: null - }; -} - -function gateFromReadiness(ready, { blocker } = {}) { - return ready ? readyGate() : blockedGate({ checked: true, blocker }); -} - -function snapshotMemory(memory) { - return { - gatewaySessions: new Map(memory.gatewaySessions), - boxResources: new Map(memory.boxResources), - boxCapabilities: new Map(memory.boxCapabilities), - hardwareOperations: new Map(memory.hardwareOperations), - auditEvents: new Map(memory.auditEvents), - evidenceRecords: new Map(memory.evidenceRecords), - agentTraceEvents: new Map(memory.agentTraceEvents), - workbenchProjectionStates: new Map(memory.workbenchProjectionStates), - workbenchSessions: new Map(memory.workbenchSessions), - workbenchMessages: new Map(memory.workbenchMessages), - workbenchParts: new Map(memory.workbenchParts), - workbenchTurns: new Map(memory.workbenchTurns), - workbenchTraceEvents: new Map(memory.workbenchTraceEvents), - workbenchProjectionCheckpoints: new Map(memory.workbenchProjectionCheckpoints) - }; -} - -function newRecords(current, before) { - return [...current.entries()] - .filter(([id]) => !before.has(id)) - .map(([, record]) => record); -} - -function changedRecords(current, before) { - return [...current.entries()] - .filter(([id, record]) => stableJson(before.get(id)) !== stableJson(record)) - .map(([, record]) => record); -} - -function withPersistence(result, persistence) { - return { - ...result, - persistence - }; -} - -function parseJsonColumn(value, fallback) { - if (!value) return fallback; - if (typeof value === "object") return value; - try { - return JSON.parse(String(value)); - } catch { - return fallback; - } -} - -function postgresPoolStats(pool) { - if (!pool || typeof pool !== "object") return null; - return { - totalCount: Number.isFinite(Number(pool.totalCount)) ? Number(pool.totalCount) : null, - idleCount: Number.isFinite(Number(pool.idleCount)) ? Number(pool.idleCount) : null, - waitingCount: Number.isFinite(Number(pool.waitingCount)) ? Number(pool.waitingCount) : null, - valuesRedacted: true - }; -} - -function toCountKey(table) { - if (table === "users") return "users"; - if (table === "user_sessions") return "userSessions"; - if (table === "gateway_sessions") return "gatewaySessions"; - if (table === "box_resources") return "boxResources"; - if (table === "box_capabilities") return "boxCapabilities"; - if (table === "hardware_operations") return "hardwareOperations"; - if (table === "audit_events") return "auditEvents"; - if (table === "evidence_records") return "evidenceRecords"; - if (table === "agent_sessions") return "agentSessions"; - if (table === "account_workspaces") return "accountWorkspaces"; - if (table === "worker_sessions") return "workerSessions"; - if (table === "agent_trace_events") return "agentTraceEvents"; - if (table === "workbench_projection_state") return "workbenchProjectionStates"; - if (table === "workbench_sessions") return "workbenchSessions"; - if (table === "workbench_messages") return "workbenchMessages"; - if (table === "workbench_parts") return "workbenchParts"; - if (table === "workbench_turns") return "workbenchTurns"; - if (table === "workbench_trace_events") return "workbenchTraceEvents"; - if (table === "workbench_projection_checkpoints") return "workbenchProjectionCheckpoints"; - return table; -} - -function normalizeRuntimeTimeoutMs(value) { - const parsed = Number.parseInt(String(value ?? ""), 10); - if (!Number.isInteger(parsed) || parsed <= 0) { - return 1200; - } - return Math.min(parsed, 5000); -} - -function normalizeRuntimePoolMax(value) { - if (value === undefined || value === null || String(value).trim() === "") return null; - const parsed = Number.parseInt(String(value), 10); - if (!Number.isInteger(parsed) || parsed <= 0 || String(parsed) !== String(value).trim()) { - const error = new Error(`${RUNTIME_DB_POOL_MAX_ENV} must be a positive integer`); - error.code = "HWLAB_CLOUD_DB_POOL_MAX_INVALID"; - throw error; - } - return parsed; -}