/* * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; draft-2026-06-20-p1-zero-split-durable-realtime; PJ2026-010403 API契约 draft-2026-06-18-r1. * 职责: Workbench durable facts store adapter。统一读取 session/message/turn/trace projection facts,不在 GET 中推进 AgentRun facts。 */ import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; import { safeConversationId, safeSessionId, safeTraceId } from "./server-http-utils.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { durableTraceStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts"; const TERMINAL_DURABLE_TRACE_CACHE_MAX = 256; const DEFAULT_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS = 5; const DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS = 80; const DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS = 1000; const terminalDurableTraceCache = new Map(); export function createWorkbenchFactsStore(options = {}, actor = null) { const accessStore = options.accessController?.store ?? options.accessController ?? null; const traceSessionCache = options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : null; const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger }); const results = options.codeAgentChatResults ?? null; async function queryFacts(params = {}) { if (typeof runtimeReader?.queryWorkbenchFacts === "function") { const retryPolicy = workbenchFactsQueryRetryPolicy(options.env ?? process.env); const retryDelaysMs = []; for (let attempt = 1; attempt <= retryPolicy.maxAttempts; attempt += 1) { try { const result = await runtimeReader.queryWorkbenchFacts(params); return { facts: normalizeWorkbenchFactsResult(result?.facts), count: Number.isFinite(Number(result?.count)) ? Number(result.count) : 0, durable: true, persistence: result?.persistence ?? null, cache: result?.cache ?? null, retry: attempt > 1 ? { attempts: attempt, maxAttempts: retryPolicy.maxAttempts, delaysMs: retryDelaysMs.slice(), valuesRedacted: true } : null }; } catch (error) { const retryable = isRetryableWorkbenchFactsQueryError(error); const finalAttempt = !retryable || attempt >= retryPolicy.maxAttempts; const retryAfterMs = finalAttempt ? null : workbenchFactsRetryDelayMs(retryPolicy, attempt); const annotatedError = annotateWorkbenchFactsQueryError(error, { retryable, retryAttempt: attempt, retryMax: retryPolicy.maxAttempts, retryAfterMs, retryDelaysMs, retryExhausted: finalAttempt && retryable }); if (finalAttempt) { emitWorkbenchFactsQueryFailureOtel(params, annotatedError); logWorkbenchFactsQueryFailure(options.logger ?? console, params, annotatedError); return { facts: emptyWorkbenchFacts(), count: 0, durable: true, error: annotatedError, projection: projectionStoreUnavailableTrace(safeTraceId(params.traceId) ?? "trc_unassigned", annotatedError) }; } logWorkbenchFactsQueryRetry(options.logger ?? console, params, annotatedError, { attempt, maxAttempts: retryPolicy.maxAttempts, retryAfterMs }); retryDelaysMs.push(retryAfterMs); await sleepMs(retryAfterMs); } } } return { facts: emptyWorkbenchFacts(), count: 0, durable: false, persistence: null }; } async function getSessionById(sessionId) { const safeId = safeSessionId(sessionId); if (!safeId) return null; const session = await accessStore?.getAgentSession?.(safeId) ?? null; return canActorReadSession(session, actor) ? session : null; } async function getSessionByRouteId(routeId) { const sessionId = safeSessionId(routeId); if (sessionId) return getSessionById(sessionId); const conversationId = safeConversationId(routeId); if (!conversationId) return null; const listed = await accessStore?.listAgentSessionsForUser?.({ ownerUserId: actor?.id, actorRole: actor?.role, ownerScoped: true, conversationId, limit: 1, includeArchived: false }) ?? []; const matched = listed.find((session) => canActorReadSession(session, actor)) ?? null; if (matched) return matched; const suffix = conversationId.slice("cnv_".length); const deterministicSessionId = safeSessionId(`ses_${suffix}`); return deterministicSessionId ? getSessionById(deterministicSessionId) : null; } async function getSessionByTraceId(traceId) { const safeId = safeTraceId(traceId); if (!safeId) return null; if (traceSessionCache?.has(safeId)) { const cached = traceSessionCache.get(safeId) ?? null; return canActorReadSession(cached, actor) ? cached : null; } const session = await accessStore?.getAgentSessionByTraceId?.(safeId) ?? null; traceSessionCache?.set(safeId, session); return canActorReadSession(session, actor) ? session : null; } async function listSessions({ limit, offset = 0, includeRouteId } = {}) { let sessions = await accessStore?.listAgentSessionsForUser?.({ ownerUserId: actor?.id, actorRole: actor?.role, ownerScoped: true, limit, offset, includeArchived: false }) ?? []; const includeSessionId = safeSessionId(includeRouteId); if (includeRouteId && !sessions.some((session) => session.id === includeSessionId)) { const included = await getSessionByRouteId(includeRouteId); if (included) sessions = [included, ...sessions]; } return sessions.filter((session) => canActorReadSession(session, actor)); } function resultForTrace(traceId) { const safeId = safeTraceId(traceId); return safeId ? results?.get?.(safeId) ?? null : null; } async function traceSnapshot(traceId) { return durableTraceSnapshot(runtimeReader, traceId); } return { queryFacts, getSessionById, getSessionByRouteId, getSessionByTraceId, listSessions, resultForTrace, traceSnapshot, canReadOwner: (ownerUserId) => canActorReadOwner(ownerUserId, actor) }; } function logWorkbenchFactsQueryFailure(logger, params = {}, error = null) { const families = Array.isArray(params.families) ? params.families : []; const event = { event: "workbench_facts_query_failed", ok: false, families, hasSessionId: Boolean(params.sessionId), hasTraceId: Boolean(params.traceId), hasOwnerUserId: Boolean(params.ownerUserId), errorName: error?.name ?? "Error", errorCode: error?.code ?? null, message: error instanceof Error ? error.message : String(error ?? "unknown"), valuesRedacted: true }; try { if (typeof logger?.warn === "function") logger.warn(JSON.stringify(event)); else if (typeof logger?.log === "function") logger.log(JSON.stringify(event)); } catch {} } function emitWorkbenchFactsQueryFailureOtel(params = {}, error = null) { const data = objectValue(error?.data) ?? {}; const families = Array.isArray(params.families) ? params.families : []; const traceId = safeTraceId(params.traceId) ?? "trc_unassigned"; void emitCodeAgentOtelSpan("workbench.projection_store.query.failed", traceId, process.env, { status: "error", error, attributes: { "workbench.projection_store.error": true, "workbench.projection_store.retryable": data.retryable === true, "workbench.projection_store.transient": data.transient === true, "workbench.projection_store.retry_attempt": otelNonNegativeIntegerOrNull(data.retryAttempt ?? data.retryAttempts), "workbench.projection_store.retry_max": otelNonNegativeIntegerOrNull(data.retryMax), "workbench.projection_store.retry_exhausted": data.retryExhausted === true, "workbench.projection_store.retry_after_ms": otelNonNegativeIntegerOrNull(data.retryAfterMs), "workbench.facts.family_count": families.length, "workbench.facts.families": families.join(","), "workbench.facts.has_session_id": Boolean(params.sessionId), "workbench.facts.has_trace_id": Boolean(params.traceId), "workbench.facts.has_owner_user_id": Boolean(params.ownerUserId), "error.code": error?.code ?? null, "error.category": "projection_store_unavailable", "error.layer": "runtime-store", valuesRedacted: true } }).catch(() => undefined); } function otelNonNegativeIntegerOrNull(value) { const parsed = Number(value); return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null; } function objectValue(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : {}; } async function durableTraceSnapshot(runtimeStore, traceId) { if (!runtimeStore || typeof runtimeStore.queryWorkbenchFacts !== "function") return null; const safeId = safeTraceId(traceId); if (!safeId) return null; const cached = terminalDurableTraceCache.get(safeId); if (cached) { terminalDurableTraceCache.delete(safeId); terminalDurableTraceCache.set(safeId, cached); return cached; } let result; try { result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId, families: ["traceEvents"] }); } catch (error) { return projectionStoreUnavailableTrace(safeId, error); } const events = Array.isArray(result?.facts?.traceEvents) ? result.facts.traceEvents : []; if (events.length === 0) return null; const normalizedEvents = events.map((event, index) => ({ ...event, traceId: safeId, seq: eventSeq(event, index) })); const firstEvent = normalizedEvents[0] ?? null; const lastEvent = normalizedEvents.at(-1) ?? null; const status = durableTraceStatus(normalizedEvents); const terminalEvidence = traceTerminalEvidence({ events: normalizedEvents, status }); const snapshot = { traceId: safeId, status, createdAt: firstEvent?.createdAt ?? null, updatedAt: lastEvent?.createdAt ?? firstEvent?.createdAt ?? null, startedAt: firstEvent?.createdAt ?? null, finishedAt: TERMINAL_STATUSES.has(status) ? lastEvent?.createdAt ?? null : null, eventCount: normalizedEvents.length, events: normalizedEvents, assistantStreams: [], eventLabels: normalizedEvents.map((event) => event.label).filter(Boolean), lastEvent, terminalEvidence, outputTruncated: normalizedEvents.some((event) => event.outputTruncated === true), valuesPrinted: false }; rememberTerminalDurableTrace(snapshot); return snapshot; } function rememberTerminalDurableTrace(snapshot) { const traceId = safeTraceId(snapshot?.traceId); if (!traceId || !TERMINAL_STATUSES.has(normalizeStatus(snapshot?.status))) return; terminalDurableTraceCache.delete(traceId); terminalDurableTraceCache.set(traceId, snapshot); while (terminalDurableTraceCache.size > TERMINAL_DURABLE_TRACE_CACHE_MAX) { const oldest = terminalDurableTraceCache.keys().next().value; terminalDurableTraceCache.delete(oldest); } } function projectionStoreUnavailableTrace(traceId, error) { const message = "运行记录投影存储暂不可读,Trace 更新暂不可见。"; const retry = workbenchFactsRetryMetadata(error); const projection = { projectionStatus: "blocked", projectionHealth: "unavailable", blocker: { code: "projection_store_unavailable", layer: "runtime-store", category: "projection-store-unavailable", message, causeCode: error?.code ?? null, retryable: true, retryAttempt: retry.retryAttempt, retryMax: retry.retryMax, retryExhausted: retry.retryExhausted, retryAfterMs: retry.retryAfterMs, valuesPrinted: false }, valuesRedacted: true }; return { traceId, status: "missing", eventCount: 0, events: [], projection, projectionStatus: projection.projectionStatus, projectionHealth: projection.projectionHealth, blocker: projection.blocker, updatedAt: new Date().toISOString(), valuesPrinted: false }; } function workbenchFactsQueryRetryPolicy(env = {}) { return { maxAttempts: boundedInteger(env.HWLAB_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS, DEFAULT_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS, 1, 10), baseDelayMs: boundedInteger(env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS, DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS, 0, 5000), maxDelayMs: boundedInteger(env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS, DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS, 1, 10000) }; } function workbenchFactsRetryDelayMs(policy, failedAttempt) { const delay = policy.baseDelayMs * Math.pow(2, Math.max(0, failedAttempt - 1)); return Math.max(0, Math.min(policy.maxDelayMs, Math.trunc(delay))); } function isRetryableWorkbenchFactsQueryError(error) { const message = String(error?.message ?? error ?? ""); const code = String(error?.code ?? "").toLowerCase(); return /timeout|connection terminated|terminating connection|connection closed|connection ended|econnreset|econnrefused|etimedout/iu.test(message) || /timeout|econnreset|econnrefused|etimedout|57p01|08006|08003|53300/u.test(code); } function annotateWorkbenchFactsQueryError(error, metadata) { const normalized = error instanceof Error ? error : new Error(String(error ?? "Workbench facts query failed.")); const data = normalized.data && typeof normalized.data === "object" && !Array.isArray(normalized.data) ? normalized.data : {}; normalized.data = { ...data, retryable: metadata.retryable, transient: metadata.retryable === true, retryAttempt: metadata.retryAttempt, retryAttempts: metadata.retryAttempt, retryMax: metadata.retryMax, retryAfterMs: metadata.retryAfterMs, retryDelaysMs: metadata.retryDelaysMs.slice(), retryExhausted: metadata.retryExhausted === true, valuesRedacted: true }; return normalized; } function workbenchFactsRetryMetadata(error) { const data = error?.data && typeof error.data === "object" && !Array.isArray(error.data) ? error.data : {}; return { retryAttempt: finiteNonNegativeNumber(data.retryAttempt), retryMax: finiteNonNegativeNumber(data.retryMax), retryAfterMs: finiteNonNegativeNumber(data.retryAfterMs), retryExhausted: data.retryExhausted === true }; } function logWorkbenchFactsQueryRetry(logger, params, error, retry) { try { const line = JSON.stringify({ event: "workbench_facts_query_retry", ok: false, attempt: retry.attempt, maxAttempts: retry.maxAttempts, retryAfterMs: retry.retryAfterMs, traceId: safeTraceId(params.traceId) ?? null, sessionId: safeSessionId(params.sessionId) ?? null, errorCode: error?.code ?? null, message: error instanceof Error ? error.message : String(error ?? "unknown"), valuesRedacted: true }); if (typeof logger?.warn === "function") logger.warn(line); } catch { // Retry logging must not affect read-model responses. } } function sleepMs(ms) { return new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.trunc(Number(ms) || 0)))); } function boundedInteger(value, fallback, min, max) { const parsed = Number(value); if (!Number.isFinite(parsed)) return fallback; return Math.max(min, Math.min(max, Math.trunc(parsed))); } function finiteNonNegativeNumber(value) { const parsed = Number(value); return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null; } function canActorReadSession(session, actor) { if (!session || !actor) return false; if (normalizeStatus(session.status) === "archived") return false; if (actor.role === "admin") return true; return session.ownerUserId === actor.id; } function canActorReadOwner(ownerUserId, actor) { if (!ownerUserId || !actor) return true; if (actor.role === "admin") return true; return ownerUserId === actor.id; } function eventSeq(event, index) { const seq = Number(event?.seq); return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; } function normalizeStatus(value) { const text = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); if (text === "cancelled") return "canceled"; return text || "unknown"; } function normalizeWorkbenchFactsResult(facts = {}) { const normalized = emptyWorkbenchFacts(); for (const key of Object.keys(normalized)) { normalized[key] = Array.isArray(facts?.[key]) ? facts[key].map((item) => ({ ...item })) : []; } return normalized; } function emptyWorkbenchFacts() { return { inputs: [], sessions: [], messages: [], parts: [], turns: [], traceEvents: [], checkpoints: [] }; }