root
423 lines
16 KiB
TypeScript
423 lines
16 KiB
TypeScript
/*
|
|
* SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; draft-2026-06-20-p1-zero-split-durable-realtime; PJ2026-010403 API契约 draft-2026-06-18-r1.
|
|
* 职责: Workbench durable facts store adapter。统一读取 session/message/turn/trace projection facts，不在 GET 中推进 AgentRun facts。
|
|
*/
|
|
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
|
import { safeConversationId, safeSessionId, safeTraceId } from "./server-http-utils.ts";
|
|
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
|
import { durableTraceStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
|
|
|
const TERMINAL_DURABLE_TRACE_CACHE_MAX = 256;
|
|
const DEFAULT_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS = 5;
|
|
const DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS = 80;
|
|
const DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS = 1000;
|
|
const terminalDurableTraceCache = new Map();
|
|
|
|
export function createWorkbenchFactsStore(options = {}, actor = null) {
|
|
const accessStore = options.accessController?.store ?? options.accessController ?? null;
|
|
const traceSessionCache = options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : null;
|
|
const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger });
|
|
const results = options.codeAgentChatResults ?? null;
|
|
|
|
async function queryFacts(params = {}) {
|
|
if (typeof runtimeReader?.queryWorkbenchFacts === "function") {
|
|
const retryPolicy = workbenchFactsQueryRetryPolicy(options.env ?? process.env);
|
|
const retryDelaysMs = [];
|
|
for (let attempt = 1; attempt <= retryPolicy.maxAttempts; attempt += 1) {
|
|
try {
|
|
const result = await runtimeReader.queryWorkbenchFacts(params);
|
|
return {
|
|
facts: normalizeWorkbenchFactsResult(result?.facts),
|
|
count: Number.isFinite(Number(result?.count)) ? Number(result.count) : 0,
|
|
durable: true,
|
|
persistence: result?.persistence ?? null,
|
|
cache: result?.cache ?? null,
|
|
retry: attempt > 1 ? { attempts: attempt, maxAttempts: retryPolicy.maxAttempts, delaysMs: retryDelaysMs.slice(), valuesRedacted: true } : null
|
|
};
|
|
} catch (error) {
|
|
const retryable = isRetryableWorkbenchFactsQueryError(error);
|
|
const finalAttempt = !retryable || attempt >= retryPolicy.maxAttempts;
|
|
const retryAfterMs = finalAttempt ? null : workbenchFactsRetryDelayMs(retryPolicy, attempt);
|
|
const annotatedError = annotateWorkbenchFactsQueryError(error, {
|
|
retryable,
|
|
retryAttempt: attempt,
|
|
retryMax: retryPolicy.maxAttempts,
|
|
retryAfterMs,
|
|
retryDelaysMs,
|
|
retryExhausted: finalAttempt && retryable
|
|
});
|
|
if (finalAttempt) {
|
|
emitWorkbenchFactsQueryFailureOtel(params, annotatedError);
|
|
logWorkbenchFactsQueryFailure(options.logger ?? console, params, annotatedError);
|
|
return {
|
|
facts: emptyWorkbenchFacts(),
|
|
count: 0,
|
|
durable: true,
|
|
error: annotatedError,
|
|
projection: projectionStoreUnavailableTrace(safeTraceId(params.traceId) ?? "trc_unassigned", annotatedError)
|
|
};
|
|
}
|
|
logWorkbenchFactsQueryRetry(options.logger ?? console, params, annotatedError, { attempt, maxAttempts: retryPolicy.maxAttempts, retryAfterMs });
|
|
retryDelaysMs.push(retryAfterMs);
|
|
await sleepMs(retryAfterMs);
|
|
}
|
|
}
|
|
}
|
|
return {
|
|
facts: emptyWorkbenchFacts(),
|
|
count: 0,
|
|
durable: false,
|
|
persistence: null
|
|
};
|
|
}
|
|
|
|
async function getSessionById(sessionId) {
|
|
const safeId = safeSessionId(sessionId);
|
|
if (!safeId) return null;
|
|
const session = await accessStore?.getAgentSession?.(safeId) ?? null;
|
|
return canActorReadSession(session, actor) ? session : null;
|
|
}
|
|
|
|
async function getSessionByRouteId(routeId) {
|
|
const sessionId = safeSessionId(routeId);
|
|
if (sessionId) return getSessionById(sessionId);
|
|
const conversationId = safeConversationId(routeId);
|
|
if (!conversationId) return null;
|
|
const listed = await accessStore?.listAgentSessionsForUser?.({
|
|
ownerUserId: actor?.id,
|
|
actorRole: actor?.role,
|
|
ownerScoped: true,
|
|
conversationId,
|
|
limit: 1,
|
|
includeArchived: false
|
|
}) ?? [];
|
|
const matched = listed.find((session) => canActorReadSession(session, actor)) ?? null;
|
|
if (matched) return matched;
|
|
const suffix = conversationId.slice("cnv_".length);
|
|
const deterministicSessionId = safeSessionId(`ses_${suffix}`);
|
|
return deterministicSessionId ? getSessionById(deterministicSessionId) : null;
|
|
}
|
|
|
|
async function getSessionByTraceId(traceId) {
|
|
const safeId = safeTraceId(traceId);
|
|
if (!safeId) return null;
|
|
if (traceSessionCache?.has(safeId)) {
|
|
const cached = traceSessionCache.get(safeId) ?? null;
|
|
return canActorReadSession(cached, actor) ? cached : null;
|
|
}
|
|
const session = await accessStore?.getAgentSessionByTraceId?.(safeId) ?? null;
|
|
traceSessionCache?.set(safeId, session);
|
|
return canActorReadSession(session, actor) ? session : null;
|
|
}
|
|
|
|
async function listSessions({ limit, offset = 0, includeRouteId } = {}) {
|
|
let sessions = await accessStore?.listAgentSessionsForUser?.({
|
|
ownerUserId: actor?.id,
|
|
actorRole: actor?.role,
|
|
ownerScoped: true,
|
|
limit,
|
|
offset,
|
|
includeArchived: false
|
|
}) ?? [];
|
|
const includeSessionId = safeSessionId(includeRouteId);
|
|
if (includeRouteId && !sessions.some((session) => session.id === includeSessionId)) {
|
|
const included = await getSessionByRouteId(includeRouteId);
|
|
if (included) sessions = [included, ...sessions];
|
|
}
|
|
return sessions.filter((session) => canActorReadSession(session, actor));
|
|
}
|
|
|
|
function resultForTrace(traceId) {
|
|
const safeId = safeTraceId(traceId);
|
|
return safeId ? results?.get?.(safeId) ?? null : null;
|
|
}
|
|
|
|
async function traceSnapshot(traceId) {
|
|
return durableTraceSnapshot(runtimeReader, traceId);
|
|
}
|
|
|
|
return {
|
|
queryFacts,
|
|
getSessionById,
|
|
getSessionByRouteId,
|
|
getSessionByTraceId,
|
|
listSessions,
|
|
resultForTrace,
|
|
traceSnapshot,
|
|
canReadOwner: (ownerUserId) => canActorReadOwner(ownerUserId, actor)
|
|
};
|
|
}
|
|
|
|
function logWorkbenchFactsQueryFailure(logger, params = {}, error = null) {
|
|
const families = Array.isArray(params.families) ? params.families : [];
|
|
const event = {
|
|
event: "workbench_facts_query_failed",
|
|
ok: false,
|
|
families,
|
|
hasSessionId: Boolean(params.sessionId),
|
|
hasTraceId: Boolean(params.traceId),
|
|
hasOwnerUserId: Boolean(params.ownerUserId),
|
|
errorName: error?.name ?? "Error",
|
|
errorCode: error?.code ?? null,
|
|
message: error instanceof Error ? error.message : String(error ?? "unknown"),
|
|
valuesRedacted: true
|
|
};
|
|
try {
|
|
if (typeof logger?.warn === "function") logger.warn(JSON.stringify(event));
|
|
else if (typeof logger?.log === "function") logger.log(JSON.stringify(event));
|
|
} catch {}
|
|
}
|
|
|
|
function emitWorkbenchFactsQueryFailureOtel(params = {}, error = null) {
|
|
const data = objectValue(error?.data) ?? {};
|
|
const families = Array.isArray(params.families) ? params.families : [];
|
|
const traceId = safeTraceId(params.traceId) ?? "trc_unassigned";
|
|
void emitCodeAgentOtelSpan("workbench.projection_store.query.failed", traceId, process.env, {
|
|
status: "error",
|
|
error,
|
|
attributes: {
|
|
"workbench.projection_store.error": true,
|
|
"workbench.projection_store.retryable": data.retryable === true,
|
|
"workbench.projection_store.transient": data.transient === true,
|
|
"workbench.projection_store.retry_attempt": otelNonNegativeIntegerOrNull(data.retryAttempt ?? data.retryAttempts),
|
|
"workbench.projection_store.retry_max": otelNonNegativeIntegerOrNull(data.retryMax),
|
|
"workbench.projection_store.retry_exhausted": data.retryExhausted === true,
|
|
"workbench.projection_store.retry_after_ms": otelNonNegativeIntegerOrNull(data.retryAfterMs),
|
|
"workbench.facts.family_count": families.length,
|
|
"workbench.facts.families": families.join(","),
|
|
"workbench.facts.has_session_id": Boolean(params.sessionId),
|
|
"workbench.facts.has_trace_id": Boolean(params.traceId),
|
|
"workbench.facts.has_owner_user_id": Boolean(params.ownerUserId),
|
|
"error.code": error?.code ?? null,
|
|
"error.category": "projection_store_unavailable",
|
|
"error.layer": "runtime-store",
|
|
valuesRedacted: true
|
|
}
|
|
}).catch(() => undefined);
|
|
}
|
|
|
|
function otelNonNegativeIntegerOrNull(value) {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null;
|
|
}
|
|
|
|
function objectValue(value) {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
}
|
|
|
|
async function durableTraceSnapshot(runtimeStore, traceId) {
|
|
if (!runtimeStore || typeof runtimeStore.queryWorkbenchFacts !== "function") return null;
|
|
const safeId = safeTraceId(traceId);
|
|
if (!safeId) return null;
|
|
const cached = terminalDurableTraceCache.get(safeId);
|
|
if (cached) {
|
|
terminalDurableTraceCache.delete(safeId);
|
|
terminalDurableTraceCache.set(safeId, cached);
|
|
return cached;
|
|
}
|
|
let result;
|
|
try {
|
|
result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId, families: ["traceEvents"] });
|
|
} catch (error) {
|
|
return projectionStoreUnavailableTrace(safeId, error);
|
|
}
|
|
const events = Array.isArray(result?.facts?.traceEvents) ? result.facts.traceEvents : [];
|
|
if (events.length === 0) return null;
|
|
const normalizedEvents = events.map((event, index) => ({ ...event, traceId: safeId, seq: eventSeq(event, index) }));
|
|
const firstEvent = normalizedEvents[0] ?? null;
|
|
const lastEvent = normalizedEvents.at(-1) ?? null;
|
|
const status = durableTraceStatus(normalizedEvents);
|
|
const terminalEvidence = traceTerminalEvidence({ events: normalizedEvents, status });
|
|
const snapshot = {
|
|
traceId: safeId,
|
|
status,
|
|
createdAt: firstEvent?.createdAt ?? null,
|
|
updatedAt: lastEvent?.createdAt ?? firstEvent?.createdAt ?? null,
|
|
startedAt: firstEvent?.createdAt ?? null,
|
|
finishedAt: TERMINAL_STATUSES.has(status) ? lastEvent?.createdAt ?? null : null,
|
|
eventCount: normalizedEvents.length,
|
|
events: normalizedEvents,
|
|
assistantStreams: [],
|
|
eventLabels: normalizedEvents.map((event) => event.label).filter(Boolean),
|
|
lastEvent,
|
|
terminalEvidence,
|
|
outputTruncated: normalizedEvents.some((event) => event.outputTruncated === true),
|
|
valuesPrinted: false
|
|
};
|
|
rememberTerminalDurableTrace(snapshot);
|
|
return snapshot;
|
|
}
|
|
|
|
function rememberTerminalDurableTrace(snapshot) {
|
|
const traceId = safeTraceId(snapshot?.traceId);
|
|
if (!traceId || !TERMINAL_STATUSES.has(normalizeStatus(snapshot?.status))) return;
|
|
terminalDurableTraceCache.delete(traceId);
|
|
terminalDurableTraceCache.set(traceId, snapshot);
|
|
while (terminalDurableTraceCache.size > TERMINAL_DURABLE_TRACE_CACHE_MAX) {
|
|
const oldest = terminalDurableTraceCache.keys().next().value;
|
|
terminalDurableTraceCache.delete(oldest);
|
|
}
|
|
}
|
|
|
|
function projectionStoreUnavailableTrace(traceId, error) {
|
|
const message = "运行记录投影存储暂不可读，Trace 更新暂不可见。";
|
|
const retry = workbenchFactsRetryMetadata(error);
|
|
const projection = {
|
|
projectionStatus: "blocked",
|
|
projectionHealth: "unavailable",
|
|
blocker: {
|
|
code: "projection_store_unavailable",
|
|
layer: "runtime-store",
|
|
category: "projection-store-unavailable",
|
|
message,
|
|
causeCode: error?.code ?? null,
|
|
retryable: true,
|
|
retryAttempt: retry.retryAttempt,
|
|
retryMax: retry.retryMax,
|
|
retryExhausted: retry.retryExhausted,
|
|
retryAfterMs: retry.retryAfterMs,
|
|
valuesPrinted: false
|
|
},
|
|
valuesRedacted: true
|
|
};
|
|
return {
|
|
traceId,
|
|
status: "missing",
|
|
eventCount: 0,
|
|
events: [],
|
|
projection,
|
|
projectionStatus: projection.projectionStatus,
|
|
projectionHealth: projection.projectionHealth,
|
|
blocker: projection.blocker,
|
|
updatedAt: new Date().toISOString(),
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function workbenchFactsQueryRetryPolicy(env = {}) {
|
|
return {
|
|
maxAttempts: boundedInteger(env.HWLAB_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS, DEFAULT_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS, 1, 10),
|
|
baseDelayMs: boundedInteger(env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS, DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS, 0, 5000),
|
|
maxDelayMs: boundedInteger(env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS, DEFAULT_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS, 1, 10000)
|
|
};
|
|
}
|
|
|
|
function workbenchFactsRetryDelayMs(policy, failedAttempt) {
|
|
const delay = policy.baseDelayMs * Math.pow(2, Math.max(0, failedAttempt - 1));
|
|
return Math.max(0, Math.min(policy.maxDelayMs, Math.trunc(delay)));
|
|
}
|
|
|
|
function isRetryableWorkbenchFactsQueryError(error) {
|
|
const message = String(error?.message ?? error ?? "");
|
|
const code = String(error?.code ?? "").toLowerCase();
|
|
return /timeout|connection terminated|terminating connection|connection closed|connection ended|econnreset|econnrefused|etimedout/iu.test(message)
|
|
|| /timeout|econnreset|econnrefused|etimedout|57p01|08006|08003|53300/u.test(code);
|
|
}
|
|
|
|
function annotateWorkbenchFactsQueryError(error, metadata) {
|
|
const normalized = error instanceof Error ? error : new Error(String(error ?? "Workbench facts query failed."));
|
|
const data = normalized.data && typeof normalized.data === "object" && !Array.isArray(normalized.data) ? normalized.data : {};
|
|
normalized.data = {
|
|
...data,
|
|
retryable: metadata.retryable,
|
|
transient: metadata.retryable === true,
|
|
retryAttempt: metadata.retryAttempt,
|
|
retryAttempts: metadata.retryAttempt,
|
|
retryMax: metadata.retryMax,
|
|
retryAfterMs: metadata.retryAfterMs,
|
|
retryDelaysMs: metadata.retryDelaysMs.slice(),
|
|
retryExhausted: metadata.retryExhausted === true,
|
|
valuesRedacted: true
|
|
};
|
|
return normalized;
|
|
}
|
|
|
|
function workbenchFactsRetryMetadata(error) {
|
|
const data = error?.data && typeof error.data === "object" && !Array.isArray(error.data) ? error.data : {};
|
|
return {
|
|
retryAttempt: finiteNonNegativeNumber(data.retryAttempt),
|
|
retryMax: finiteNonNegativeNumber(data.retryMax),
|
|
retryAfterMs: finiteNonNegativeNumber(data.retryAfterMs),
|
|
retryExhausted: data.retryExhausted === true
|
|
};
|
|
}
|
|
|
|
function logWorkbenchFactsQueryRetry(logger, params, error, retry) {
|
|
try {
|
|
const line = JSON.stringify({
|
|
event: "workbench_facts_query_retry",
|
|
ok: false,
|
|
attempt: retry.attempt,
|
|
maxAttempts: retry.maxAttempts,
|
|
retryAfterMs: retry.retryAfterMs,
|
|
traceId: safeTraceId(params.traceId) ?? null,
|
|
sessionId: safeSessionId(params.sessionId) ?? null,
|
|
errorCode: error?.code ?? null,
|
|
message: error instanceof Error ? error.message : String(error ?? "unknown"),
|
|
valuesRedacted: true
|
|
});
|
|
if (typeof logger?.warn === "function") logger.warn(line);
|
|
} catch {
|
|
// Retry logging must not affect read-model responses.
|
|
}
|
|
}
|
|
|
|
function sleepMs(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.trunc(Number(ms) || 0))));
|
|
}
|
|
|
|
function boundedInteger(value, fallback, min, max) {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed)) return fallback;
|
|
return Math.max(min, Math.min(max, Math.trunc(parsed)));
|
|
}
|
|
|
|
function finiteNonNegativeNumber(value) {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null;
|
|
}
|
|
|
|
function canActorReadSession(session, actor) {
|
|
if (!session || !actor) return false;
|
|
if (normalizeStatus(session.status) === "archived") return false;
|
|
if (actor.role === "admin") return true;
|
|
return session.ownerUserId === actor.id;
|
|
}
|
|
|
|
function canActorReadOwner(ownerUserId, actor) {
|
|
if (!ownerUserId || !actor) return true;
|
|
if (actor.role === "admin") return true;
|
|
return ownerUserId === actor.id;
|
|
}
|
|
|
|
function eventSeq(event, index) {
|
|
const seq = Number(event?.seq);
|
|
return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1;
|
|
}
|
|
|
|
function normalizeStatus(value) {
|
|
const text = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
|
if (text === "cancelled") return "canceled";
|
|
return text || "unknown";
|
|
}
|
|
|
|
function normalizeWorkbenchFactsResult(facts = {}) {
|
|
const normalized = emptyWorkbenchFacts();
|
|
for (const key of Object.keys(normalized)) {
|
|
normalized[key] = Array.isArray(facts?.[key]) ? facts[key].map((item) => ({ ...item })) : [];
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function emptyWorkbenchFacts() {
|
|
return {
|
|
inputs: [],
|
|
sessions: [],
|
|
messages: [],
|
|
parts: [],
|
|
turns: [],
|
|
traceEvents: [],
|
|
checkpoints: []
|
|
};
|
|
}
|