feat: add workbench projection performance metrics (#1592)

2026-06-19 11:13:26 +08:00
parent 034b9b5017
commit 0169125f8f
6 changed files with 765 additions and 19 deletions
@@ -855,6 +855,7 @@ function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, tr
    isTerminal: isTraceCommandTerminalStatus,
    activitySignature: agentRunProjectionActivitySignature,
    sleep: sleepAgentRunProjectionSync,
+    performanceStore: options.backendPerformanceStore,
    syncResult: ({ result }) => syncAgentRunChatResult({
      traceId,
      currentResult: result,
@@ -1181,10 +1182,11 @@ export async function handleCodeAgentChatResultHttp(request, response, url, opti
 }

 export async function handleCodeAgentTurnHttp(request, response, url, options) {
+  const startedAt = nowMs();
  const parts = url.pathname.split("/").filter(Boolean);
  const traceId = decodeURIComponent(parts[3] ?? "");
  if (!safeTraceId(traceId)) {
-    sendJson(response, 400, {
+    const body = {
      ok: false,
      status: "unknown",
      running: false,
@@ -1193,16 +1195,20 @@ export async function handleCodeAgentTurnHttp(request, response, url, options) {
        code: "invalid_trace_id",
        message: "traceId must start with trc_ and contain only safe identifier characters"
      }
-    });
+    };
+    recordCodeAgentTurnReadMetric(options, url, 400, body, startedAt);
+    sendJson(response, 400, body);
    return;
  }
  const requestOptions = codeAgentRequestScopedOptions(options);
  const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "turn_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "turn"));
  if (projectMismatch) {
+    recordCodeAgentTurnReadMetric(options, url, projectMismatch.statusCode, projectMismatch.body, startedAt);
    sendJson(response, projectMismatch.statusCode, projectMismatch.body);
    return;
  }
  const resolved = await measureCodeAgentHttpPhase(requestOptions, "turn_resolve_status", () => resolveCodeAgentTurnStatusSnapshot(traceId, requestOptions));
+  recordCodeAgentTurnReadMetric(options, url, resolved.statusCode, resolved.body, startedAt);
  sendJson(response, resolved.statusCode, resolved.body);
 }

@@ -1240,13 +1246,16 @@ async function refreshAgentRunProjectionForCompatRead({ traceId, result = null,
  if (!safeTraceId(traceId) || !shouldRefreshAgentRunProjectionOnRead(result)) return { result, runnerTrace: trace, refreshError: null };
  const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
  const refreshOptions = codeAgentTurnStatusRefreshOptions(options);
+  const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
+  const startedAt = nowMs();
+  performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "seen", reason: "read_side_refresh" });
  try {
    const synced = await syncAgentRunChatResult({
      traceId,
      currentResult: result,
      options: refreshOptions,
      traceStore,
-      forceResultSync: true
+      forceResultSync: agentRunProjectionObservedTerminal(result, trace)
    });
    const payload = synced.result ?? result;
    if (payload && isTraceCommandTerminalStatus(payload.status)) {
@@ -1256,8 +1265,11 @@ async function refreshAgentRunProjectionForCompatRead({ traceId, result = null,
        options: refreshOptions
      });
    }
+    performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "turn_finalize", status: "ok", durationMs: nowMs() - startedAt });
    return { result: payload, runnerTrace: synced.runnerTrace ?? traceStore.snapshot(traceId), refreshError: null };
  } catch (error) {
+    performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "turn_finalize", status: error?.code === "agentrun_timeout" ? "timeout" : "error", durationMs: nowMs() - startedAt });
+    performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_read_side_sync_failed" });
    traceStore.append(traceId, {
      type: "projection_refresh",
      status: "degraded",
@@ -1336,6 +1348,7 @@ async function readCodeAgentCompatProjection(traceId, options = {}) {
    trace = traceSnapshotWithTerminalEvidence(refreshed.runnerTrace ?? trace, result, traceId, refreshed.refreshError ?? null);
  }
  const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: refreshed.refreshError ?? null });
+  recordCodeAgentProjectionMetrics(options, { result, trace, projection, refreshError: refreshed.refreshError ?? null });
  const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true);
  return { result, session, trace, projection, found, refreshError: refreshed.refreshError ?? null };
 }
@@ -3028,8 +3041,105 @@ function createCodeAgentM3HwlabApiRequestJson(options = {}) {
    };
  };
 }
+function recordCodeAgentProjectionMetrics(options = {}, { result = null, trace = null, projection = null, refreshError = null } = {}) {
+  const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
+  if (!performanceStore?.recordWorkbenchProjection || !projection) return;
+  const sourceLatestSeq = sourceLatestSeqFromResult(result);
+  const lastProjectedSeq = Number(projection.lastProjectedSeq ?? 0);
+  if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return;
+  performanceStore.recordWorkbenchProjection({
+    sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq,
+    lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0,
+    sourceLatestEventAt: sourceLatestAtFromResult(result, trace),
+    projectedLatestEventAt: projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null,
+    terminalSourceAt: terminalSourceAtFromResult(result),
+    terminalProjectedAt: projection.projectionStatus === "caught-up" ? projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null,
+    status: result?.status ?? trace?.status ?? "unknown",
+    reason: projectionReason(projection, refreshError),
+    projectionStatus: projection.projectionStatus ?? "unknown",
+    source: result?.agentRun ? "agentrun_v01" : "workbench_read_model"
+  });
+}

+function recordCodeAgentTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) {
+  const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
+  if (!performanceStore?.recordWorkbenchTurnRead) return;
+  performanceStore.recordWorkbenchTurnRead({
+    route: url?.pathname ?? "/v1/agent/turns/:id",
+    status: body?.status ?? statusClassLabel(statusCode),
+    degradedReason: turnReadDegradedReason(body),
+    durationMs: nowMs() - startedAt,
+    responseBytes: responseBodyBytes(body),
+    timedOut: statusCode === 504 || body?.error?.code === "timeout" || body?.refreshError?.code === "agentrun_timeout"
+  });
+}

+function turnReadDegradedReason(body = {}) {
+  if (body?.refreshError?.code) return body.refreshError.code;
+  if (body?.projection?.blocker?.code) return body.projection.blocker.code;
+  if (body?.blocker?.code) return body.blocker.code;
+  if (body?.error?.code) return body.error.code;
+  if (body?.projectionStatus === "projecting") return "projecting";
+  if (body?.projectionStatus === "blocked") return "projection_blocked";
+  return "none";
+}
+
+function responseBodyBytes(body = {}) {
+  try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; }
+}
+
+function statusClassLabel(value) {
+  const status = Number(value);
+  if (!Number.isFinite(status) || status <= 0) return "unknown";
+  return `${Math.floor(status / 100)}xx`;
+}
+
+function sourceLatestSeqFromResult(result = null) {
+  const candidates = [
+    result?.agentRun?.lastSeq,
+    result?.traceSummary?.agentRun?.lastSeq,
+    result?.traceSummary?.lastSeq,
+    result?.providerTrace?.lastSeq,
+    result?.runnerTrace?.lastEvent?.sourceSeq,
+    result?.runnerTrace?.lastEvent?.seq,
+    result?.runnerTrace?.eventCount
+  ];
+  for (const value of candidates) {
+    const seq = Number(value);
+    if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq);
+  }
+  return NaN;
+}
+
+function sourceLatestAtFromResult(result = null, trace = null) {
+  return result?.traceSummary?.updatedAt
+    ?? result?.agentRun?.updatedAt
+    ?? result?.providerTrace?.updatedAt
+    ?? result?.runnerTrace?.updatedAt
+    ?? trace?.updatedAt
+    ?? result?.updatedAt
+    ?? null;
+}
+
+function terminalSourceAtFromResult(result = null) {
+  if (!result) return null;
+  const terminalStatus = result?.agentRun?.terminalStatus ?? result?.traceSummary?.terminalStatus ?? result?.status;
+  if (!isTraceCommandTerminalStatus(terminalStatus)) return null;
+  return result?.traceSummary?.updatedAt ?? result?.agentRun?.updatedAt ?? result?.updatedAt ?? null;
+}
+
+function projectionReason(projection = null, refreshError = null) {
+  if (refreshError) return refreshError.code ?? "refresh_error";
+  if (projection?.blocker) return projection.blocker.code ?? "projection_blocked";
+  if (projection?.projectionStatus === "projecting") return "projecting";
+  if (projection?.projectionStatus === "caught-up") return "none";
+  return projection?.projectionStatus ?? "unknown";
+}
+
+function nowMs() {
+  if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
+  return Date.now();
+}

 function uniqueStrings(values) {
  return [...new Set(values.map((value) => String(value ?? "").trim()).filter(Boolean))];