fix workbench otel http span timing (#1815)
This commit is contained in:
@@ -273,6 +273,7 @@ export async function emitHttpServerRequestSpan(context, env = process.env, opti
|
||||
"http.response.status_code": statusCode || null,
|
||||
"http.response.status_class": statusClassValue,
|
||||
"hwlab.http.stage": "http.server.request",
|
||||
...options.attributes,
|
||||
...(errorCode ? { "error.code": errorCode, "error.category": options.errorCategory ?? null, "error.layer": options.errorLayer ?? null } : {})
|
||||
};
|
||||
const body = {
|
||||
|
||||
@@ -124,6 +124,13 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
|
||||
const activeTraceId = requestedTraceId
|
||||
?? safeTraceId(initialSession?.lastTraceId ?? initialSessionSnapshot.lastTraceId ?? initialSessionSnapshot.currentTraceId)
|
||||
?? null;
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
sessionId: streamSessionId ?? requestedSessionId,
|
||||
traceId: activeTraceId,
|
||||
threadId: streamThreadId,
|
||||
heartbeatMs,
|
||||
outboxMode: typeof (options.runtimeStore ?? null)?.readWorkbenchProjectionOutbox === "function"
|
||||
});
|
||||
if (activeTraceId) {
|
||||
await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" }));
|
||||
const runtimeStore = options.runtimeStore ?? null;
|
||||
@@ -222,6 +229,22 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
|
||||
});
|
||||
}
|
||||
|
||||
function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context) return;
|
||||
context.route = "/v1/workbench/events";
|
||||
context.otelAttributes = {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"workbench.route": "events",
|
||||
"workbench.session_id": fields.sessionId ?? null,
|
||||
traceId: fields.traceId ?? null,
|
||||
"workbench.trace_id": fields.traceId ?? null,
|
||||
"workbench.thread_id": fields.threadId ?? null,
|
||||
"workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null,
|
||||
"workbench.sse.outbox_mode": fields.outboxMode === true
|
||||
};
|
||||
}
|
||||
|
||||
function realtimeEventCreatedAt(payload) {
|
||||
const candidates = [payload?.event?.createdAt, payload?.snapshot?.lastEvent?.createdAt, payload?.turn?.updatedAt];
|
||||
for (const value of candidates) {
|
||||
|
||||
@@ -131,23 +131,32 @@ export function createCloudApiServer(options = {}) {
|
||||
const requestContext = buildHttpRequestContext(request);
|
||||
attachHttpRequestContext(request, response, requestContext);
|
||||
const backendPerformance = backendPerformanceStore.beginHttpRequest(request, response);
|
||||
response.once("finish", () => {
|
||||
let otelHttpSpanEmitted = false;
|
||||
const emitHttpSpan = (closedBy) => {
|
||||
if (otelHttpSpanEmitted) return;
|
||||
otelHttpSpanEmitted = true;
|
||||
const httpCtx = request.hwlabHttpRequestContext;
|
||||
if (!httpCtx?.traceId) return;
|
||||
if (isOtelNoisyProbeRoute(httpCtx.route)) return;
|
||||
const lastError = httpCtx.lastHttpError;
|
||||
void emitHttpServerRequestSpan(httpCtx, env, {
|
||||
startTimeMs: backendPerformance.startedAt,
|
||||
startTimeMs: httpCtx.startedAtMs,
|
||||
endTimeMs: Date.now(),
|
||||
statusCode: Number(response.statusCode ?? backendPerformance.statusCode ?? 0),
|
||||
method: backendPerformance.method || httpCtx.method,
|
||||
route: backendPerformance.route || httpCtx.route,
|
||||
attributes: {
|
||||
"http.closed_by": closedBy,
|
||||
...(httpCtx.otelAttributes ?? {})
|
||||
},
|
||||
errorCode: lastError?.code,
|
||||
errorCategory: lastError?.category,
|
||||
errorLayer: lastError?.layer,
|
||||
statusMessage: lastError?.code
|
||||
}).catch(() => undefined);
|
||||
});
|
||||
};
|
||||
response.once("finish", () => emitHttpSpan("finish"));
|
||||
response.once("close", () => emitHttpSpan("close"));
|
||||
await withBackendPerformanceContext(backendPerformance, async () => {
|
||||
try {
|
||||
await routeRequest(request, response, { ...options, env, runtimeStore, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance });
|
||||
@@ -178,6 +187,7 @@ function buildHttpRequestContext(request) {
|
||||
});
|
||||
return {
|
||||
requestId: safeRequestId(getHeader(request, "x-request-id")) || `req_${randomUUID()}`,
|
||||
startedAtMs: Date.now(),
|
||||
traceId: otelContext.traceId,
|
||||
parentSpanId: otelContext.parentSpanId,
|
||||
spanId: otelContext.spanId,
|
||||
|
||||
Reference in New Issue
Block a user