diff --git a/internal/cloud/otel-trace.ts b/internal/cloud/otel-trace.ts index 1c6a9ab9..53ca3ed4 100644 --- a/internal/cloud/otel-trace.ts +++ b/internal/cloud/otel-trace.ts @@ -273,6 +273,7 @@ export async function emitHttpServerRequestSpan(context, env = process.env, opti "http.response.status_code": statusCode || null, "http.response.status_class": statusClassValue, "hwlab.http.stage": "http.server.request", + ...options.attributes, ...(errorCode ? { "error.code": errorCode, "error.category": options.errorCategory ?? null, "error.layer": options.errorLayer ?? null } : {}) }; const body = { diff --git a/internal/cloud/server-workbench-http.ts b/internal/cloud/server-workbench-http.ts index f072d69e..a46f6f48 100644 --- a/internal/cloud/server-workbench-http.ts +++ b/internal/cloud/server-workbench-http.ts @@ -124,6 +124,13 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option const activeTraceId = requestedTraceId ?? safeTraceId(initialSession?.lastTraceId ?? initialSessionSnapshot.lastTraceId ?? initialSessionSnapshot.currentTraceId) ?? null; + attachWorkbenchRealtimeOtelContext(request, { + sessionId: streamSessionId ?? requestedSessionId, + traceId: activeTraceId, + threadId: streamThreadId, + heartbeatMs, + outboxMode: typeof (options.runtimeStore ?? null)?.readWorkbenchProjectionOutbox === "function" + }); if (activeTraceId) { await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })); const runtimeStore = options.runtimeStore ?? null; @@ -222,6 +229,22 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option }); } +function attachWorkbenchRealtimeOtelContext(request, fields = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context) return; + context.route = "/v1/workbench/events"; + context.otelAttributes = { + ...(context.otelAttributes ?? {}), + "workbench.route": "events", + "workbench.session_id": fields.sessionId ?? null, + traceId: fields.traceId ?? null, + "workbench.trace_id": fields.traceId ?? null, + "workbench.thread_id": fields.threadId ?? null, + "workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null, + "workbench.sse.outbox_mode": fields.outboxMode === true + }; +} + function realtimeEventCreatedAt(payload) { const candidates = [payload?.event?.createdAt, payload?.snapshot?.lastEvent?.createdAt, payload?.turn?.updatedAt]; for (const value of candidates) { diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index 050d3c0b..28c56d0b 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -131,23 +131,32 @@ export function createCloudApiServer(options = {}) { const requestContext = buildHttpRequestContext(request); attachHttpRequestContext(request, response, requestContext); const backendPerformance = backendPerformanceStore.beginHttpRequest(request, response); - response.once("finish", () => { + let otelHttpSpanEmitted = false; + const emitHttpSpan = (closedBy) => { + if (otelHttpSpanEmitted) return; + otelHttpSpanEmitted = true; const httpCtx = request.hwlabHttpRequestContext; if (!httpCtx?.traceId) return; if (isOtelNoisyProbeRoute(httpCtx.route)) return; const lastError = httpCtx.lastHttpError; void emitHttpServerRequestSpan(httpCtx, env, { - startTimeMs: backendPerformance.startedAt, + startTimeMs: httpCtx.startedAtMs, endTimeMs: Date.now(), statusCode: Number(response.statusCode ?? backendPerformance.statusCode ?? 0), method: backendPerformance.method || httpCtx.method, route: backendPerformance.route || httpCtx.route, + attributes: { + "http.closed_by": closedBy, + ...(httpCtx.otelAttributes ?? {}) + }, errorCode: lastError?.code, errorCategory: lastError?.category, errorLayer: lastError?.layer, statusMessage: lastError?.code }).catch(() => undefined); - }); + }; + response.once("finish", () => emitHttpSpan("finish")); + response.once("close", () => emitHttpSpan("close")); await withBackendPerformanceContext(backendPerformance, async () => { try { await routeRequest(request, response, { ...options, env, runtimeStore, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance }); @@ -178,6 +187,7 @@ function buildHttpRequestContext(request) { }); return { requestId: safeRequestId(getHeader(request, "x-request-id")) || `req_${randomUUID()}`, + startedAtMs: Date.now(), traceId: otelContext.traceId, parentSpanId: otelContext.parentSpanId, spanId: otelContext.spanId,