diff --git a/internal/cloud/error-envelope.ts b/internal/cloud/error-envelope.ts index 46dcea1a..2a045ddd 100644 --- a/internal/cloud/error-envelope.ts +++ b/internal/cloud/error-envelope.ts @@ -38,6 +38,10 @@ export function decorateErrorPayloadForHttp(payload, options = {}) { }, valuesPrinted: originalError.valuesPrinted === true ? true : false }; + const httpContext = options.context ?? options.response?.hwlabHttpRequestContext ?? options.request?.hwlabHttpRequestContext; + if (plainObject(httpContext)) { + httpContext.lastHttpError = { code: diagnosticCode, category: diagnostic.category, layer: diagnostic.layer, httpStatus: statusCode, valuesPrinted: false }; + } return { ...payload, error: nextError }; } diff --git a/internal/cloud/otel-trace.ts b/internal/cloud/otel-trace.ts index b3587718..1c6a9ab9 100644 --- a/internal/cloud/otel-trace.ts +++ b/internal/cloud/otel-trace.ts @@ -248,6 +248,81 @@ export async function emitAuthOtelSpan(name, context, env = process.env, options } } +export async function emitHttpServerRequestSpan(context, env = process.env, options = {}) { + const endpoint = resolveOtlpTracesEndpoint(env); + if (!endpoint || typeof fetch !== "function") return { ok: false, skipped: true, reason: "otlp-endpoint-missing", valuesPrinted: false }; + const traceId = normalizeOtelTraceId(context?.traceId); + if (!traceId) return { ok: false, skipped: true, reason: "trace-id-missing", valuesPrinted: false }; + const spanId = normalizeOtelSpanId(options.spanId) ?? newOtelSpanId(); + const parentSpanId = normalizeOtelSpanId(context?.parentSpanId) ?? null; + const now = Date.now(); + const startedAtMs = epochUnixMs(options.startTimeMs, now); + const endedAtMs = epochUnixMs(options.endTimeMs, startedAtMs); + const method = String(options.method ?? context?.method ?? "GET").toUpperCase(); + const route = String(options.route ?? context?.route ?? "/").split("?")[0] || "/"; + const statusCode = Number.isInteger(Number(options.statusCode)) ? Number(options.statusCode) : 0; + const statusClassValue = statusClassFor(statusCode); + const errorCode = options.errorCode ? String(options.errorCode) : null; + const isError = statusCode >= 500; + const name = `${method} ${route}`; + const attributes = { + "otel.trace_id": traceId, + "request.id": context?.requestId ?? null, + "http.request.method": method, + "http.route": route, + "http.response.status_code": statusCode || null, + "http.response.status_class": statusClassValue, + "hwlab.http.stage": "http.server.request", + ...(errorCode ? { "error.code": errorCode, "error.category": options.errorCategory ?? null, "error.layer": options.errorLayer ?? null } : {}) + }; + const body = { + resourceSpans: [{ + resource: { attributes: attributesFromRecord(resourceAttributes(env)) }, + scopeSpans: [{ + scope: { name: "hwlab.http.server", version: "1" }, + spans: [{ + traceId, + spanId, + ...(parentSpanId ? { parentSpanId } : {}), + name, + kind: 2, + startTimeUnixNano: unixNano(startedAtMs), + endTimeUnixNano: unixNano(Math.max(startedAtMs, endedAtMs)), + attributes: attributesFromRecord(attributes), + status: { + code: isError ? 2 : 1, + ...(isError && options.statusMessage ? { message: String(options.statusMessage).slice(0, 300) } : {}) + } + }] + }] + }] + }; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), OTLP_TIMEOUT_MS); + try { + const response = await fetch(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: controller.signal + }); + return { ok: response.ok, status: response.status, traceId, valuesPrinted: false }; + } catch (error) { + return { ok: false, error: error?.name === "AbortError" ? "otlp-timeout" : "otlp-send-failed", traceId, valuesPrinted: false }; + } finally { + clearTimeout(timeout); + } +} + +function statusClassFor(statusCode) { + if (statusCode >= 500) return "5xx"; + if (statusCode >= 400) return "4xx"; + if (statusCode >= 300) return "3xx"; + if (statusCode >= 200) return "2xx"; + if (statusCode >= 100) return "1xx"; + return "unknown"; +} + function resolveOtlpTracesEndpoint(env = process.env) { const explicit = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT); if (explicit) return explicit.replace(/\/+$/u, ""); diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index 892f9f21..fe85dd61 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -73,7 +73,7 @@ import { handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./ser import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts"; import { handleM3IoControlHttp } from "./server-m3-http.ts"; import { handleSkillsHttp } from "./server-skills-http.ts"; -import { httpRequestOtelTraceContext } from "./otel-trace.ts"; +import { emitHttpServerRequestSpan, httpRequestOtelTraceContext } from "./otel-trace.ts"; import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts"; import { handleProviderProfileCatalogHttp, handleProviderProfilesHttp } from "./provider-profile-management.ts"; import { @@ -131,6 +131,23 @@ export function createCloudApiServer(options = {}) { const requestContext = buildHttpRequestContext(request); attachHttpRequestContext(request, response, requestContext); const backendPerformance = backendPerformanceStore.beginHttpRequest(request, response); + response.once("finish", () => { + const httpCtx = request.hwlabHttpRequestContext; + if (!httpCtx?.traceId) return; + if (isOtelNoisyProbeRoute(httpCtx.route)) return; + const lastError = httpCtx.lastHttpError; + void emitHttpServerRequestSpan(httpCtx, env, { + startTimeMs: backendPerformance.startedAt, + endTimeMs: Date.now(), + statusCode: Number(response.statusCode ?? backendPerformance.statusCode ?? 0), + method: backendPerformance.method || httpCtx.method, + route: backendPerformance.route || httpCtx.route, + errorCode: lastError?.code, + errorCategory: lastError?.category, + errorLayer: lastError?.layer, + statusMessage: lastError?.code + }).catch(() => undefined); + }); await withBackendPerformanceContext(backendPerformance, async () => { try { await routeRequest(request, response, { ...options, env, runtimeStore, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance }); @@ -171,6 +188,11 @@ function buildHttpRequestContext(request) { }; } +function isOtelNoisyProbeRoute(route) { + const path = String(route ?? "/").split("?")[0] || "/"; + return path === "/metrics" || path.startsWith("/health"); +} + function attachHttpRequestContext(request, response, context) { request.hwlabHttpRequestContext = context; response.hwlabHttpRequestContext = context;