feat(otel): 服务端 HTTP request span 追穿 hwlab-cloud-api 503 (#1764)

问题：hwlab-cloud-api 服务端 503（如 workbench 投影存储不可用）在 Tempo
里只有 workbench-ui 浏览器侧 span，没有服务端 request handler span，导致
后端根因（error.code=projection_store_unavailable 等）只能在应用日志里看，
OTel 链路追不到服务端处理层。

修复：
- otel-trace.ts 新增 emitHttpServerRequestSpan（scope hwlab.http.server，
  SPAN_KIND_SERVER），捕获 http.route/method/status_code/status_class、
  request.id，5xx 时落 STATUS_CODE_ERROR 并把 error.code/category/layer
  写入 span 属性与 status.message。
- error-envelope.ts 的 decorateErrorPayloadForHttp 把诊断 error code/
  category/layer 回写到 hwlabHttpRequestContext.lastHttpError，供 finish
  时复用，避免重复解析。
- server.ts 在 response finish 时按 http context + backend performance
  时序发服务端 span；health/metrics 探针路由跳过，避免 OTel 噪声洪泛。

验收：canary 调用 emitHttpServerRequestSpan 发 503 projection_store_unavailable
span 到 D601 platform-infra OTel collector，Tempo 可查到 service=hwlab-cloud-api、
scope=hwlab.http.server、http.response.status_code=503、error.code=
projection_store_unavailable 的 ERROR span。

internal/cloud/error-envelope.ts

@@ -38,6 +38,10 @@ export function decorateErrorPayloadForHttp(payload, options = {}) {
     },
     valuesPrinted: originalError.valuesPrinted === true ? true : false
   };
+  const httpContext = options.context ?? options.response?.hwlabHttpRequestContext ?? options.request?.hwlabHttpRequestContext;
+  if (plainObject(httpContext)) {
+    httpContext.lastHttpError = { code: diagnosticCode, category: diagnostic.category, layer: diagnostic.layer, httpStatus: statusCode, valuesPrinted: false };
+  }
   return { ...payload, error: nextError };
 }
 

internal/cloud/otel-trace.ts

@@ -248,6 +248,81 @@ export async function emitAuthOtelSpan(name, context, env = process.env, options
   }
 }
 
+export async function emitHttpServerRequestSpan(context, env = process.env, options = {}) {
+  const endpoint = resolveOtlpTracesEndpoint(env);
+  if (!endpoint || typeof fetch !== "function") return { ok: false, skipped: true, reason: "otlp-endpoint-missing", valuesPrinted: false };
+  const traceId = normalizeOtelTraceId(context?.traceId);
+  if (!traceId) return { ok: false, skipped: true, reason: "trace-id-missing", valuesPrinted: false };
+  const spanId = normalizeOtelSpanId(options.spanId) ?? newOtelSpanId();
+  const parentSpanId = normalizeOtelSpanId(context?.parentSpanId) ?? null;
+  const now = Date.now();
+  const startedAtMs = epochUnixMs(options.startTimeMs, now);
+  const endedAtMs = epochUnixMs(options.endTimeMs, startedAtMs);
+  const method = String(options.method ?? context?.method ?? "GET").toUpperCase();
+  const route = String(options.route ?? context?.route ?? "/").split("?")[0] || "/";
+  const statusCode = Number.isInteger(Number(options.statusCode)) ? Number(options.statusCode) : 0;
+  const statusClassValue = statusClassFor(statusCode);
+  const errorCode = options.errorCode ? String(options.errorCode) : null;
+  const isError = statusCode >= 500;
+  const name = `${method} ${route}`;
+  const attributes = {
+    "otel.trace_id": traceId,
+    "request.id": context?.requestId ?? null,
+    "http.request.method": method,
+    "http.route": route,
+    "http.response.status_code": statusCode || null,
+    "http.response.status_class": statusClassValue,
+    "hwlab.http.stage": "http.server.request",
+    ...(errorCode ? { "error.code": errorCode, "error.category": options.errorCategory ?? null, "error.layer": options.errorLayer ?? null } : {})
+  };
+  const body = {
+    resourceSpans: [{
+      resource: { attributes: attributesFromRecord(resourceAttributes(env)) },
+      scopeSpans: [{
+        scope: { name: "hwlab.http.server", version: "1" },
+        spans: [{
+          traceId,
+          spanId,
+          ...(parentSpanId ? { parentSpanId } : {}),
+          name,
+          kind: 2,
+          startTimeUnixNano: unixNano(startedAtMs),
+          endTimeUnixNano: unixNano(Math.max(startedAtMs, endedAtMs)),
+          attributes: attributesFromRecord(attributes),
+          status: {
+            code: isError ? 2 : 1,
+            ...(isError && options.statusMessage ? { message: String(options.statusMessage).slice(0, 300) } : {})
+          }
+        }]
+      }]
+    }]
+  };
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), OTLP_TIMEOUT_MS);
+  try {
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+      signal: controller.signal
+    });
+    return { ok: response.ok, status: response.status, traceId, valuesPrinted: false };
+  } catch (error) {
+    return { ok: false, error: error?.name === "AbortError" ? "otlp-timeout" : "otlp-send-failed", traceId, valuesPrinted: false };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function statusClassFor(statusCode) {
+  if (statusCode >= 500) return "5xx";
+  if (statusCode >= 400) return "4xx";
+  if (statusCode >= 300) return "3xx";
+  if (statusCode >= 200) return "2xx";
+  if (statusCode >= 100) return "1xx";
+  return "unknown";
+}
+
 function resolveOtlpTracesEndpoint(env = process.env) {
   const explicit = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT);
   if (explicit) return explicit.replace(/\/+$/u, "");

internal/cloud/server.ts

@@ -73,7 +73,7 @@ import { handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./ser
 import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts";
 import { handleM3IoControlHttp } from "./server-m3-http.ts";
 import { handleSkillsHttp } from "./server-skills-http.ts";
-import { httpRequestOtelTraceContext } from "./otel-trace.ts";
+import { emitHttpServerRequestSpan, httpRequestOtelTraceContext } from "./otel-trace.ts";
 import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts";
 import { handleProviderProfileCatalogHttp, handleProviderProfilesHttp } from "./provider-profile-management.ts";
 import {
@@ -131,6 +131,23 @@ export function createCloudApiServer(options = {}) {
     const requestContext = buildHttpRequestContext(request);
     attachHttpRequestContext(request, response, requestContext);
     const backendPerformance = backendPerformanceStore.beginHttpRequest(request, response);
+    response.once("finish", () => {
+      const httpCtx = request.hwlabHttpRequestContext;
+      if (!httpCtx?.traceId) return;
+      if (isOtelNoisyProbeRoute(httpCtx.route)) return;
+      const lastError = httpCtx.lastHttpError;
+      void emitHttpServerRequestSpan(httpCtx, env, {
+        startTimeMs: backendPerformance.startedAt,
+        endTimeMs: Date.now(),
+        statusCode: Number(response.statusCode ?? backendPerformance.statusCode ?? 0),
+        method: backendPerformance.method || httpCtx.method,
+        route: backendPerformance.route || httpCtx.route,
+        errorCode: lastError?.code,
+        errorCategory: lastError?.category,
+        errorLayer: lastError?.layer,
+        statusMessage: lastError?.code
+      }).catch(() => undefined);
+    });
     await withBackendPerformanceContext(backendPerformance, async () => {
       try {
         await routeRequest(request, response, { ...options, env, runtimeStore, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance });
@@ -171,6 +188,11 @@ function buildHttpRequestContext(request) {
   };
 }
 
+function isOtelNoisyProbeRoute(route) {
+  const path = String(route ?? "/").split("?")[0] || "/";
+  return path === "/metrics" || path.startsWith("/health");
+}
+
 function attachHttpRequestContext(request, response, context) {
   request.hwlabHttpRequestContext = context;
   response.hwlabHttpRequestContext = context;