fix: trace auth login with otel

deploy/deploy.yaml

@@ -482,6 +482,8 @@ lanes:
           HWLAB_USER_BILLING_CREDIT_PER_1K_TOKENS: "1"
           HWLAB_USER_BILLING_MIGRATE_ON_START: "1"
           HWLAB_USER_BILLING_REDIS_URL: redis://hwlab-user-billing-redis.hwlab-v03.svc.cluster.local:6379/0
+          OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces
+          OTEL_SERVICE_NAME: hwlab-user-billing
 services:
   - serviceId: hwlab-cloud-api
     namespace: hwlab-dev

internal/cloud/access-control.ts

@@ -11,6 +11,7 @@ import {
   openFgaObject
 } from "./openfga-authorization.ts";
 import { getHeader, readBody, sendJson, sendRedirect, truthyFlag } from "./server-http-utils.ts";
+import { authLoginOtelTraceContext, emitAuthOtelSpan, newOtelSpanId, traceparentForOtelSpan } from "./otel-trace.ts";
 import { createUserBillingClient } from "./user-billing-client.ts";
 
 const SESSION_COOKIE = "hwlab_session";
@@ -807,26 +808,77 @@ class AccessController {
   }
 
   async handleLogin(request, response) {
-    const body = await jsonBody(request);
-    const userBilling = await this.userBillingLogin(body, request, response);
-    if (userBilling?.handled) return;
-    const password = requiredText(body.password, "password");
-    const username = textOr(body.username, "");
-    if (!username) {
-      if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503));
-      requiredText(body.username, "username");
+    const otelContext = authLoginOtelTraceContext({ traceparent: getHeader(request, "traceparent") });
+    setAuthTraceHeaders(response, otelContext);
+    const startedAtMs = Date.now();
+    let httpStatus = 500;
+    let outcome = "error";
+    let errorCode = "";
+    let dependencyAttributes = {};
+    const mark = (status, nextOutcome, attributes = {}) => {
+      httpStatus = Number(status ?? httpStatus);
+      outcome = nextOutcome;
+      dependencyAttributes = { ...dependencyAttributes, ...attributes };
+    };
+    try {
+      const body = await jsonBody(request);
+      const userBilling = await this.userBillingLogin(body, request, response, otelContext);
+      if (userBilling?.handled) {
+        mark(userBilling.status ?? 200, userBilling.outcome ?? "user_billing", userBillingTraceAttributes(userBilling));
+        return;
+      }
+      const password = requiredText(body.password, "password");
+      const username = textOr(body.username, "");
+      if (!username) {
+        if (userBilling?.retryable) {
+          const status = userBilling.status ?? 503;
+          mark(status, "user_billing_retryable", userBillingTraceAttributes(userBilling));
+          return sendJson(response, status, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", status));
+        }
+        requiredText(body.username, "username");
+      }
+      const user = await this.store.findUserByUsername(username);
+      if (user && (user.status !== "active" || !verifyPassword(user.passwordHash, password))) {
+        mark(401, "invalid_credentials");
+        return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401));
+      }
+      if (!user) {
+        if (userBilling?.retryable) {
+          const status = userBilling.status ?? 503;
+          mark(status, "user_billing_retryable", userBillingTraceAttributes(userBilling));
+          return sendJson(response, status, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", status));
+        }
+        mark(401, "invalid_credentials");
+        return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401));
+      }
+      const { token, session } = await this.issueWebSessionForActor(user);
+      setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env);
+      mark(200, "local_session");
+      return sendJson(response, 200, sessionResponseFromAuth({ actor: user, session: publicSession({ ...session, user }), authMethod: AUTH_METHOD_WEB_SESSION }));
+    } catch (error) {
+      httpStatus = Number(error?.statusCode ?? httpStatus ?? 500);
+      errorCode = textOr(error?.code, "auth_login_error");
+      throw error;
+    } finally {
+      void emitAuthOtelSpan("auth.login", otelContext, this.env, {
+        spanId: otelContext.rootSpanId,
+        parentSpanId: otelContext.parentSpanId,
+        kind: 2,
+        startTimeMs: startedAtMs,
+        endTimeMs: Date.now(),
+        status: httpStatus >= 500 || errorCode ? "error" : "ok",
+        error: errorCode ? new Error(errorCode) : null,
+        attributes: {
+          "http.method": "POST",
+          "http.route": "/auth/login",
+          "http.status_code": httpStatus,
+          "auth.outcome": outcome,
+          "auth.trace_visible": true,
+          ...(errorCode ? { "error.code": errorCode } : {}),
+          ...dependencyAttributes
+        }
+      });
     }
-    const user = await this.store.findUserByUsername(username);
-    if (user && (user.status !== "active" || !verifyPassword(user.passwordHash, password))) {
-      return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401));
-    }
-    if (!user) {
-      if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503));
-      return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401));
-    }
-    const { token, session } = await this.issueWebSessionForActor(user);
-    setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env);
-    return sendJson(response, 200, sessionResponseFromAuth({ actor: user, session: publicSession({ ...session, user }), authMethod: AUTH_METHOD_WEB_SESSION }));
   }
 
   async handleUserBillingRegister(request, response) {
@@ -853,43 +905,80 @@ class AccessController {
     }));
   }
 
-  async userBillingLogin(body, request, response) {
+  async userBillingLogin(body, request, response, otelContext = null) {
     if (!this.userBilling?.configured || typeof this.userBilling.login !== "function") return { handled: false };
     const username = textOr(body.username ?? body.email, "");
     if (!username || !body.password) return { handled: false };
-    const result = await this.issueUserBillingWebSession({ username, password: requiredText(body.password, "password") }, request, response);
+    const result = await this.issueUserBillingWebSession({ username, password: requiredText(body.password, "password") }, request, response, otelContext);
     if (!result.ok) {
-      if (result.status === 401) return { handled: false };
-      if (isRetryableUserBillingResult(result)) return { handled: false, retryable: true, status: result.status ?? 503, result };
+      if (result.status === 401) return { handled: false, status: 401, outcome: "user_billing_invalid_credentials", result };
+      if (isRetryableUserBillingResult(result)) return { handled: false, retryable: true, status: result.status ?? 503, outcome: "user_billing_retryable", result };
       return {
         handled: true,
+        status: result.status ?? 502,
+        outcome: "user_billing_failed",
+        result,
         response: sendJson(response, result.status ?? 502, userBillingDependencyErrorPayload(result, "user_billing_login_failed", "user-billing login failed", result.status ?? 502))
       };
     }
     sendJson(response, 200, sessionResponseFromAuth(result.auth));
-    return { handled: true };
+    return { handled: true, status: 200, outcome: "user_billing_session", result };
   }
 
-  async issueUserBillingWebSession({ username, password }, request, response) {
-    const login = await this.userBilling.login({ username, email: username, password });
-    if (!login.ok || !login.body?.token) {
-      return {
-        ok: false,
-        status: login.status,
-        error: login.error,
-        retryable: login.retryable,
-        retryCount: login.retryCount,
-        transientObserved: login.transientObserved,
-        valuesRedacted: login.valuesRedacted
-      };
+  async issueUserBillingWebSession({ username, password }, request, response, otelContext = null) {
+    const spanId = otelContext ? newOtelSpanId() : "";
+    const startedAtMs = Date.now();
+    let login = null;
+    let thrownCode = "";
+    try {
+      login = await this.userBilling.login({ username, email: username, password }, otelContext ? { traceparent: traceparentForOtelSpan(otelContext, spanId) } : {});
+      if (!login.ok || !login.body?.token) {
+        return {
+          ok: false,
+          status: login.status,
+          error: login.error,
+          retryable: login.retryable,
+          retryCount: login.retryCount,
+          transientObserved: login.transientObserved,
+          valuesRedacted: login.valuesRedacted
+        };
+      }
+      const token = textOr(login.body.token, "");
+      const auth = await this.authenticateUserBillingToken(token);
+      if (!auth.ok) {
+        return { ok: false, status: auth.status, error: auth.error };
+      }
+      setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env);
+      return { ok: true, auth, token, status: login.status, retryCount: login.retryCount, transientObserved: login.transientObserved };
+    } catch (error) {
+      thrownCode = textOr(error?.code, "user_billing_login_exception");
+      throw error;
+    } finally {
+      if (otelContext) {
+        const status = Number(login?.status ?? (thrownCode ? 503 : 0));
+        const errorCode = textOr(thrownCode || login?.error?.code, "");
+        void emitAuthOtelSpan("auth.user_billing.login", otelContext, this.env, {
+          spanId,
+          parentSpanId: otelContext.rootSpanId,
+          kind: 3,
+          startTimeMs: startedAtMs,
+          endTimeMs: Date.now(),
+          status: login?.ok && !errorCode ? "ok" : "error",
+          error: errorCode ? new Error(errorCode) : null,
+          attributes: {
+            "http.method": "POST",
+            "http.route": "/v1/auth/login",
+            "http.status_code": status,
+            "server.address": "hwlab-user-billing",
+            "user_billing.service_id": "hwlab-user-billing",
+            "user_billing.retry_count": nonNegativeResultInteger(login?.retryCount),
+            "user_billing.transient_observed": Boolean(login?.transientObserved),
+            "user_billing.values_redacted": true,
+            ...(errorCode ? { "error.code": errorCode } : {})
+          }
+        });
+      }
     }
-    const token = textOr(login.body.token, "");
-    const auth = await this.authenticateUserBillingToken(token);
-    if (!auth.ok) {
-      return { ok: false, status: auth.status, error: auth.error };
-    }
-    setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env);
-    return { ok: true, auth, token };
   }
 
   async handleLogout(request, response) {
@@ -2076,6 +2165,28 @@ function shouldUseSecureSessionCookie(request, env = process.env) {
 }
 function errorPayload(code, message, status) { return { ok: false, status, error: { code, message } }; }
 
+function setAuthTraceHeaders(response, context) {
+  if (!context?.traceId || typeof response?.setHeader !== "function") return;
+  response.setHeader("traceparent", context.traceparent);
+  response.setHeader("x-hwlab-otel-trace-id", context.traceId);
+}
+
+function userBillingTraceAttributes(input = {}) {
+  const result = input?.result ?? input;
+  const status = Number(input?.status ?? result?.status ?? 0);
+  const retryable = Boolean(input?.retryable ?? isRetryableUserBillingResult(result));
+  const errorCode = textOr(input?.errorCode ?? result?.error?.code, "");
+  return {
+    "user_billing.service_id": "hwlab-user-billing",
+    "user_billing.retryable": retryable,
+    "user_billing.retry_count": nonNegativeResultInteger(result?.retryCount),
+    "user_billing.transient_observed": Boolean(result?.transientObserved ?? retryable),
+    "user_billing.values_redacted": true,
+    ...(status ? { "user_billing.http_status_code": status } : {}),
+    ...(errorCode ? { "user_billing.error_code": errorCode, "error.code": errorCode } : {})
+  };
+}
+
 function userBillingDependencyErrorPayload(result, fallbackCode, fallbackMessage, fallbackStatus) {
   const status = result?.status ?? fallbackStatus;
   const payload = errorPayload(result?.error?.code ?? fallbackCode, result?.error?.message ?? fallbackMessage, status);

internal/cloud/otel-trace.ts

@@ -48,6 +48,29 @@ export function workbenchUiOtelTraceContext(input = {}) {
   };
 }
 
+export function authLoginOtelTraceContext(input = {}) {
+  const parsed = parseOtelTraceparent(input.traceparent);
+  const traceId = parsed?.traceId ?? nonZeroHex(randomBytes(16).toString("hex"), ZERO_TRACE_ID);
+  const rootSpanId = normalizeOtelSpanId(input.rootSpanId) ?? newOtelSpanId();
+  return {
+    traceId,
+    parentSpanId: parsed?.spanId ?? null,
+    rootSpanId,
+    traceparent: `00-${traceId}-${rootSpanId}-01`,
+    valuesPrinted: false
+  };
+}
+
+export function newOtelSpanId() {
+  return nonZeroHex(randomBytes(8).toString("hex"), ZERO_SPAN_ID);
+}
+
+export function traceparentForOtelSpan(context, spanId) {
+  const traceId = normalizeOtelTraceId(context?.traceId);
+  const normalizedSpanId = normalizeOtelSpanId(spanId);
+  return traceId && normalizedSpanId ? `00-${traceId}-${normalizedSpanId}-01` : "";
+}
+
 export async function emitCodeAgentOtelSpan(name, traceId, env = process.env, options = {}) {
   const endpoint = resolveOtlpTracesEndpoint(env);
   if (!endpoint || typeof fetch !== "function") return { ok: false, skipped: true, reason: "otlp-endpoint-missing", valuesPrinted: false };
@@ -156,6 +179,60 @@ export async function emitWorkbenchUiOtelSpan(name, uiTraceId, env = process.env
   }
 }
 
+export async function emitAuthOtelSpan(name, context, env = process.env, options = {}) {
+  const endpoint = resolveOtlpTracesEndpoint(env);
+  if (!endpoint || typeof fetch !== "function") return { ok: false, skipped: true, reason: "otlp-endpoint-missing", valuesPrinted: false };
+  const traceContext = normalizeOtelTraceId(context?.traceId) ? context : authLoginOtelTraceContext({ traceparent: context?.traceparent });
+  const now = Date.now();
+  const startedAtMs = epochUnixMs(options.startTimeMs, now);
+  const endedAtMs = epochUnixMs(options.endTimeMs, startedAtMs);
+  const spanId = normalizeOtelSpanId(options.spanId) ?? newOtelSpanId();
+  const parentSpanId = normalizeOtelSpanId(options.parentSpanId) ?? null;
+  const statusCode = options.status === "error" || options.error ? 2 : 1;
+  const span = {
+    traceId: traceContext.traceId,
+    spanId,
+    ...(parentSpanId ? { parentSpanId } : {}),
+    name,
+    kind: Number(options.kind ?? 1),
+    startTimeUnixNano: unixNano(startedAtMs),
+    endTimeUnixNano: unixNano(Math.max(startedAtMs, endedAtMs)),
+    attributes: attributesFromRecord({
+      "otel.trace_id": traceContext.traceId,
+      "auth.stage": name,
+      ...options.attributes
+    }),
+    status: {
+      code: statusCode,
+      ...(options.error ? { message: String(options.error?.message ?? options.error).slice(0, 300) } : {})
+    }
+  };
+  const body = {
+    resourceSpans: [{
+      resource: { attributes: attributesFromRecord(resourceAttributes(env)) },
+      scopeSpans: [{
+        scope: { name: "hwlab.auth", version: "1" },
+        spans: [span]
+      }]
+    }]
+  };
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), OTLP_TIMEOUT_MS);
+  try {
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+      signal: controller.signal
+    });
+    return { ok: response.ok, status: response.status, traceId: traceContext.traceId, valuesPrinted: false };
+  } catch (error) {
+    return { ok: false, error: error?.name === "AbortError" ? "otlp-timeout" : "otlp-send-failed", traceId: traceContext.traceId, valuesPrinted: false };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
 function resolveOtlpTracesEndpoint(env = process.env) {
   const explicit = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT);
   if (explicit) return explicit.replace(/\/+$/u, "");
@@ -217,3 +294,11 @@ function normalizeOtelSpanId(value) {
   const text = String(value ?? "").trim().toLowerCase();
   return /^[0-9a-f]{16}$/u.test(text) && text !== ZERO_SPAN_ID ? text : null;
 }
+
+function parseOtelTraceparent(value) {
+  const match = String(value ?? "").trim().toLowerCase().match(/^[0-9a-f]{2}-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/u);
+  if (!match) return null;
+  const traceId = normalizeOtelTraceId(match[1]);
+  const spanId = normalizeOtelSpanId(match[2]);
+  return traceId && spanId ? { traceId, spanId } : null;
+}

internal/cloud/user-billing-client.test.ts

@@ -62,3 +62,20 @@ test("user-billing login retries transient dependency failures only", async () =
   assert.equal(unconfigured.retryable, undefined);
   assert.equal(unexpectedCalls.length, 0);
 });
+
+test("user-billing login propagates traceparent to dependency requests", async () => {
+  const traceparent = "00-11111111111111111111111111111111-2222222222222222-01";
+  const calls = [];
+  const client = createUserBillingClient({
+    env: { HWLAB_USER_BILLING_URL: "http://user-billing.test" },
+    fetchImpl: async (url, init) => {
+      calls.push({ url: String(url), traceparent: init?.headers?.traceparent });
+      return new Response(JSON.stringify({ token: "hws_trace_ok", tokenType: "Bearer" }), { status: 200, headers: { "content-type": "application/json" } });
+    }
+  });
+
+  const result = await client.login({ username: "admin", password: "redacted" }, { traceparent });
+  assert.equal(result.ok, true);
+  assert.equal(calls.length, 1);
+  assert.equal(calls[0].traceparent, traceparent);
+});

internal/cloud/user-billing-client.ts

@@ -14,17 +14,17 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch }
   const loginRetryAttempts = nonNegativeInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS, DEFAULT_LOGIN_RETRY_ATTEMPTS);
   const loginRetryDelayMs = positiveInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS, DEFAULT_LOGIN_RETRY_DELAY_MS);
 
-  async function requestJson(path, { method = "GET", body = null, bearerToken = "" } = {}) {
-    return requestJsonWithRetry(path, { method, body, bearerToken });
+  async function requestJson(path, { method = "GET", body = null, bearerToken = "", traceparent = "" } = {}) {
+    return requestJsonWithRetry(path, { method, body, bearerToken, traceparent });
   }
 
-  async function requestJsonWithRetry(path, { method = "GET", body = null, bearerToken = "", retry = null } = {}) {
+  async function requestJsonWithRetry(path, { method = "GET", body = null, bearerToken = "", traceparent = "", retry = null } = {}) {
     const retryAttempts = nonNegativeInteger(retry?.attempts, 0);
     const retryDelayMs = positiveInteger(retry?.delayMs, 0);
     let retryCount = 0;
     let transientObserved = false;
     for (;;) {
-      const result = await requestJsonOnce(path, { method, body, bearerToken });
+      const result = await requestJsonOnce(path, { method, body, bearerToken, traceparent });
       const retryable = typeof retry?.shouldRetry === "function" ? retry.shouldRetry(result) : false;
       if (!retryable || retryCount >= retryAttempts) {
         if (retryCount > 0 || transientObserved || retryable) {
@@ -38,7 +38,7 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch }
     }
   }
 
-  async function requestJsonOnce(path, { method = "GET", body = null, bearerToken = "" } = {}) {
+  async function requestJsonOnce(path, { method = "GET", body = null, bearerToken = "", traceparent = "" } = {}) {
     if (!baseUrl) {
       return { ok: false, status: 503, error: { code: "user_billing_not_configured", message: "HWLAB user-billing URL is not configured" } };
     }
@@ -48,6 +48,8 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch }
     if (body !== null) headers["content-type"] = "application/json";
     const token = text(bearerToken);
     if (token) headers.authorization = `Bearer ${token}`;
+    const normalizedTraceparent = normalizeTraceparent(traceparent);
+    if (normalizedTraceparent) headers.traceparent = normalizedTraceparent;
     if (internalToken) headers["x-hwlab-internal-token"] = internalToken;
     try {
       const init = {
@@ -122,10 +124,11 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch }
     async register(body = {}) {
       return publicPost("/v1/auth/register", body);
     },
-    async login(body = {}) {
+    async login(body = {}, options = {}) {
       return requestJsonWithRetry("/v1/auth/login", {
         method: "POST",
         body,
+        traceparent: options.traceparent,
         retry: { attempts: loginRetryAttempts, delayMs: loginRetryDelayMs, shouldRetry: isRetryableUserBillingResult }
       });
     },
@@ -309,6 +312,11 @@ function text(value) {
   return String(value ?? "").trim();
 }
 
+function normalizeTraceparent(value) {
+  const textValue = text(value).toLowerCase();
+  return /^[0-9a-f]{2}-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u.test(textValue) ? textValue : "";
+}
+
 function parseJson(value, fallback) {
   if (!value) return fallback;
   try {

internal/cloud/user-billing-integration.test.ts

@@ -1119,3 +1119,43 @@ test("cloud api proxies R6 redeem subscription payment and admin redeem routes",
     await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
   }
 });
+
+test("cloud api exposes auth login OTel trace headers and forwards traceparent", async () => {
+  const calls = [];
+  const sessionToken = "hws_otel_login_session_secret";
+  const userBillingClient = {
+    configured: true,
+    async login(body = {}, options = {}) {
+      calls.push({ op: "login", username: body.username, traceparent: options.traceparent });
+      return { ok: true, status: 200, body: { token: sessionToken, user: { id: "usr_otel_admin", username: "admin", role: "admin", status: "active" } } };
+    },
+    async introspect(token) {
+      calls.push({ op: "introspect", tokenPrefix: token.slice(0, 8) });
+      assert.equal(token, sessionToken);
+      return { ok: true, status: 200, body: { active: true, principal: { userId: "usr_otel_admin", username: "admin", email: "admin@hwlab.local", role: "admin", scopes: ["session"], authType: "session" } } };
+    }
+  };
+  const server = createCloudApiServer({ env: { HWLAB_ACCESS_CONTROL_REQUIRED: "1" }, userBillingClient });
+  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+  try {
+    const { port } = server.address();
+    const upstreamTraceparent = "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01";
+    const login = await fetch(`http://127.0.0.1:${port}/auth/login`, {
+      method: "POST",
+      headers: { "content-type": "application/json", traceparent: upstreamTraceparent },
+      body: JSON.stringify({ username: "admin", password: "redacted-password" })
+    });
+    assert.equal(login.status, 200);
+    const responseTraceparent = login.headers.get("traceparent") ?? "";
+    assert.match(responseTraceparent, /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/u);
+    assert.equal(login.headers.get("x-hwlab-otel-trace-id"), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    assert.equal(responseTraceparent.split("-")[1], "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+
+    const loginCall = calls.find((call) => call.op === "login");
+    assert.match(loginCall?.traceparent ?? "", /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/u);
+    assert.equal(loginCall.traceparent.split("-")[1], "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    assert.notEqual(loginCall.traceparent.split("-")[2], responseTraceparent.split("-")[2]);
+  } finally {
+    await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
+  }
+});

internal/userbilling/otel.go

@@ -0,0 +1,258 @@
+package userbilling
+
+import (
+	"bytes"
+	"context"
+	crand "crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	otelTraceIDZero      = "00000000000000000000000000000000"
+	otelSpanIDZero       = "0000000000000000"
+	otelSpanKindInternal = 1
+	otelSpanKindServer   = 2
+	otelExportTimeout    = 1500 * time.Millisecond
+)
+
+type otelTraceContext struct {
+	TraceID      string
+	ParentSpanID string
+	ServerSpanID string
+}
+
+func newOtelTraceContext(traceparent string) otelTraceContext {
+	traceID, parentSpanID, ok := parseOtelTraceparent(traceparent)
+	if !ok {
+		traceID = newOtelTraceID()
+		parentSpanID = ""
+	}
+	return otelTraceContext{TraceID: traceID, ParentSpanID: parentSpanID, ServerSpanID: newOtelSpanID()}
+}
+
+func (ctx otelTraceContext) traceparent() string {
+	if !validOtelHex(ctx.TraceID, 32, otelTraceIDZero) || !validOtelHex(ctx.ServerSpanID, 16, otelSpanIDZero) {
+		return ""
+	}
+	return "00-" + ctx.TraceID + "-" + ctx.ServerSpanID + "-01"
+}
+
+func newOtelTraceID() string {
+	return randomOtelHex(16, otelTraceIDZero)
+}
+
+func newOtelSpanID() string {
+	return randomOtelHex(8, otelSpanIDZero)
+}
+
+func randomOtelHex(byteCount int, zero string) string {
+	buf := make([]byte, byteCount)
+	if _, err := crand.Read(buf); err != nil {
+		return strings.TrimSuffix(zero, "0") + "1"
+	}
+	value := hex.EncodeToString(buf)
+	if value == zero {
+		return strings.TrimSuffix(zero, "0") + "1"
+	}
+	return value
+}
+
+func parseOtelTraceparent(value string) (string, string, bool) {
+	parts := strings.Split(strings.ToLower(strings.TrimSpace(value)), "-")
+	if len(parts) != 4 {
+		return "", "", false
+	}
+	traceID := parts[1]
+	spanID := parts[2]
+	if !validOtelHex(traceID, 32, otelTraceIDZero) || !validOtelHex(spanID, 16, otelSpanIDZero) {
+		return "", "", false
+	}
+	return traceID, spanID, true
+}
+
+func validOtelHex(value string, length int, zero string) bool {
+	if len(value) != length || value == zero {
+		return false
+	}
+	for _, ch := range value {
+		if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') {
+			return false
+		}
+	}
+	return true
+}
+
+func (s *Server) emitOtelSpanAsync(name string, trace otelTraceContext, spanID string, parentSpanID string, kind int, startedAt time.Time, attrs map[string]any, httpStatus int, errorCode string) {
+	go s.emitOtelSpan(context.Background(), name, trace, spanID, parentSpanID, kind, startedAt, attrs, httpStatus, errorCode)
+}
+
+func (s *Server) emitOtelSpan(ctx context.Context, name string, trace otelTraceContext, spanID string, parentSpanID string, kind int, startedAt time.Time, attrs map[string]any, httpStatus int, errorCode string) {
+	endpoint := resolveOtelTracesEndpoint()
+	if endpoint == "" || !validOtelHex(trace.TraceID, 32, otelTraceIDZero) || !validOtelHex(spanID, 16, otelSpanIDZero) {
+		return
+	}
+	endedAt := time.Now().UTC()
+	if startedAt.IsZero() {
+		startedAt = endedAt
+	}
+	attributes := map[string]any{
+		"otel.trace_id": trace.TraceID,
+	}
+	for key, value := range attrs {
+		if value != nil {
+			attributes[key] = value
+		}
+	}
+	if errorCode != "" {
+		attributes["error.code"] = errorCode
+	}
+	statusCode := 1
+	status := map[string]any{"code": statusCode}
+	if errorCode != "" || httpStatus >= 500 {
+		statusCode = 2
+		status = map[string]any{"code": statusCode, "message": first(errorCode, "request_failed")}
+	}
+	span := map[string]any{
+		"traceId":           trace.TraceID,
+		"spanId":            spanID,
+		"name":              name,
+		"kind":              kind,
+		"startTimeUnixNano": strconv.FormatInt(startedAt.UTC().UnixNano(), 10),
+		"endTimeUnixNano":   strconv.FormatInt(endedAt.UnixNano(), 10),
+		"attributes":        otelAttributes(attributes),
+		"status":            status,
+	}
+	if validOtelHex(parentSpanID, 16, otelSpanIDZero) {
+		span["parentSpanId"] = parentSpanID
+	}
+	body := map[string]any{
+		"resourceSpans": []any{map[string]any{
+			"resource": map[string]any{"attributes": otelAttributes(s.otelResourceAttributes())},
+			"scopeSpans": []any{map[string]any{
+				"scope": map[string]any{"name": "hwlab.user-billing", "version": "1"},
+				"spans": []any{span},
+			}},
+		}},
+	}
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return
+	}
+	requestCtx, cancel := context.WithTimeout(ctx, otelExportTimeout)
+	defer cancel()
+	request, err := http.NewRequestWithContext(requestCtx, http.MethodPost, endpoint, bytes.NewReader(payload))
+	if err != nil {
+		return
+	}
+	request.Header.Set("Content-Type", "application/json")
+	client := http.Client{Timeout: otelExportTimeout}
+	response, err := client.Do(request)
+	if err != nil {
+		return
+	}
+	_, _ = io.Copy(io.Discard, response.Body)
+	_ = response.Body.Close()
+}
+
+func resolveOtelTracesEndpoint() string {
+	if endpoint := first(os.Getenv("HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"), os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")); endpoint != "" {
+		return strings.TrimRight(endpoint, "/")
+	}
+	if endpoint := first(os.Getenv("HWLAB_OTEL_EXPORTER_OTLP_ENDPOINT"), os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")); endpoint != "" {
+		return strings.TrimRight(endpoint, "/") + "/v1/traces"
+	}
+	return ""
+}
+
+func (s *Server) otelResourceAttributes() map[string]any {
+	return map[string]any{
+		"service.name":           first(os.Getenv("OTEL_SERVICE_NAME"), "hwlab-user-billing"),
+		"deployment.environment": first(os.Getenv("HWLAB_ENVIRONMENT"), os.Getenv("HWLAB_RUNTIME_LANE"), "unknown"),
+		"hwlab.lane":             first(os.Getenv("HWLAB_RUNTIME_LANE"), os.Getenv("HWLAB_GITOPS_PROFILE"), "unknown"),
+		"k8s.namespace.name":     first(os.Getenv("POD_NAMESPACE"), os.Getenv("HWLAB_NAMESPACE"), s.config.RuntimeNamespace, "unknown"),
+		"git.commit":             first(os.Getenv("HWLAB_COMMIT_ID"), os.Getenv("HWLAB_GITOPS_SOURCE_COMMIT"), os.Getenv("HWLAB_REVISION"), "unknown"),
+	}
+}
+
+func otelAttributes(values map[string]any) []map[string]any {
+	attrs := make([]map[string]any, 0, len(values))
+	for key, value := range values {
+		if key == "" || value == nil {
+			continue
+		}
+		attrs = append(attrs, map[string]any{"key": key, "value": otelAnyValue(value)})
+	}
+	return attrs
+}
+
+func otelAnyValue(value any) map[string]any {
+	switch typed := value.(type) {
+	case bool:
+		return map[string]any{"boolValue": typed}
+	case int:
+		return map[string]any{"intValue": strconv.FormatInt(int64(typed), 10)}
+	case int64:
+		return map[string]any{"intValue": strconv.FormatInt(typed, 10)}
+	case float64:
+		return map[string]any{"doubleValue": typed}
+	case string:
+		return map[string]any{"stringValue": typed}
+	default:
+		return map[string]any{"stringValue": first(toJSONString(typed), "{}")}
+	}
+}
+
+func toJSONString(value any) string {
+	payload, err := json.Marshal(value)
+	if err != nil {
+		return ""
+	}
+	return string(payload)
+}
+
+type sqlStateError interface {
+	SQLState() string
+}
+
+func sqlErrorCode(err error) string {
+	if err == nil {
+		return ""
+	}
+	var stateErr sqlStateError
+	if errors.As(err, &stateErr) {
+		if code := strings.ToUpper(strings.TrimSpace(stateErr.SQLState())); validSQLState(code) {
+			return code
+		}
+	}
+	const marker = "SQLSTATE "
+	message := strings.ToUpper(err.Error())
+	index := strings.Index(message, marker)
+	if index < 0 || len(message) < index+len(marker)+5 {
+		return ""
+	}
+	code := strings.Trim(message[index+len(marker):index+len(marker)+5], "()[]{} ")
+	if validSQLState(code) {
+		return code
+	}
+	return ""
+}
+
+func validSQLState(code string) bool {
+	if len(code) != 5 {
+		return false
+	}
+	for _, ch := range code {
+		if (ch < '0' || ch > '9') && (ch < 'A' || ch > 'Z') {
+			return false
+		}
+	}
+	return true
+}

internal/userbilling/service.go

@@ -548,7 +548,30 @@ func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
+	trace := newOtelTraceContext(r.Header.Get("traceparent"))
+	if traceparent := trace.traceparent(); traceparent != "" {
+		w.Header().Set("traceparent", traceparent)
+		w.Header().Set("x-hwlab-otel-trace-id", trace.TraceID)
+	}
+	startedAt := time.Now().UTC()
+	status := http.StatusInternalServerError
+	outcome := "error"
+	errorCode := ""
+	defer func() {
+		attrs := map[string]any{
+			"http.method":                  http.MethodPost,
+			"http.route":                   "/v1/auth/login",
+			"http.status_code":             status,
+			"auth.outcome":                 outcome,
+			"user_billing.service_id":      serviceID,
+			"user_billing.values_redacted": true,
+		}
+		s.emitOtelSpanAsync("user-billing.auth.login", trace, trace.ServerSpanID, trace.ParentSpanID, otelSpanKindServer, startedAt, attrs, status, errorCode)
+	}()
 	if !s.databaseAvailable(w) {
+		status = http.StatusServiceUnavailable
+		outcome = "state_backend_not_configured"
+		errorCode = "state_backend_not_configured"
 		return
 	}
 	var req struct {
@@ -557,10 +580,16 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
 		Password string `json:"password"`
 	}
 	if !decodeJSON(w, r, &req) {
+		status = http.StatusBadRequest
+		outcome = "invalid_json"
+		errorCode = "invalid_json"
 		return
 	}
 	login := strings.ToLower(strings.TrimSpace(first(req.Email, req.Username)))
 	if login == "" || req.Password == "" {
+		status = http.StatusBadRequest
+		outcome = "invalid_login"
+		errorCode = "invalid_login"
 		writeAPIError(w, http.StatusBadRequest, "invalid_login", "email or username and password are required")
 		return
 	}
@@ -568,20 +597,45 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
 	defer cancel()
 	var user User
 	var passwordHash string
+	dbStartedAt := time.Now().UTC()
 	err := s.db.QueryRowContext(ctx, `SELECT id, email, username, display_name, password_hash, status, role, email_verified, created_at FROM hwlab_users WHERE lower(email) = $1 OR lower(username) = $1`, login).Scan(&user.ID, &user.Email, &user.Username, &user.DisplayName, &passwordHash, &user.Status, &user.Role, &user.EmailVerified, &user.CreatedAt)
+	dbErrorCode := ""
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		dbErrorCode = first(sqlErrorCode(err), "db_query_failed")
+	}
+	s.emitOtelSpanAsync("user-billing.postgres.login_user", trace, newOtelSpanID(), trace.ServerSpanID, otelSpanKindInternal, dbStartedAt, map[string]any{
+		"db.system":                    "postgresql",
+		"db.operation":                 "SELECT",
+		"db.sql.table":                 "hwlab_users",
+		"db.rows_found":                err == nil,
+		"user_billing.values_redacted": true,
+		"db.error_code":                dbErrorCode,
+	}, 0, dbErrorCode)
 	if errors.Is(err, sql.ErrNoRows) {
+		status = http.StatusUnauthorized
+		outcome = "invalid_credentials"
+		errorCode = "invalid_credentials"
 		writeAPIError(w, http.StatusUnauthorized, "invalid_credentials", "invalid email, username or password")
 		return
 	}
 	if err != nil {
-		writeAPIError(w, http.StatusInternalServerError, "login_failed", "could not load user")
+		status = http.StatusInternalServerError
+		outcome = "db_error"
+		errorCode = first(dbErrorCode, "login_failed")
+		writeAPIError(w, http.StatusInternalServerError, errorCode, "could not load user")
 		return
 	}
 	if !verifyPassword(passwordHash, req.Password) {
+		status = http.StatusUnauthorized
+		outcome = "invalid_credentials"
+		errorCode = "invalid_credentials"
 		writeAPIError(w, http.StatusUnauthorized, "invalid_credentials", "invalid email, username or password")
 		return
 	}
 	if user.Status != "active" {
+		status = http.StatusForbidden
+		outcome = "user_disabled"
+		errorCode = "user_disabled"
 		writeAPIError(w, http.StatusForbidden, "user_disabled", "user is not active")
 		return
 	}
@@ -589,9 +643,14 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
 	expiresAt := time.Now().UTC().Add(s.config.SessionTTL)
 	_, err = s.db.ExecContext(ctx, `INSERT INTO hwlab_user_sessions (id, user_id, token_hash, expires_at) VALUES ($1, $2, $3, $4)`, newID("ses"), user.ID, tokenHash(sessionToken), expiresAt)
 	if err != nil {
-		writeAPIError(w, http.StatusInternalServerError, "session_create_failed", "could not create session")
+		status = http.StatusInternalServerError
+		outcome = "session_create_failed"
+		errorCode = first(sqlErrorCode(err), "session_create_failed")
+		writeAPIError(w, http.StatusInternalServerError, errorCode, "could not create session")
 		return
 	}
+	status = http.StatusOK
+	outcome = "session_created"
 	writeJSON(w, http.StatusOK, map[string]any{"token": sessionToken, "tokenType": "Bearer", "expiresAt": expiresAt, "user": user})
 }