From 786569eac0a9231db11c8d043a53e6c3ce233e13 Mon Sep 17 00:00:00 2001 From: lyon Date: Sat, 20 Jun 2026 08:37:55 +0800 Subject: [PATCH] fix: trace auth login with otel --- deploy/deploy.yaml | 2 + internal/cloud/access-control.ts | 197 ++++++++++--- internal/cloud/otel-trace.ts | 85 ++++++ internal/cloud/user-billing-client.test.ts | 17 ++ internal/cloud/user-billing-client.ts | 20 +- .../cloud/user-billing-integration.test.ts | 40 +++ internal/userbilling/otel.go | 258 ++++++++++++++++++ internal/userbilling/service.go | 63 ++++- 8 files changed, 631 insertions(+), 51 deletions(-) create mode 100644 internal/userbilling/otel.go diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index 85020d22..a6b60dcc 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -482,6 +482,8 @@ lanes: HWLAB_USER_BILLING_CREDIT_PER_1K_TOKENS: "1" HWLAB_USER_BILLING_MIGRATE_ON_START: "1" HWLAB_USER_BILLING_REDIS_URL: redis://hwlab-user-billing-redis.hwlab-v03.svc.cluster.local:6379/0 + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces + OTEL_SERVICE_NAME: hwlab-user-billing services: - serviceId: hwlab-cloud-api namespace: hwlab-dev diff --git a/internal/cloud/access-control.ts b/internal/cloud/access-control.ts index ebf4c97a..2e335b57 100644 --- a/internal/cloud/access-control.ts +++ b/internal/cloud/access-control.ts @@ -11,6 +11,7 @@ import { openFgaObject } from "./openfga-authorization.ts"; import { getHeader, readBody, sendJson, sendRedirect, truthyFlag } from "./server-http-utils.ts"; +import { authLoginOtelTraceContext, emitAuthOtelSpan, newOtelSpanId, traceparentForOtelSpan } from "./otel-trace.ts"; import { createUserBillingClient } from "./user-billing-client.ts"; const SESSION_COOKIE = "hwlab_session"; @@ -807,26 +808,77 @@ class AccessController { } async handleLogin(request, response) { - const body = await jsonBody(request); - const userBilling = await this.userBillingLogin(body, request, response); - if (userBilling?.handled) return; - const password = requiredText(body.password, "password"); - const username = textOr(body.username, ""); - if (!username) { - if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503)); - requiredText(body.username, "username"); + const otelContext = authLoginOtelTraceContext({ traceparent: getHeader(request, "traceparent") }); + setAuthTraceHeaders(response, otelContext); + const startedAtMs = Date.now(); + let httpStatus = 500; + let outcome = "error"; + let errorCode = ""; + let dependencyAttributes = {}; + const mark = (status, nextOutcome, attributes = {}) => { + httpStatus = Number(status ?? httpStatus); + outcome = nextOutcome; + dependencyAttributes = { ...dependencyAttributes, ...attributes }; + }; + try { + const body = await jsonBody(request); + const userBilling = await this.userBillingLogin(body, request, response, otelContext); + if (userBilling?.handled) { + mark(userBilling.status ?? 200, userBilling.outcome ?? "user_billing", userBillingTraceAttributes(userBilling)); + return; + } + const password = requiredText(body.password, "password"); + const username = textOr(body.username, ""); + if (!username) { + if (userBilling?.retryable) { + const status = userBilling.status ?? 503; + mark(status, "user_billing_retryable", userBillingTraceAttributes(userBilling)); + return sendJson(response, status, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", status)); + } + requiredText(body.username, "username"); + } + const user = await this.store.findUserByUsername(username); + if (user && (user.status !== "active" || !verifyPassword(user.passwordHash, password))) { + mark(401, "invalid_credentials"); + return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401)); + } + if (!user) { + if (userBilling?.retryable) { + const status = userBilling.status ?? 503; + mark(status, "user_billing_retryable", userBillingTraceAttributes(userBilling)); + return sendJson(response, status, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", status)); + } + mark(401, "invalid_credentials"); + return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401)); + } + const { token, session } = await this.issueWebSessionForActor(user); + setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env); + mark(200, "local_session"); + return sendJson(response, 200, sessionResponseFromAuth({ actor: user, session: publicSession({ ...session, user }), authMethod: AUTH_METHOD_WEB_SESSION })); + } catch (error) { + httpStatus = Number(error?.statusCode ?? httpStatus ?? 500); + errorCode = textOr(error?.code, "auth_login_error"); + throw error; + } finally { + void emitAuthOtelSpan("auth.login", otelContext, this.env, { + spanId: otelContext.rootSpanId, + parentSpanId: otelContext.parentSpanId, + kind: 2, + startTimeMs: startedAtMs, + endTimeMs: Date.now(), + status: httpStatus >= 500 || errorCode ? "error" : "ok", + error: errorCode ? new Error(errorCode) : null, + attributes: { + "http.method": "POST", + "http.route": "/auth/login", + "http.status_code": httpStatus, + "auth.outcome": outcome, + "auth.trace_visible": true, + ...(errorCode ? { "error.code": errorCode } : {}), + ...dependencyAttributes + } + }); } - const user = await this.store.findUserByUsername(username); - if (user && (user.status !== "active" || !verifyPassword(user.passwordHash, password))) { - return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401)); - } - if (!user) { - if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503)); - return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401)); - } - const { token, session } = await this.issueWebSessionForActor(user); - setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env); - return sendJson(response, 200, sessionResponseFromAuth({ actor: user, session: publicSession({ ...session, user }), authMethod: AUTH_METHOD_WEB_SESSION })); } async handleUserBillingRegister(request, response) { @@ -853,43 +905,80 @@ class AccessController { })); } - async userBillingLogin(body, request, response) { + async userBillingLogin(body, request, response, otelContext = null) { if (!this.userBilling?.configured || typeof this.userBilling.login !== "function") return { handled: false }; const username = textOr(body.username ?? body.email, ""); if (!username || !body.password) return { handled: false }; - const result = await this.issueUserBillingWebSession({ username, password: requiredText(body.password, "password") }, request, response); + const result = await this.issueUserBillingWebSession({ username, password: requiredText(body.password, "password") }, request, response, otelContext); if (!result.ok) { - if (result.status === 401) return { handled: false }; - if (isRetryableUserBillingResult(result)) return { handled: false, retryable: true, status: result.status ?? 503, result }; + if (result.status === 401) return { handled: false, status: 401, outcome: "user_billing_invalid_credentials", result }; + if (isRetryableUserBillingResult(result)) return { handled: false, retryable: true, status: result.status ?? 503, outcome: "user_billing_retryable", result }; return { handled: true, + status: result.status ?? 502, + outcome: "user_billing_failed", + result, response: sendJson(response, result.status ?? 502, userBillingDependencyErrorPayload(result, "user_billing_login_failed", "user-billing login failed", result.status ?? 502)) }; } sendJson(response, 200, sessionResponseFromAuth(result.auth)); - return { handled: true }; + return { handled: true, status: 200, outcome: "user_billing_session", result }; } - async issueUserBillingWebSession({ username, password }, request, response) { - const login = await this.userBilling.login({ username, email: username, password }); - if (!login.ok || !login.body?.token) { - return { - ok: false, - status: login.status, - error: login.error, - retryable: login.retryable, - retryCount: login.retryCount, - transientObserved: login.transientObserved, - valuesRedacted: login.valuesRedacted - }; + async issueUserBillingWebSession({ username, password }, request, response, otelContext = null) { + const spanId = otelContext ? newOtelSpanId() : ""; + const startedAtMs = Date.now(); + let login = null; + let thrownCode = ""; + try { + login = await this.userBilling.login({ username, email: username, password }, otelContext ? { traceparent: traceparentForOtelSpan(otelContext, spanId) } : {}); + if (!login.ok || !login.body?.token) { + return { + ok: false, + status: login.status, + error: login.error, + retryable: login.retryable, + retryCount: login.retryCount, + transientObserved: login.transientObserved, + valuesRedacted: login.valuesRedacted + }; + } + const token = textOr(login.body.token, ""); + const auth = await this.authenticateUserBillingToken(token); + if (!auth.ok) { + return { ok: false, status: auth.status, error: auth.error }; + } + setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env); + return { ok: true, auth, token, status: login.status, retryCount: login.retryCount, transientObserved: login.transientObserved }; + } catch (error) { + thrownCode = textOr(error?.code, "user_billing_login_exception"); + throw error; + } finally { + if (otelContext) { + const status = Number(login?.status ?? (thrownCode ? 503 : 0)); + const errorCode = textOr(thrownCode || login?.error?.code, ""); + void emitAuthOtelSpan("auth.user_billing.login", otelContext, this.env, { + spanId, + parentSpanId: otelContext.rootSpanId, + kind: 3, + startTimeMs: startedAtMs, + endTimeMs: Date.now(), + status: login?.ok && !errorCode ? "ok" : "error", + error: errorCode ? new Error(errorCode) : null, + attributes: { + "http.method": "POST", + "http.route": "/v1/auth/login", + "http.status_code": status, + "server.address": "hwlab-user-billing", + "user_billing.service_id": "hwlab-user-billing", + "user_billing.retry_count": nonNegativeResultInteger(login?.retryCount), + "user_billing.transient_observed": Boolean(login?.transientObserved), + "user_billing.values_redacted": true, + ...(errorCode ? { "error.code": errorCode } : {}) + } + }); + } } - const token = textOr(login.body.token, ""); - const auth = await this.authenticateUserBillingToken(token); - if (!auth.ok) { - return { ok: false, status: auth.status, error: auth.error }; - } - setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env); - return { ok: true, auth, token }; } async handleLogout(request, response) { @@ -2076,6 +2165,28 @@ function shouldUseSecureSessionCookie(request, env = process.env) { } function errorPayload(code, message, status) { return { ok: false, status, error: { code, message } }; } +function setAuthTraceHeaders(response, context) { + if (!context?.traceId || typeof response?.setHeader !== "function") return; + response.setHeader("traceparent", context.traceparent); + response.setHeader("x-hwlab-otel-trace-id", context.traceId); +} + +function userBillingTraceAttributes(input = {}) { + const result = input?.result ?? input; + const status = Number(input?.status ?? result?.status ?? 0); + const retryable = Boolean(input?.retryable ?? isRetryableUserBillingResult(result)); + const errorCode = textOr(input?.errorCode ?? result?.error?.code, ""); + return { + "user_billing.service_id": "hwlab-user-billing", + "user_billing.retryable": retryable, + "user_billing.retry_count": nonNegativeResultInteger(result?.retryCount), + "user_billing.transient_observed": Boolean(result?.transientObserved ?? retryable), + "user_billing.values_redacted": true, + ...(status ? { "user_billing.http_status_code": status } : {}), + ...(errorCode ? { "user_billing.error_code": errorCode, "error.code": errorCode } : {}) + }; +} + function userBillingDependencyErrorPayload(result, fallbackCode, fallbackMessage, fallbackStatus) { const status = result?.status ?? fallbackStatus; const payload = errorPayload(result?.error?.code ?? fallbackCode, result?.error?.message ?? fallbackMessage, status); diff --git a/internal/cloud/otel-trace.ts b/internal/cloud/otel-trace.ts index 23690eec..fe1da309 100644 --- a/internal/cloud/otel-trace.ts +++ b/internal/cloud/otel-trace.ts @@ -48,6 +48,29 @@ export function workbenchUiOtelTraceContext(input = {}) { }; } +export function authLoginOtelTraceContext(input = {}) { + const parsed = parseOtelTraceparent(input.traceparent); + const traceId = parsed?.traceId ?? nonZeroHex(randomBytes(16).toString("hex"), ZERO_TRACE_ID); + const rootSpanId = normalizeOtelSpanId(input.rootSpanId) ?? newOtelSpanId(); + return { + traceId, + parentSpanId: parsed?.spanId ?? null, + rootSpanId, + traceparent: `00-${traceId}-${rootSpanId}-01`, + valuesPrinted: false + }; +} + +export function newOtelSpanId() { + return nonZeroHex(randomBytes(8).toString("hex"), ZERO_SPAN_ID); +} + +export function traceparentForOtelSpan(context, spanId) { + const traceId = normalizeOtelTraceId(context?.traceId); + const normalizedSpanId = normalizeOtelSpanId(spanId); + return traceId && normalizedSpanId ? `00-${traceId}-${normalizedSpanId}-01` : ""; +} + export async function emitCodeAgentOtelSpan(name, traceId, env = process.env, options = {}) { const endpoint = resolveOtlpTracesEndpoint(env); if (!endpoint || typeof fetch !== "function") return { ok: false, skipped: true, reason: "otlp-endpoint-missing", valuesPrinted: false }; @@ -156,6 +179,60 @@ export async function emitWorkbenchUiOtelSpan(name, uiTraceId, env = process.env } } +export async function emitAuthOtelSpan(name, context, env = process.env, options = {}) { + const endpoint = resolveOtlpTracesEndpoint(env); + if (!endpoint || typeof fetch !== "function") return { ok: false, skipped: true, reason: "otlp-endpoint-missing", valuesPrinted: false }; + const traceContext = normalizeOtelTraceId(context?.traceId) ? context : authLoginOtelTraceContext({ traceparent: context?.traceparent }); + const now = Date.now(); + const startedAtMs = epochUnixMs(options.startTimeMs, now); + const endedAtMs = epochUnixMs(options.endTimeMs, startedAtMs); + const spanId = normalizeOtelSpanId(options.spanId) ?? newOtelSpanId(); + const parentSpanId = normalizeOtelSpanId(options.parentSpanId) ?? null; + const statusCode = options.status === "error" || options.error ? 2 : 1; + const span = { + traceId: traceContext.traceId, + spanId, + ...(parentSpanId ? { parentSpanId } : {}), + name, + kind: Number(options.kind ?? 1), + startTimeUnixNano: unixNano(startedAtMs), + endTimeUnixNano: unixNano(Math.max(startedAtMs, endedAtMs)), + attributes: attributesFromRecord({ + "otel.trace_id": traceContext.traceId, + "auth.stage": name, + ...options.attributes + }), + status: { + code: statusCode, + ...(options.error ? { message: String(options.error?.message ?? options.error).slice(0, 300) } : {}) + } + }; + const body = { + resourceSpans: [{ + resource: { attributes: attributesFromRecord(resourceAttributes(env)) }, + scopeSpans: [{ + scope: { name: "hwlab.auth", version: "1" }, + spans: [span] + }] + }] + }; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), OTLP_TIMEOUT_MS); + try { + const response = await fetch(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: controller.signal + }); + return { ok: response.ok, status: response.status, traceId: traceContext.traceId, valuesPrinted: false }; + } catch (error) { + return { ok: false, error: error?.name === "AbortError" ? "otlp-timeout" : "otlp-send-failed", traceId: traceContext.traceId, valuesPrinted: false }; + } finally { + clearTimeout(timeout); + } +} + function resolveOtlpTracesEndpoint(env = process.env) { const explicit = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT); if (explicit) return explicit.replace(/\/+$/u, ""); @@ -217,3 +294,11 @@ function normalizeOtelSpanId(value) { const text = String(value ?? "").trim().toLowerCase(); return /^[0-9a-f]{16}$/u.test(text) && text !== ZERO_SPAN_ID ? text : null; } + +function parseOtelTraceparent(value) { + const match = String(value ?? "").trim().toLowerCase().match(/^[0-9a-f]{2}-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/u); + if (!match) return null; + const traceId = normalizeOtelTraceId(match[1]); + const spanId = normalizeOtelSpanId(match[2]); + return traceId && spanId ? { traceId, spanId } : null; +} diff --git a/internal/cloud/user-billing-client.test.ts b/internal/cloud/user-billing-client.test.ts index 1beafbe2..72c1ada9 100644 --- a/internal/cloud/user-billing-client.test.ts +++ b/internal/cloud/user-billing-client.test.ts @@ -62,3 +62,20 @@ test("user-billing login retries transient dependency failures only", async () = assert.equal(unconfigured.retryable, undefined); assert.equal(unexpectedCalls.length, 0); }); + +test("user-billing login propagates traceparent to dependency requests", async () => { + const traceparent = "00-11111111111111111111111111111111-2222222222222222-01"; + const calls = []; + const client = createUserBillingClient({ + env: { HWLAB_USER_BILLING_URL: "http://user-billing.test" }, + fetchImpl: async (url, init) => { + calls.push({ url: String(url), traceparent: init?.headers?.traceparent }); + return new Response(JSON.stringify({ token: "hws_trace_ok", tokenType: "Bearer" }), { status: 200, headers: { "content-type": "application/json" } }); + } + }); + + const result = await client.login({ username: "admin", password: "redacted" }, { traceparent }); + assert.equal(result.ok, true); + assert.equal(calls.length, 1); + assert.equal(calls[0].traceparent, traceparent); +}); diff --git a/internal/cloud/user-billing-client.ts b/internal/cloud/user-billing-client.ts index c184fba9..bcfd9e1e 100644 --- a/internal/cloud/user-billing-client.ts +++ b/internal/cloud/user-billing-client.ts @@ -14,17 +14,17 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch } const loginRetryAttempts = nonNegativeInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS, DEFAULT_LOGIN_RETRY_ATTEMPTS); const loginRetryDelayMs = positiveInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS, DEFAULT_LOGIN_RETRY_DELAY_MS); - async function requestJson(path, { method = "GET", body = null, bearerToken = "" } = {}) { - return requestJsonWithRetry(path, { method, body, bearerToken }); + async function requestJson(path, { method = "GET", body = null, bearerToken = "", traceparent = "" } = {}) { + return requestJsonWithRetry(path, { method, body, bearerToken, traceparent }); } - async function requestJsonWithRetry(path, { method = "GET", body = null, bearerToken = "", retry = null } = {}) { + async function requestJsonWithRetry(path, { method = "GET", body = null, bearerToken = "", traceparent = "", retry = null } = {}) { const retryAttempts = nonNegativeInteger(retry?.attempts, 0); const retryDelayMs = positiveInteger(retry?.delayMs, 0); let retryCount = 0; let transientObserved = false; for (;;) { - const result = await requestJsonOnce(path, { method, body, bearerToken }); + const result = await requestJsonOnce(path, { method, body, bearerToken, traceparent }); const retryable = typeof retry?.shouldRetry === "function" ? retry.shouldRetry(result) : false; if (!retryable || retryCount >= retryAttempts) { if (retryCount > 0 || transientObserved || retryable) { @@ -38,7 +38,7 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch } } } - async function requestJsonOnce(path, { method = "GET", body = null, bearerToken = "" } = {}) { + async function requestJsonOnce(path, { method = "GET", body = null, bearerToken = "", traceparent = "" } = {}) { if (!baseUrl) { return { ok: false, status: 503, error: { code: "user_billing_not_configured", message: "HWLAB user-billing URL is not configured" } }; } @@ -48,6 +48,8 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch } if (body !== null) headers["content-type"] = "application/json"; const token = text(bearerToken); if (token) headers.authorization = `Bearer ${token}`; + const normalizedTraceparent = normalizeTraceparent(traceparent); + if (normalizedTraceparent) headers.traceparent = normalizedTraceparent; if (internalToken) headers["x-hwlab-internal-token"] = internalToken; try { const init = { @@ -122,10 +124,11 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch } async register(body = {}) { return publicPost("/v1/auth/register", body); }, - async login(body = {}) { + async login(body = {}, options = {}) { return requestJsonWithRetry("/v1/auth/login", { method: "POST", body, + traceparent: options.traceparent, retry: { attempts: loginRetryAttempts, delayMs: loginRetryDelayMs, shouldRetry: isRetryableUserBillingResult } }); }, @@ -309,6 +312,11 @@ function text(value) { return String(value ?? "").trim(); } +function normalizeTraceparent(value) { + const textValue = text(value).toLowerCase(); + return /^[0-9a-f]{2}-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u.test(textValue) ? textValue : ""; +} + function parseJson(value, fallback) { if (!value) return fallback; try { diff --git a/internal/cloud/user-billing-integration.test.ts b/internal/cloud/user-billing-integration.test.ts index b6d5ce37..e2d53af6 100644 --- a/internal/cloud/user-billing-integration.test.ts +++ b/internal/cloud/user-billing-integration.test.ts @@ -1119,3 +1119,43 @@ test("cloud api proxies R6 redeem subscription payment and admin redeem routes", await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); } }); + +test("cloud api exposes auth login OTel trace headers and forwards traceparent", async () => { + const calls = []; + const sessionToken = "hws_otel_login_session_secret"; + const userBillingClient = { + configured: true, + async login(body = {}, options = {}) { + calls.push({ op: "login", username: body.username, traceparent: options.traceparent }); + return { ok: true, status: 200, body: { token: sessionToken, user: { id: "usr_otel_admin", username: "admin", role: "admin", status: "active" } } }; + }, + async introspect(token) { + calls.push({ op: "introspect", tokenPrefix: token.slice(0, 8) }); + assert.equal(token, sessionToken); + return { ok: true, status: 200, body: { active: true, principal: { userId: "usr_otel_admin", username: "admin", email: "admin@hwlab.local", role: "admin", scopes: ["session"], authType: "session" } } }; + } + }; + const server = createCloudApiServer({ env: { HWLAB_ACCESS_CONTROL_REQUIRED: "1" }, userBillingClient }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const { port } = server.address(); + const upstreamTraceparent = "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01"; + const login = await fetch(`http://127.0.0.1:${port}/auth/login`, { + method: "POST", + headers: { "content-type": "application/json", traceparent: upstreamTraceparent }, + body: JSON.stringify({ username: "admin", password: "redacted-password" }) + }); + assert.equal(login.status, 200); + const responseTraceparent = login.headers.get("traceparent") ?? ""; + assert.match(responseTraceparent, /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/u); + assert.equal(login.headers.get("x-hwlab-otel-trace-id"), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + assert.equal(responseTraceparent.split("-")[1], "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + + const loginCall = calls.find((call) => call.op === "login"); + assert.match(loginCall?.traceparent ?? "", /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/u); + assert.equal(loginCall.traceparent.split("-")[1], "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + assert.notEqual(loginCall.traceparent.split("-")[2], responseTraceparent.split("-")[2]); + } finally { + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + } +}); diff --git a/internal/userbilling/otel.go b/internal/userbilling/otel.go new file mode 100644 index 00000000..7796b901 --- /dev/null +++ b/internal/userbilling/otel.go @@ -0,0 +1,258 @@ +package userbilling + +import ( + "bytes" + "context" + crand "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" +) + +const ( + otelTraceIDZero = "00000000000000000000000000000000" + otelSpanIDZero = "0000000000000000" + otelSpanKindInternal = 1 + otelSpanKindServer = 2 + otelExportTimeout = 1500 * time.Millisecond +) + +type otelTraceContext struct { + TraceID string + ParentSpanID string + ServerSpanID string +} + +func newOtelTraceContext(traceparent string) otelTraceContext { + traceID, parentSpanID, ok := parseOtelTraceparent(traceparent) + if !ok { + traceID = newOtelTraceID() + parentSpanID = "" + } + return otelTraceContext{TraceID: traceID, ParentSpanID: parentSpanID, ServerSpanID: newOtelSpanID()} +} + +func (ctx otelTraceContext) traceparent() string { + if !validOtelHex(ctx.TraceID, 32, otelTraceIDZero) || !validOtelHex(ctx.ServerSpanID, 16, otelSpanIDZero) { + return "" + } + return "00-" + ctx.TraceID + "-" + ctx.ServerSpanID + "-01" +} + +func newOtelTraceID() string { + return randomOtelHex(16, otelTraceIDZero) +} + +func newOtelSpanID() string { + return randomOtelHex(8, otelSpanIDZero) +} + +func randomOtelHex(byteCount int, zero string) string { + buf := make([]byte, byteCount) + if _, err := crand.Read(buf); err != nil { + return strings.TrimSuffix(zero, "0") + "1" + } + value := hex.EncodeToString(buf) + if value == zero { + return strings.TrimSuffix(zero, "0") + "1" + } + return value +} + +func parseOtelTraceparent(value string) (string, string, bool) { + parts := strings.Split(strings.ToLower(strings.TrimSpace(value)), "-") + if len(parts) != 4 { + return "", "", false + } + traceID := parts[1] + spanID := parts[2] + if !validOtelHex(traceID, 32, otelTraceIDZero) || !validOtelHex(spanID, 16, otelSpanIDZero) { + return "", "", false + } + return traceID, spanID, true +} + +func validOtelHex(value string, length int, zero string) bool { + if len(value) != length || value == zero { + return false + } + for _, ch := range value { + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return false + } + } + return true +} + +func (s *Server) emitOtelSpanAsync(name string, trace otelTraceContext, spanID string, parentSpanID string, kind int, startedAt time.Time, attrs map[string]any, httpStatus int, errorCode string) { + go s.emitOtelSpan(context.Background(), name, trace, spanID, parentSpanID, kind, startedAt, attrs, httpStatus, errorCode) +} + +func (s *Server) emitOtelSpan(ctx context.Context, name string, trace otelTraceContext, spanID string, parentSpanID string, kind int, startedAt time.Time, attrs map[string]any, httpStatus int, errorCode string) { + endpoint := resolveOtelTracesEndpoint() + if endpoint == "" || !validOtelHex(trace.TraceID, 32, otelTraceIDZero) || !validOtelHex(spanID, 16, otelSpanIDZero) { + return + } + endedAt := time.Now().UTC() + if startedAt.IsZero() { + startedAt = endedAt + } + attributes := map[string]any{ + "otel.trace_id": trace.TraceID, + } + for key, value := range attrs { + if value != nil { + attributes[key] = value + } + } + if errorCode != "" { + attributes["error.code"] = errorCode + } + statusCode := 1 + status := map[string]any{"code": statusCode} + if errorCode != "" || httpStatus >= 500 { + statusCode = 2 + status = map[string]any{"code": statusCode, "message": first(errorCode, "request_failed")} + } + span := map[string]any{ + "traceId": trace.TraceID, + "spanId": spanID, + "name": name, + "kind": kind, + "startTimeUnixNano": strconv.FormatInt(startedAt.UTC().UnixNano(), 10), + "endTimeUnixNano": strconv.FormatInt(endedAt.UnixNano(), 10), + "attributes": otelAttributes(attributes), + "status": status, + } + if validOtelHex(parentSpanID, 16, otelSpanIDZero) { + span["parentSpanId"] = parentSpanID + } + body := map[string]any{ + "resourceSpans": []any{map[string]any{ + "resource": map[string]any{"attributes": otelAttributes(s.otelResourceAttributes())}, + "scopeSpans": []any{map[string]any{ + "scope": map[string]any{"name": "hwlab.user-billing", "version": "1"}, + "spans": []any{span}, + }}, + }}, + } + payload, err := json.Marshal(body) + if err != nil { + return + } + requestCtx, cancel := context.WithTimeout(ctx, otelExportTimeout) + defer cancel() + request, err := http.NewRequestWithContext(requestCtx, http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + return + } + request.Header.Set("Content-Type", "application/json") + client := http.Client{Timeout: otelExportTimeout} + response, err := client.Do(request) + if err != nil { + return + } + _, _ = io.Copy(io.Discard, response.Body) + _ = response.Body.Close() +} + +func resolveOtelTracesEndpoint() string { + if endpoint := first(os.Getenv("HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"), os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")); endpoint != "" { + return strings.TrimRight(endpoint, "/") + } + if endpoint := first(os.Getenv("HWLAB_OTEL_EXPORTER_OTLP_ENDPOINT"), os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")); endpoint != "" { + return strings.TrimRight(endpoint, "/") + "/v1/traces" + } + return "" +} + +func (s *Server) otelResourceAttributes() map[string]any { + return map[string]any{ + "service.name": first(os.Getenv("OTEL_SERVICE_NAME"), "hwlab-user-billing"), + "deployment.environment": first(os.Getenv("HWLAB_ENVIRONMENT"), os.Getenv("HWLAB_RUNTIME_LANE"), "unknown"), + "hwlab.lane": first(os.Getenv("HWLAB_RUNTIME_LANE"), os.Getenv("HWLAB_GITOPS_PROFILE"), "unknown"), + "k8s.namespace.name": first(os.Getenv("POD_NAMESPACE"), os.Getenv("HWLAB_NAMESPACE"), s.config.RuntimeNamespace, "unknown"), + "git.commit": first(os.Getenv("HWLAB_COMMIT_ID"), os.Getenv("HWLAB_GITOPS_SOURCE_COMMIT"), os.Getenv("HWLAB_REVISION"), "unknown"), + } +} + +func otelAttributes(values map[string]any) []map[string]any { + attrs := make([]map[string]any, 0, len(values)) + for key, value := range values { + if key == "" || value == nil { + continue + } + attrs = append(attrs, map[string]any{"key": key, "value": otelAnyValue(value)}) + } + return attrs +} + +func otelAnyValue(value any) map[string]any { + switch typed := value.(type) { + case bool: + return map[string]any{"boolValue": typed} + case int: + return map[string]any{"intValue": strconv.FormatInt(int64(typed), 10)} + case int64: + return map[string]any{"intValue": strconv.FormatInt(typed, 10)} + case float64: + return map[string]any{"doubleValue": typed} + case string: + return map[string]any{"stringValue": typed} + default: + return map[string]any{"stringValue": first(toJSONString(typed), "{}")} + } +} + +func toJSONString(value any) string { + payload, err := json.Marshal(value) + if err != nil { + return "" + } + return string(payload) +} + +type sqlStateError interface { + SQLState() string +} + +func sqlErrorCode(err error) string { + if err == nil { + return "" + } + var stateErr sqlStateError + if errors.As(err, &stateErr) { + if code := strings.ToUpper(strings.TrimSpace(stateErr.SQLState())); validSQLState(code) { + return code + } + } + const marker = "SQLSTATE " + message := strings.ToUpper(err.Error()) + index := strings.Index(message, marker) + if index < 0 || len(message) < index+len(marker)+5 { + return "" + } + code := strings.Trim(message[index+len(marker):index+len(marker)+5], "()[]{} ") + if validSQLState(code) { + return code + } + return "" +} + +func validSQLState(code string) bool { + if len(code) != 5 { + return false + } + for _, ch := range code { + if (ch < '0' || ch > '9') && (ch < 'A' || ch > 'Z') { + return false + } + } + return true +} diff --git a/internal/userbilling/service.go b/internal/userbilling/service.go index a8c70c27..0c9d8046 100644 --- a/internal/userbilling/service.go +++ b/internal/userbilling/service.go @@ -548,7 +548,30 @@ func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) { } func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { + trace := newOtelTraceContext(r.Header.Get("traceparent")) + if traceparent := trace.traceparent(); traceparent != "" { + w.Header().Set("traceparent", traceparent) + w.Header().Set("x-hwlab-otel-trace-id", trace.TraceID) + } + startedAt := time.Now().UTC() + status := http.StatusInternalServerError + outcome := "error" + errorCode := "" + defer func() { + attrs := map[string]any{ + "http.method": http.MethodPost, + "http.route": "/v1/auth/login", + "http.status_code": status, + "auth.outcome": outcome, + "user_billing.service_id": serviceID, + "user_billing.values_redacted": true, + } + s.emitOtelSpanAsync("user-billing.auth.login", trace, trace.ServerSpanID, trace.ParentSpanID, otelSpanKindServer, startedAt, attrs, status, errorCode) + }() if !s.databaseAvailable(w) { + status = http.StatusServiceUnavailable + outcome = "state_backend_not_configured" + errorCode = "state_backend_not_configured" return } var req struct { @@ -557,10 +580,16 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { Password string `json:"password"` } if !decodeJSON(w, r, &req) { + status = http.StatusBadRequest + outcome = "invalid_json" + errorCode = "invalid_json" return } login := strings.ToLower(strings.TrimSpace(first(req.Email, req.Username))) if login == "" || req.Password == "" { + status = http.StatusBadRequest + outcome = "invalid_login" + errorCode = "invalid_login" writeAPIError(w, http.StatusBadRequest, "invalid_login", "email or username and password are required") return } @@ -568,20 +597,45 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { defer cancel() var user User var passwordHash string + dbStartedAt := time.Now().UTC() err := s.db.QueryRowContext(ctx, `SELECT id, email, username, display_name, password_hash, status, role, email_verified, created_at FROM hwlab_users WHERE lower(email) = $1 OR lower(username) = $1`, login).Scan(&user.ID, &user.Email, &user.Username, &user.DisplayName, &passwordHash, &user.Status, &user.Role, &user.EmailVerified, &user.CreatedAt) + dbErrorCode := "" + if err != nil && !errors.Is(err, sql.ErrNoRows) { + dbErrorCode = first(sqlErrorCode(err), "db_query_failed") + } + s.emitOtelSpanAsync("user-billing.postgres.login_user", trace, newOtelSpanID(), trace.ServerSpanID, otelSpanKindInternal, dbStartedAt, map[string]any{ + "db.system": "postgresql", + "db.operation": "SELECT", + "db.sql.table": "hwlab_users", + "db.rows_found": err == nil, + "user_billing.values_redacted": true, + "db.error_code": dbErrorCode, + }, 0, dbErrorCode) if errors.Is(err, sql.ErrNoRows) { + status = http.StatusUnauthorized + outcome = "invalid_credentials" + errorCode = "invalid_credentials" writeAPIError(w, http.StatusUnauthorized, "invalid_credentials", "invalid email, username or password") return } if err != nil { - writeAPIError(w, http.StatusInternalServerError, "login_failed", "could not load user") + status = http.StatusInternalServerError + outcome = "db_error" + errorCode = first(dbErrorCode, "login_failed") + writeAPIError(w, http.StatusInternalServerError, errorCode, "could not load user") return } if !verifyPassword(passwordHash, req.Password) { + status = http.StatusUnauthorized + outcome = "invalid_credentials" + errorCode = "invalid_credentials" writeAPIError(w, http.StatusUnauthorized, "invalid_credentials", "invalid email, username or password") return } if user.Status != "active" { + status = http.StatusForbidden + outcome = "user_disabled" + errorCode = "user_disabled" writeAPIError(w, http.StatusForbidden, "user_disabled", "user is not active") return } @@ -589,9 +643,14 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { expiresAt := time.Now().UTC().Add(s.config.SessionTTL) _, err = s.db.ExecContext(ctx, `INSERT INTO hwlab_user_sessions (id, user_id, token_hash, expires_at) VALUES ($1, $2, $3, $4)`, newID("ses"), user.ID, tokenHash(sessionToken), expiresAt) if err != nil { - writeAPIError(w, http.StatusInternalServerError, "session_create_failed", "could not create session") + status = http.StatusInternalServerError + outcome = "session_create_failed" + errorCode = first(sqlErrorCode(err), "session_create_failed") + writeAPIError(w, http.StatusInternalServerError, errorCode, "could not create session") return } + status = http.StatusOK + outcome = "session_created" writeJSON(w, http.StatusOK, map[string]any{"token": sessionToken, "tokenType": "Bearer", "expiresAt": expiresAt, "user": user}) }