fix: trace opencode provider requests

2026-06-30 11:20:18 +08:00
parent 5863a51188
commit 00ebac6639
9 changed files with 667 additions and 16 deletions
@@ -665,16 +665,20 @@ export async function proxyCloudApi({ request, response, url, cloudApiBaseUrl, c

 async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, opencodeUpstreamUrl, opencodeProxyTimeoutMs, opencodeUsername, opencodePassword, serviceId, sendJson }) {
  const traceContext = cloudWebTraceContext(request);
+  const startedAtMs = Date.now();
  if (!cloudApiBaseUrl) {
+    emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, startedAtMs, statusCode: 503, errorCode: "cloud_api_base_url_missing" });
    sendJsonWithTraceContext(response, 503, { error: "cloud_api_base_url_missing", serviceId, path: url.pathname, diagnostic: opencodeTraceDiagnostic(traceContext) }, traceContext, sendJson);
    return;
  }
  if (!opencodeUpstreamUrl) {
+    emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, startedAtMs, statusCode: 503, errorCode: "opencode_upstream_url_missing" });
    sendJsonWithTraceContext(response, 503, { error: "opencode_upstream_url_missing", serviceId, path: url.pathname, diagnostic: opencodeTraceDiagnostic(traceContext) }, traceContext, sendJson);
    return;
  }
  const authorization = opencodeBasicAuthorization(opencodeUsername, opencodePassword);
  if (!authorization) {
+    emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, startedAtMs, statusCode: 503, errorCode: "opencode_credentials_missing" });
    sendJsonWithTraceContext(response, 503, { error: "opencode_credentials_missing", serviceId, path: url.pathname, diagnostic: opencodeTraceDiagnostic(traceContext) }, traceContext, sendJson);
    return;
  }
@@ -702,6 +706,17 @@ async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, c
        valuesPrinted: false
      }
    });
+    emitOpencodeProxySpanAsync({
+      traceContext,
+      serviceId,
+      request,
+      url,
+      startedAtMs,
+      statusCode: session.statusCode === 403 ? 403 : 401,
+      errorCode: "opencode_auth_required",
+      ticketAuth,
+      sessionStatusCode: session.statusCode || 0
+    });
    sendJsonWithTraceContext(response, session.statusCode === 403 ? 403 : 401, {
      authenticated: false,
      error: "opencode_auth_required",
@@ -727,7 +742,7 @@ async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, c
    headers: opencodeUpstreamHeaders({ headers: { ...request.headers, ...cloudWebTraceHeaders(traceContext, serviceId) } }, body, authorization)
  };
  try {
-    await proxyCloudApiRequest({
+    const proxyResult = await proxyCloudApiRequest({
      target,
      request: upstreamRequest,
      response,
@@ -735,11 +750,36 @@ async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, c
      timeoutMs: opencodeProxyTimeoutMs,
      extraResponseHeaders
    });
+    emitOpencodeProxySpanAsync({
+      traceContext,
+      serviceId,
+      request,
+      url,
+      target,
+      startedAtMs,
+      statusCode: proxyResult?.statusCode || 0,
+      errorCode: proxyResult?.errorCode || "",
+      ticketAuth,
+      sessionStatusCode: session.statusCode || 0,
+      streaming: proxyResult?.streaming === true
+    });
  } catch (error) {
    if (response.headersSent || response.writableEnded) {
      if (!response.writableEnded) response.destroy(error);
      return;
    }
+    emitOpencodeProxySpanAsync({
+      traceContext,
+      serviceId,
+      request,
+      url,
+      target,
+      startedAtMs,
+      statusCode: 502,
+      errorCode: error?.timedOut ? "opencode_proxy_timeout" : "opencode_proxy_unavailable",
+      ticketAuth,
+      sessionStatusCode: session.statusCode || 0
+    });
    sendJsonWithTraceContext(response, 502, {
      status: "failed",
      error: "opencode_proxy_unavailable",
@@ -876,6 +916,131 @@ function opencodeTraceDiagnostic(traceContext, extra = {}) {
  };
 }

+function emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, target = null, startedAtMs, statusCode = 0, errorCode = "", ticketAuth = null, sessionStatusCode = 0, streaming = false } = {}) {
+  const endpoint = otelTracesEndpoint(process.env);
+  if (!endpoint || !traceContext?.traceId || !traceContext?.spanId) return;
+  const endedAtMs = Date.now();
+  const route = opencodeProxyRoute(url?.pathname || "");
+  const span = {
+    traceId: traceContext.traceId,
+    spanId: traceContext.spanId,
+    parentSpanId: traceContext.parentSpanId || "",
+    name: "opencode.proxy.request",
+    kind: 2,
+    startTimeMs: startedAtMs || endedAtMs,
+    endTimeMs: endedAtMs,
+    statusCode,
+    errorCode,
+    attributes: {
+      "http.request.method": request?.method || "GET",
+      "http.route": route,
+      "http.response.status_code": statusCode || undefined,
+      "http.response.status_class": statusCode ? `${Math.floor(statusCode / 100)}xx` : "unknown",
+      "url.scheme": target?.protocol ? target.protocol.replace(":", "") : undefined,
+      "server.address": target?.hostname || undefined,
+      "server.port": target ? Number(target.port || (target.protocol === "https:" ? 443 : 80)) : undefined,
+      "opencode.proxy.duration_ms": Math.max(0, endedAtMs - (startedAtMs || endedAtMs)),
+      "opencode.proxy.ticket_present": ticketAuth?.ticketPresent === true,
+      "opencode.proxy.ticket_accepted": Boolean(ticketAuth?.ticket),
+      "opencode.proxy.session_status_code": sessionStatusCode || undefined,
+      "opencode.proxy.streaming": streaming === true,
+      valuesPrinted: false
+    }
+  };
+  setTimeout(() => {
+    emitOtelSpan(endpoint, span, serviceId || "hwlab-cloud-web", "hwlab.cloud_web.opencode_proxy").catch((error) => {
+      process.stderr.write(JSON.stringify({
+        event: "cloud-web-opencode-otel-emit-failed",
+        serviceId: serviceId || "hwlab-cloud-web",
+        traceId: traceContext.traceId,
+        requestId: traceContext.requestId,
+        error: error instanceof Error ? error.message : String(error),
+        valuesPrinted: false
+      }) + "\n");
+    });
+  }, 0);
+}
+
+function opencodeProxyRoute(pathname) {
+  const path = String(pathname || "/");
+  if (path === "/global/event") return "/global/event";
+  if (path === "/project" || path === "/project/current" || path === "/project/git/init") return path;
+  if (path === "/provider" || path === "/path" || path === "/global/health") return path;
+  if (/^\/session\/[^/]+\/prompt_async$/u.test(path)) return "/session/:sessionID/prompt_async";
+  if (/^\/session\/[^/]+\/message$/u.test(path)) return "/session/:sessionID/message";
+  if (/^\/session\/[^/]+\/abort$/u.test(path)) return "/session/:sessionID/abort";
+  if (/^\/session\/[^/]+(?:\/.*)?$/u.test(path)) return "/session/:sessionID/*";
+  if (path.startsWith("/assets/")) return "/assets/*";
+  return path;
+}
+
+async function emitOtelSpan(endpoint, span, serviceId, scopeName) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 1500);
+  try {
+    await fetch(endpoint, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(otelPayload(span, serviceId, scopeName)),
+      signal: controller.signal
+    });
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function otelPayload(span, serviceId, scopeName) {
+  return {
+    resourceSpans: [{
+      resource: { attributes: otelAttributes({ "service.name": serviceId, "deployment.environment": process.env.HWLAB_ENVIRONMENT || "v03" }) },
+      scopeSpans: [{
+        scope: { name: scopeName, version: "1" },
+        spans: [{
+          traceId: span.traceId,
+          spanId: span.spanId,
+          parentSpanId: span.parentSpanId || undefined,
+          name: span.name,
+          kind: span.kind,
+          startTimeUnixNano: unixNano(span.startTimeMs),
+          endTimeUnixNano: unixNano(span.endTimeMs),
+          attributes: otelAttributes({ ...span.attributes, "otel.trace_id": span.traceId }),
+          status: span.errorCode || (span.statusCode && span.statusCode >= 500) ? { code: 2, message: span.errorCode || `http_${span.statusCode}` } : { code: 1 }
+        }]
+      }]
+    }]
+  };
+}
+
+function otelAttributes(values) {
+  const result = [];
+  for (const [key, value] of Object.entries(values || {})) {
+    if (value === undefined || value === null || value === "") continue;
+    if (typeof value === "number" && Number.isFinite(value)) result.push({ key, value: Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value } });
+    else if (typeof value === "boolean") result.push({ key, value: { boolValue: value } });
+    else result.push({ key, value: { stringValue: String(value) } });
+  }
+  return result;
+}
+
+function unixNano(ms) {
+  return String(BigInt(Math.max(0, Math.trunc(ms))) * 1000000n);
+}
+
+function otelTracesEndpoint(env) {
+  const explicit = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT);
+  if (explicit) return explicit;
+  const base = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_ENDPOINT, env.OTEL_EXPORTER_OTLP_ENDPOINT);
+  return base ? `${base.replace(/\/+$/u, "")}/v1/traces` : "";
+}
+
+function firstNonEmpty(...values) {
+  for (const value of values) {
+    const text = String(value || "").trim();
+    if (text) return text;
+  }
+  return "";
+}
+
 function parseOtelTraceparent(value) {
  const match = firstHeaderValue(value).trim().match(OTEL_TRACEPARENT_PATTERN);
  if (!match) return null;