package workbenchruntime import ( "bytes" "context" crand "crypto/rand" "encoding/hex" "encoding/json" "errors" "io" "net/http" "os" "strconv" "strings" "time" ) const ( otelTraceIDZero = "00000000000000000000000000000000" otelSpanIDZero = "0000000000000000" otelSpanKindInternal = 1 otelSpanKindServer = 2 otelExportTimeout = 1500 * time.Millisecond ) type otelTraceContext struct { TraceID string ParentSpanID string ServerSpanID string } type otelTraceContextKey struct{} type queryStageError struct { stage string err error } func (e queryStageError) Error() string { return e.err.Error() } func (e queryStageError) Unwrap() error { return e.err } func wrapQueryStage(stage string, err error) error { if err == nil || strings.TrimSpace(stage) == "" { return err } if queryErrorStage(err) != "" { return err } return queryStageError{stage: stage, err: err} } func queryErrorStage(err error) string { var stageErr queryStageError if errors.As(err, &stageErr) { return stageErr.stage } return "" } func newOtelTraceContext(traceparent string) otelTraceContext { traceID, parentSpanID, ok := parseOtelTraceparent(traceparent) if !ok { traceID = newOtelTraceID() parentSpanID = "" } return otelTraceContext{TraceID: traceID, ParentSpanID: parentSpanID, ServerSpanID: newOtelSpanID()} } func (ctx otelTraceContext) traceparent() string { if !validOtelHex(ctx.TraceID, 32, otelTraceIDZero) || !validOtelHex(ctx.ServerSpanID, 16, otelSpanIDZero) { return "" } return "00-" + ctx.TraceID + "-" + ctx.ServerSpanID + "-01" } func newOtelTraceID() string { return randomOtelHex(16, otelTraceIDZero) } func newOtelSpanID() string { return randomOtelHex(8, otelSpanIDZero) } func randomOtelHex(byteCount int, zero string) string { buf := make([]byte, byteCount) if _, err := crand.Read(buf); err != nil { return strings.TrimSuffix(zero, "0") + "1" } value := hex.EncodeToString(buf) if value == zero { return strings.TrimSuffix(zero, "0") + "1" } return value } func parseOtelTraceparent(value string) (string, string, bool) { parts := strings.Split(strings.ToLower(strings.TrimSpace(value)), "-") if len(parts) != 4 { return "", "", false } traceID := parts[1] spanID := parts[2] if !validOtelHex(traceID, 32, otelTraceIDZero) || !validOtelHex(spanID, 16, otelSpanIDZero) { return "", "", false } return traceID, spanID, true } func validOtelHex(value string, length int, zero string) bool { if len(value) != length || value == zero { return false } for _, ch := range value { if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { return false } } return true } func (s *Server) otelHTTPHandler(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { startedAt := time.Now().UTC() trace := newOtelTraceContext(r.Header.Get("traceparent")) if traceparent := trace.traceparent(); traceparent != "" { w.Header().Set("traceparent", traceparent) w.Header().Set("x-hwlab-otel-trace-id", trace.TraceID) } ctx := context.WithValue(r.Context(), otelTraceContextKey{}, trace) writer := &otelStatusWriter{ResponseWriter: w, statusCode: http.StatusOK} next.ServeHTTP(writer, r.WithContext(ctx)) statusCode := writer.statusCode errorCode := "" if statusCode >= 500 { errorCode = "http_" + strconv.Itoa(statusCode) } s.emitOtelSpanAsync("workbench-runtime.http "+r.Method+" "+r.URL.Path, trace, trace.ServerSpanID, trace.ParentSpanID, otelSpanKindServer, startedAt, map[string]any{ "http.request.method": r.Method, "http.route": r.URL.Path, "http.response.status_code": int64(statusCode), "hwlab.runtime.stage": "http.server.request", "valuesRedacted": true, }, statusCode, errorCode) }) } type otelStatusWriter struct { http.ResponseWriter statusCode int } func (w *otelStatusWriter) WriteHeader(statusCode int) { w.statusCode = statusCode w.ResponseWriter.WriteHeader(statusCode) } func (w *otelStatusWriter) Write(payload []byte) (int, error) { if w.statusCode == 0 { w.statusCode = http.StatusOK } return w.ResponseWriter.Write(payload) } func otelTraceFromContext(ctx context.Context) (otelTraceContext, bool) { trace, ok := ctx.Value(otelTraceContextKey{}).(otelTraceContext) if !ok || !validOtelHex(trace.TraceID, 32, otelTraceIDZero) { return otelTraceContext{}, false } return trace, true } func (s *Server) withOtelInternalSpan(ctx context.Context, name string, attrs map[string]any, fn func(context.Context) error) error { trace, ok := otelTraceFromContext(ctx) if !ok { return fn(ctx) } startedAt := time.Now().UTC() spanID := newOtelSpanID() err := fn(ctx) errorCode := "" if err != nil { errorCode = otelFirst(sqlErrorCode(err), sqlCauseCode(err), queryErrorStage(err), "query_failed") } attributes := map[string]any{ "hwlab.runtime.stage": name, "valuesRedacted": true, } for key, value := range attrs { if value != nil { attributes[key] = value } } if err != nil { attributes["causeStage"] = name } addOtelErrorCauseAttributes(attributes, err) s.emitOtelSpanAsync(name, trace, spanID, trace.ServerSpanID, otelSpanKindInternal, startedAt, attributes, 0, errorCode) return err } func addOtelErrorCauseAttributes(attributes map[string]any, err error) { if err == nil { return } if stage := queryErrorStage(err); stage != "" { attributes["causeStage"] = stage } if code := sqlErrorCode(err); code != "" { attributes["causeCode"] = code return } if code := sqlCauseCode(err); code != "" { attributes["causeCode"] = code } } func sqlCauseCode(err error) string { if err == nil { return "" } message := strings.ToLower(err.Error()) switch { case strings.Contains(message, "invalid input syntax for type json"): return "invalid_json_text" case strings.Contains(message, "cannot cast") && strings.Contains(message, "json"): return "invalid_json_cast" case strings.Contains(message, "context deadline exceeded") || strings.Contains(message, "timeout"): return "query_timeout" case strings.Contains(message, "context canceled") || strings.Contains(message, "canceling statement due to user request"): return "query_context_canceled" case strings.Contains(message, "too many clients") || strings.Contains(message, "remaining connection slots"): return "postgres_connection_slots_exhausted" case strings.Contains(message, "server closed") || strings.Contains(message, "connection reset") || strings.Contains(message, "broken pipe") || strings.Contains(message, "bad connection"): return "postgres_connection_unavailable" case strings.Contains(message, "scan") || strings.Contains(message, "converting driver.value"): return "row_scan_failed" default: return "" } } func (s *Server) emitOtelSpanAsync(name string, trace otelTraceContext, spanID string, parentSpanID string, kind int, startedAt time.Time, attrs map[string]any, httpStatus int, errorCode string) { go s.emitOtelSpan(context.Background(), name, trace, spanID, parentSpanID, kind, startedAt, attrs, httpStatus, errorCode) } func (s *Server) emitOtelSpan(ctx context.Context, name string, trace otelTraceContext, spanID string, parentSpanID string, kind int, startedAt time.Time, attrs map[string]any, httpStatus int, errorCode string) { endpoint := resolveOtelTracesEndpoint() if endpoint == "" || !validOtelHex(trace.TraceID, 32, otelTraceIDZero) || !validOtelHex(spanID, 16, otelSpanIDZero) { return } endedAt := time.Now().UTC() if startedAt.IsZero() { startedAt = endedAt } attributes := map[string]any{"otel.trace_id": trace.TraceID} for key, value := range attrs { if value != nil { attributes[key] = value } } if errorCode != "" { attributes["error.code"] = errorCode } statusCode := 1 status := map[string]any{"code": statusCode} if errorCode != "" || httpStatus >= 500 { statusCode = 2 status = map[string]any{"code": statusCode, "message": otelFirst(errorCode, "request_failed")} } span := map[string]any{ "traceId": trace.TraceID, "spanId": spanID, "name": name, "kind": kind, "startTimeUnixNano": strconv.FormatInt(startedAt.UTC().UnixNano(), 10), "endTimeUnixNano": strconv.FormatInt(endedAt.UnixNano(), 10), "attributes": otelAttributes(attributes), "status": status, } if validOtelHex(parentSpanID, 16, otelSpanIDZero) { span["parentSpanId"] = parentSpanID } body := map[string]any{ "resourceSpans": []any{map[string]any{ "resource": map[string]any{"attributes": otelAttributes(s.otelResourceAttributes())}, "scopeSpans": []any{map[string]any{ "scope": map[string]any{"name": "hwlab.workbench-runtime", "version": "1"}, "spans": []any{span}, }}, }}, } payload, err := json.Marshal(body) if err != nil { return } requestCtx, cancel := context.WithTimeout(ctx, otelExportTimeout) defer cancel() request, err := http.NewRequestWithContext(requestCtx, http.MethodPost, endpoint, bytes.NewReader(payload)) if err != nil { return } request.Header.Set("Content-Type", "application/json") client := http.Client{Timeout: otelExportTimeout} response, err := client.Do(request) if err != nil { return } _, _ = io.Copy(io.Discard, response.Body) _ = response.Body.Close() } func resolveOtelTracesEndpoint() string { if endpoint := otelFirst(os.Getenv("HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"), os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")); endpoint != "" { return strings.TrimRight(endpoint, "/") } if endpoint := otelFirst(os.Getenv("HWLAB_OTEL_EXPORTER_OTLP_ENDPOINT"), os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")); endpoint != "" { return strings.TrimRight(endpoint, "/") + "/v1/traces" } return "" } func (s *Server) otelResourceAttributes() map[string]any { return map[string]any{ "service.name": otelFirst(os.Getenv("OTEL_SERVICE_NAME"), serviceID), "deployment.environment": otelFirst(os.Getenv("HWLAB_ENVIRONMENT"), os.Getenv("HWLAB_RUNTIME_LANE"), "unknown"), "hwlab.lane": otelFirst(os.Getenv("HWLAB_RUNTIME_LANE"), os.Getenv("HWLAB_GITOPS_PROFILE"), "unknown"), "k8s.namespace.name": otelFirst(os.Getenv("POD_NAMESPACE"), os.Getenv("HWLAB_NAMESPACE"), "unknown"), "git.commit": otelFirst(os.Getenv("HWLAB_COMMIT_ID"), os.Getenv("HWLAB_GITOPS_SOURCE_COMMIT"), os.Getenv("HWLAB_REVISION"), "unknown"), } } func otelAttributes(values map[string]any) []map[string]any { attrs := make([]map[string]any, 0, len(values)) for key, value := range values { if key == "" || value == nil { continue } attrs = append(attrs, map[string]any{"key": key, "value": otelAnyValue(value)}) } return attrs } func otelAnyValue(value any) map[string]any { switch typed := value.(type) { case bool: return map[string]any{"boolValue": typed} case int: return map[string]any{"intValue": strconv.FormatInt(int64(typed), 10)} case int64: return map[string]any{"intValue": strconv.FormatInt(typed, 10)} case float64: return map[string]any{"doubleValue": typed} case string: return map[string]any{"stringValue": typed} default: payload, err := json.Marshal(typed) if err != nil { return map[string]any{"stringValue": "{}"} } return map[string]any{"stringValue": string(payload)} } } func otelFirst(values ...string) string { for _, value := range values { if text := strings.TrimSpace(value); text != "" { return text } } return "" }