fix: expose user billing write readiness (#1148)

internal/userbilling/service.go

@@ -269,10 +269,16 @@ func (s *Server) handleReady(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
 	defer cancel()
 	if err := s.db.PingContext(ctx); err != nil {
-		writeJSON(w, http.StatusServiceUnavailable, map[string]any{"serviceId": serviceID, "status": "degraded", "reason": "database-ping-failed"})
+		s.logDatabaseError("ready_ping", err)
+		writeJSON(w, http.StatusServiceUnavailable, map[string]any{"serviceId": serviceID, "status": "degraded", "reason": "database-ping-failed", "database": map[string]any{"configured": true, "authority": s.config.ExternalDatabaseLabel, "errorClass": classifyDatabaseError(err)}})
 		return
 	}
-	writeJSON(w, http.StatusOK, map[string]any{"serviceId": serviceID, "status": "ready"})
+	if err := s.readyWriteProbe(ctx); err != nil {
+		s.logDatabaseError("ready_write_probe", err)
+		writeJSON(w, http.StatusServiceUnavailable, map[string]any{"serviceId": serviceID, "status": "degraded", "reason": "database-transaction-probe-failed", "database": map[string]any{"configured": true, "authority": s.config.ExternalDatabaseLabel, "errorClass": classifyDatabaseError(err)}})
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"serviceId": serviceID, "status": "ready", "database": map[string]any{"configured": true, "authority": s.config.ExternalDatabaseLabel, "transactionProbe": "write-rollback-ok"}})
 }
 
 func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) {
@@ -303,6 +309,7 @@ func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) {
 	defer cancel()
 	tx, err := s.db.BeginTx(ctx, &sql.TxOptions{})
 	if err != nil {
+		s.logDatabaseError("register_begin_tx", err)
 		writeAPIError(w, http.StatusInternalServerError, "transaction_failed", "could not start transaction")
 		return
 	}
@@ -871,6 +878,62 @@ func (s *Server) RegistrationEnabled() bool {
 	return s.config.RegistrationEnabled
 }
 
+func (s *Server) readyWriteProbe(ctx context.Context) error {
+	tx, err := s.db.BeginTx(ctx, &sql.TxOptions{})
+	if err != nil {
+		return err
+	}
+	probeID := newID("usr_probe")
+	_, err = tx.ExecContext(ctx, `INSERT INTO hwlab_users (id, email, username, display_name, password_hash, status, role) VALUES ($1, $2, $3, 'readiness probe', 'readiness-probe', 'pending', 'user')`, probeID, probeID+"@readiness.invalid", probeID)
+	if err != nil {
+		_ = tx.Rollback()
+		return err
+	}
+	return tx.Rollback()
+}
+
+func (s *Server) logDatabaseError(operation string, err error) {
+	log.Printf("%s database operation=%s errorClass=%s errorCode=%s valuesRedacted=true", serviceID, operation, classifyDatabaseError(err), databaseErrorCode(err))
+}
+
+func classifyDatabaseError(err error) string {
+	if err == nil {
+		return "none"
+	}
+	text := strings.ToLower(err.Error())
+	code := strings.ToUpper(databaseErrorCode(err))
+	if code == "57014" || errors.Is(err, context.DeadlineExceeded) || strings.Contains(text, "deadline exceeded") || strings.Contains(text, "timeout") {
+		return "timeout"
+	}
+	if strings.Contains(text, "certificate") || strings.Contains(text, "ssl") || strings.Contains(text, "tls") {
+		return "ssl"
+	}
+	if strings.Contains(text, "password") || strings.Contains(text, "authentication") || strings.Contains(text, "pg_hba") || code == "28P01" || strings.HasPrefix(code, "28") {
+		return "auth"
+	}
+	if strings.Contains(text, "too many connections") || strings.Contains(text, "connection refused") || strings.Contains(text, "connection reset") || strings.Contains(text, "broken pipe") {
+		return "connection"
+	}
+	return "database"
+}
+
+func databaseErrorCode(err error) string {
+	if err == nil {
+		return ""
+	}
+	type sqlState interface{ SQLState() string }
+	var state sqlState
+	if errors.As(err, &state) {
+		return state.SQLState()
+	}
+	type coder interface{ Code() string }
+	var coded coder
+	if errors.As(err, &coded) {
+		return coded.Code()
+	}
+	return ""
+}
+
 func (s *Server) accountBalance(ctx context.Context, userID string) (int64, int64, error) {
 	var balance, reserved int64
 	err := s.db.QueryRowContext(ctx, `SELECT balance_credits, reserved_credits FROM hwlab_credit_accounts WHERE user_id = $1`, userID).Scan(&balance, &reserved)