feat: implement horizontal scaling optimizations for authz service

- Add /health and /ready endpoints for load balancer health checks
- Replace in-memory JWT token cache with Redis for multi-replica support
- Reduce DB connection pool from 100 to 25 connections per replica
- Add distributed rate limiting (100 req/min + 20 burst) using Redis
- Implement circuit breakers for DB and Redis to prevent cascading failures

This enables the service to scale horizontally with multiple replicas
behind a load balancer without exhausting database connections or
maintaining separate token caches per instance.
This commit is contained in:
2025-12-16 10:03:18 +08:00
parent ee8079e65c
commit 0d8f5b9600
9 changed files with 400 additions and 67 deletions
+16 -5
View File
@@ -1,6 +1,7 @@
package db
import (
"authorization/helper"
"database/sql"
"fmt"
"log"
@@ -13,6 +14,9 @@ import (
// DB is the global database connection pool
var DB *sql.DB
// DBCircuitBreaker protects database operations
var DBCircuitBreaker *helper.CircuitBreaker
func InitDB() (*sql.DB, error) {
// Get connection details from environment variables (loaded in main)
connStr := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true",
@@ -29,13 +33,20 @@ func InitDB() (*sql.DB, error) {
if err != nil {
return nil, fmt.Errorf("error opening database: %v", err)
}
// Set connection pool parameters
DB.SetMaxOpenConns(100) // Maximum number of open connections to the database
DB.SetMaxIdleConns(100) // Maximum number of connections in the idle connection pool
// Initialize circuit breaker
DBCircuitBreaker = helper.NewCircuitBreaker("database", 5, 2*time.Second)
// Set connection pool parameters optimized for horizontal scaling
// Lower per-replica to allow more replicas without exhausting DB connections
DB.SetMaxOpenConns(25) // Maximum number of open connections to the database
DB.SetMaxIdleConns(10) // Maximum number of connections in the idle connection pool
DB.SetConnMaxLifetime(5 * time.Minute) // Maximum amount of time a connection may be reused
// Check if the database connection is working
if err := DB.Ping(); err != nil {
// Check if the database connection is working with circuit breaker
err = DBCircuitBreaker.Call(func() error {
return DB.Ping()
})
if err != nil {
log.Printf("Database connection lost: %v. Reconnecting...", err)
DB, err = InitDB()
if err != nil {