feat: implement horizontal scaling optimizations for authz service

- Add /health and /ready endpoints for load balancer health checks
- Replace in-memory JWT token cache with Redis for multi-replica support
- Reduce DB connection pool from 100 to 25 connections per replica
- Add distributed rate limiting (100 req/min + 20 burst) using Redis
- Implement circuit breakers for DB and Redis to prevent cascading failures

This enables the service to scale horizontally with multiple replicas
behind a load balancer without exhausting database connections or
maintaining separate token caches per instance.
This commit is contained in:
2025-12-16 10:03:18 +08:00
parent ee8079e65c
commit 0d8f5b9600
9 changed files with 400 additions and 67 deletions
+125
View File
@@ -0,0 +1,125 @@
package helper
import (
"sync"
"time"
)
// CircuitState represents the state of a circuit breaker
type CircuitState int
const (
StateClosed CircuitState = iota
StateOpen
StateHalfOpen
)
// CircuitBreaker implements the circuit breaker pattern
type CircuitBreaker struct {
name string
maxFailures int
timeout time.Duration
resetTimeout time.Duration
failures int
lastFailureTime time.Time
state CircuitState
mutex sync.RWMutex
}
// NewCircuitBreaker creates a new circuit breaker
func NewCircuitBreaker(name string, maxFailures int, timeout time.Duration) *CircuitBreaker {
return &CircuitBreaker{
name: name,
maxFailures: maxFailures,
timeout: timeout,
resetTimeout: 30 * time.Second,
state: StateClosed,
}
}
// Call executes the given function with circuit breaker protection
func (cb *CircuitBreaker) Call(fn func() error) error {
cb.mutex.Lock()
// Check if circuit should transition from Open to HalfOpen
if cb.state == StateOpen {
if time.Since(cb.lastFailureTime) > cb.resetTimeout {
cb.state = StateHalfOpen
cb.failures = 0
} else {
cb.mutex.Unlock()
return &CircuitBreakerError{
Name: cb.name,
State: "open",
}
}
}
currentState := cb.state
cb.mutex.Unlock()
// Execute the function
err := fn()
cb.mutex.Lock()
defer cb.mutex.Unlock()
if err != nil {
cb.failures++
cb.lastFailureTime = time.Now()
if currentState == StateHalfOpen {
// If it fails in HalfOpen, go back to Open
cb.state = StateOpen
} else if cb.failures >= cb.maxFailures {
// If too many failures, open the circuit
cb.state = StateOpen
LogError(err, cb.name+" circuit breaker opened")
}
return err
}
// Success - reset if in HalfOpen, or reset failure count
if cb.state == StateHalfOpen {
cb.state = StateClosed
cb.failures = 0
LogInfo(cb.name + " circuit breaker closed")
} else if cb.state == StateClosed && cb.failures > 0 {
// Gradually reduce failure count on success
cb.failures--
}
return nil
}
// GetState returns the current state of the circuit breaker
func (cb *CircuitBreaker) GetState() CircuitState {
cb.mutex.RLock()
defer cb.mutex.RUnlock()
return cb.state
}
// Reset manually resets the circuit breaker
func (cb *CircuitBreaker) Reset() {
cb.mutex.Lock()
defer cb.mutex.Unlock()
cb.state = StateClosed
cb.failures = 0
}
// CircuitBreakerError represents a circuit breaker error
type CircuitBreakerError struct {
Name string
State string
}
func (e *CircuitBreakerError) Error() string {
return "circuit breaker '" + e.Name + "' is " + e.State
}
// IsCircuitBreakerError checks if an error is a circuit breaker error
func IsCircuitBreakerError(err error) bool {
_, ok := err.(*CircuitBreakerError)
return ok
}