feat: implement horizontal scaling optimizations for authz service
- Add /health and /ready endpoints for load balancer health checks - Replace in-memory JWT token cache with Redis for multi-replica support - Reduce DB connection pool from 100 to 25 connections per replica - Add distributed rate limiting (100 req/min + 20 burst) using Redis - Implement circuit breakers for DB and Redis to prevent cascading failures This enables the service to scale horizontally with multiple replicas behind a load balancer without exhausting database connections or maintaining separate token caches per instance.
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CircuitState represents the state of a circuit breaker
|
||||
type CircuitState int
|
||||
|
||||
const (
|
||||
StateClosed CircuitState = iota
|
||||
StateOpen
|
||||
StateHalfOpen
|
||||
)
|
||||
|
||||
// CircuitBreaker implements the circuit breaker pattern
|
||||
type CircuitBreaker struct {
|
||||
name string
|
||||
maxFailures int
|
||||
timeout time.Duration
|
||||
resetTimeout time.Duration
|
||||
failures int
|
||||
lastFailureTime time.Time
|
||||
state CircuitState
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
// NewCircuitBreaker creates a new circuit breaker
|
||||
func NewCircuitBreaker(name string, maxFailures int, timeout time.Duration) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
name: name,
|
||||
maxFailures: maxFailures,
|
||||
timeout: timeout,
|
||||
resetTimeout: 30 * time.Second,
|
||||
state: StateClosed,
|
||||
}
|
||||
}
|
||||
|
||||
// Call executes the given function with circuit breaker protection
|
||||
func (cb *CircuitBreaker) Call(fn func() error) error {
|
||||
cb.mutex.Lock()
|
||||
|
||||
// Check if circuit should transition from Open to HalfOpen
|
||||
if cb.state == StateOpen {
|
||||
if time.Since(cb.lastFailureTime) > cb.resetTimeout {
|
||||
cb.state = StateHalfOpen
|
||||
cb.failures = 0
|
||||
} else {
|
||||
cb.mutex.Unlock()
|
||||
return &CircuitBreakerError{
|
||||
Name: cb.name,
|
||||
State: "open",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
currentState := cb.state
|
||||
cb.mutex.Unlock()
|
||||
|
||||
// Execute the function
|
||||
err := fn()
|
||||
|
||||
cb.mutex.Lock()
|
||||
defer cb.mutex.Unlock()
|
||||
|
||||
if err != nil {
|
||||
cb.failures++
|
||||
cb.lastFailureTime = time.Now()
|
||||
|
||||
if currentState == StateHalfOpen {
|
||||
// If it fails in HalfOpen, go back to Open
|
||||
cb.state = StateOpen
|
||||
} else if cb.failures >= cb.maxFailures {
|
||||
// If too many failures, open the circuit
|
||||
cb.state = StateOpen
|
||||
LogError(err, cb.name+" circuit breaker opened")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Success - reset if in HalfOpen, or reset failure count
|
||||
if cb.state == StateHalfOpen {
|
||||
cb.state = StateClosed
|
||||
cb.failures = 0
|
||||
LogInfo(cb.name + " circuit breaker closed")
|
||||
} else if cb.state == StateClosed && cb.failures > 0 {
|
||||
// Gradually reduce failure count on success
|
||||
cb.failures--
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetState returns the current state of the circuit breaker
|
||||
func (cb *CircuitBreaker) GetState() CircuitState {
|
||||
cb.mutex.RLock()
|
||||
defer cb.mutex.RUnlock()
|
||||
return cb.state
|
||||
}
|
||||
|
||||
// Reset manually resets the circuit breaker
|
||||
func (cb *CircuitBreaker) Reset() {
|
||||
cb.mutex.Lock()
|
||||
defer cb.mutex.Unlock()
|
||||
cb.state = StateClosed
|
||||
cb.failures = 0
|
||||
}
|
||||
|
||||
// CircuitBreakerError represents a circuit breaker error
|
||||
type CircuitBreakerError struct {
|
||||
Name string
|
||||
State string
|
||||
}
|
||||
|
||||
func (e *CircuitBreakerError) Error() string {
|
||||
return "circuit breaker '" + e.Name + "' is " + e.State
|
||||
}
|
||||
|
||||
// IsCircuitBreakerError checks if an error is a circuit breaker error
|
||||
func IsCircuitBreakerError(err error) bool {
|
||||
_, ok := err.(*CircuitBreakerError)
|
||||
return ok
|
||||
}
|
||||
Reference in New Issue
Block a user