admin
0d8f5b9600
- Add /health and /ready endpoints for load balancer health checks - Replace in-memory JWT token cache with Redis for multi-replica support - Reduce DB connection pool from 100 to 25 connections per replica - Add distributed rate limiting (100 req/min + 20 burst) using Redis - Implement circuit breakers for DB and Redis to prevent cascading failures This enables the service to scale horizontally with multiple replicas behind a load balancer without exhausting database connections or maintaining separate token caches per instance.
126 lines
2.7 KiB
Go
126 lines
2.7 KiB
Go
package helper
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// CircuitState represents the state of a circuit breaker
|
|
type CircuitState int
|
|
|
|
const (
|
|
StateClosed CircuitState = iota
|
|
StateOpen
|
|
StateHalfOpen
|
|
)
|
|
|
|
// CircuitBreaker implements the circuit breaker pattern
|
|
type CircuitBreaker struct {
|
|
name string
|
|
maxFailures int
|
|
timeout time.Duration
|
|
resetTimeout time.Duration
|
|
failures int
|
|
lastFailureTime time.Time
|
|
state CircuitState
|
|
mutex sync.RWMutex
|
|
}
|
|
|
|
// NewCircuitBreaker creates a new circuit breaker
|
|
func NewCircuitBreaker(name string, maxFailures int, timeout time.Duration) *CircuitBreaker {
|
|
return &CircuitBreaker{
|
|
name: name,
|
|
maxFailures: maxFailures,
|
|
timeout: timeout,
|
|
resetTimeout: 30 * time.Second,
|
|
state: StateClosed,
|
|
}
|
|
}
|
|
|
|
// Call executes the given function with circuit breaker protection
|
|
func (cb *CircuitBreaker) Call(fn func() error) error {
|
|
cb.mutex.Lock()
|
|
|
|
// Check if circuit should transition from Open to HalfOpen
|
|
if cb.state == StateOpen {
|
|
if time.Since(cb.lastFailureTime) > cb.resetTimeout {
|
|
cb.state = StateHalfOpen
|
|
cb.failures = 0
|
|
} else {
|
|
cb.mutex.Unlock()
|
|
return &CircuitBreakerError{
|
|
Name: cb.name,
|
|
State: "open",
|
|
}
|
|
}
|
|
}
|
|
|
|
currentState := cb.state
|
|
cb.mutex.Unlock()
|
|
|
|
// Execute the function
|
|
err := fn()
|
|
|
|
cb.mutex.Lock()
|
|
defer cb.mutex.Unlock()
|
|
|
|
if err != nil {
|
|
cb.failures++
|
|
cb.lastFailureTime = time.Now()
|
|
|
|
if currentState == StateHalfOpen {
|
|
// If it fails in HalfOpen, go back to Open
|
|
cb.state = StateOpen
|
|
} else if cb.failures >= cb.maxFailures {
|
|
// If too many failures, open the circuit
|
|
cb.state = StateOpen
|
|
LogError(err, cb.name+" circuit breaker opened")
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
// Success - reset if in HalfOpen, or reset failure count
|
|
if cb.state == StateHalfOpen {
|
|
cb.state = StateClosed
|
|
cb.failures = 0
|
|
LogInfo(cb.name + " circuit breaker closed")
|
|
} else if cb.state == StateClosed && cb.failures > 0 {
|
|
// Gradually reduce failure count on success
|
|
cb.failures--
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetState returns the current state of the circuit breaker
|
|
func (cb *CircuitBreaker) GetState() CircuitState {
|
|
cb.mutex.RLock()
|
|
defer cb.mutex.RUnlock()
|
|
return cb.state
|
|
}
|
|
|
|
// Reset manually resets the circuit breaker
|
|
func (cb *CircuitBreaker) Reset() {
|
|
cb.mutex.Lock()
|
|
defer cb.mutex.Unlock()
|
|
cb.state = StateClosed
|
|
cb.failures = 0
|
|
}
|
|
|
|
// CircuitBreakerError represents a circuit breaker error
|
|
type CircuitBreakerError struct {
|
|
Name string
|
|
State string
|
|
}
|
|
|
|
func (e *CircuitBreakerError) Error() string {
|
|
return "circuit breaker '" + e.Name + "' is " + e.State
|
|
}
|
|
|
|
// IsCircuitBreakerError checks if an error is a circuit breaker error
|
|
func IsCircuitBreakerError(err error) bool {
|
|
_, ok := err.(*CircuitBreakerError)
|
|
return ok
|
|
}
|