Circuit Breaker Pattern: защита API от каскадных сбоев

Один упавший микросервис убивает всю систему? Cascading failures разрушают ваш API? В этом руководстве мы разберем Circuit Breaker Pattern — паттерн отказоустойчивости, который предотвращает каскадные сбои в распределенных системах. Вы узнаете про три состояния (Closed/Open/Half-Open), timeout, retry, fallback и реализацию для Node.js, Python, Go.

🔥 Проблема каскадных сбоев

В микросервисной архитектуре один упавший сервис может вызвать cascading failure — каскадный сбой всей системы.

User → API Gateway → Order Service → Payment Service (DOWN ❌)
                         ↓ timeout 30s
                         ↓ retry 3x
                         ↓ thread pool exhausted
                         ↓ Order Service crashes 💥
                         ↓
         API Gateway can't reach Order Service 💥
                         ↓
         Entire system DOWN 🔥

⚡ Как работает Circuit Breaker

Circuit Breaker Pattern работает как электрический автомат: если сервис начинает падать, circuit breaker открывается и блокирует запросы к нему, давая сервису время на восстановление.

User → API Gateway → Order Service → [Circuit Breaker] → Payment Service (DOWN ❌)
                                              ↓
                                         OPEN state
                                              ↓
                                    Block requests immediately
                                              ↓
                                    Return fallback response
                                              ↓
                         Order Service continues working ✅
                         API Gateway continues working ✅
                         System degraded but operational 🟡

🔄 Три состояния Circuit Breaker

🟢 Node.js: Opossum Circuit Breaker

Opossum — самая популярная библиотека для Circuit Breaker в Node.js (от Netflix).

Установка

npm install opossum

Базовый пример

// circuit-breaker.js
const CircuitBreaker = require('opossum');
const axios = require('axios');

// Функция которую защищаем circuit breaker
async function callPaymentService(orderId) {
  const response = await axios.post('https://payment-service/api/charge', {
    orderId,
    amount: 100
  });
  return response.data;
}

// Circuit Breaker options
const options = {
  timeout: 5000,           // Timeout 5 секунд
  errorThresholdPercentage: 50,  // Открыть при 50% ошибок
  resetTimeout: 30000,     // Тестировать восстановление через 30 секунд
  volumeThreshold: 10,     // Минимум 10 запросов для статистики
  rollingCountTimeout: 10000,    // Окно для подсчета ошибок: 10 секунд
};

// Создаем circuit breaker
const breaker = new CircuitBreaker(callPaymentService, options);

// Fallback function (если circuit breaker открыт)
breaker.fallback((orderId) => {
  console.log(`Circuit breaker OPEN, using fallback for order ${orderId}`);
  return {
    status: 'pending',
    message: 'Payment service unavailable, order queued'
  };
});

// Events для мониторинга
breaker.on('open', () => {
  console.log('🔴 Circuit breaker OPEN - blocking requests');
});

breaker.on('halfOpen', () => {
  console.log('🟡 Circuit breaker HALF-OPEN - testing recovery');
});

breaker.on('close', () => {
  console.log('🟢 Circuit breaker CLOSED - normal operation');
});

breaker.on('fallback', (result) => {
  console.log('⚠️  Fallback executed:', result);
});

// Использование
async function processOrder(orderId) {
  try {
    const result = await breaker.fire(orderId);
    console.log('Payment successful:', result);
    return result;
  } catch (error) {
    console.error('Payment failed:', error.message);
    throw error;
  }
}

module.exports = { breaker, processOrder };

Продвинутый пример с Express API

// api.js
const express = require('express');
const CircuitBreaker = require('opossum');
const axios = require('axios');

const app = express();
app.use(express.json());

// Multiple circuit breakers для разных сервисов
const services = {
  payment: createCircuitBreaker(
    (data) => axios.post('https://payment-service/api/charge', data),
    {
      name: 'payment-service',
      fallback: (data) => ({ status: 'queued', queueId: Date.now() })
    }
  ),

  inventory: createCircuitBreaker(
    (productId) => axios.get(`https://inventory-service/api/stock/${productId}`),
    {
      name: 'inventory-service',
      fallback: (productId) => ({ stock: null, cached: true })
    }
  ),

  notification: createCircuitBreaker(
    (data) => axios.post('https://notification-service/api/email', data),
    {
      name: 'notification-service',
      fallback: (data) => ({ queued: true }),
      // Более агрессивные настройки для non-critical сервиса
      errorThresholdPercentage: 25,
      timeout: 2000
    }
  )
};

function createCircuitBreaker(fn, config) {
  const breaker = new CircuitBreaker(fn, {
    timeout: config.timeout || 5000,
    errorThresholdPercentage: config.errorThresholdPercentage || 50,
    resetTimeout: 30000,
    volumeThreshold: 5,
  });

  if (config.fallback) {
    breaker.fallback(config.fallback);
  }

  // Логирование
  breaker.on('open', () =>
    console.log(`🔴 ${config.name} circuit breaker OPEN`)
  );
  breaker.on('halfOpen', () =>
    console.log(`🟡 ${config.name} circuit breaker HALF-OPEN`)
  );
  breaker.on('close', () =>
    console.log(`🟢 ${config.name} circuit breaker CLOSED`)
  );

  return breaker;
}

// API endpoints
app.post('/api/orders', async (req, res) => {
  try {
    const { productId, quantity, userId } = req.body;

    // 1. Check inventory (с circuit breaker)
    const inventory = await services.inventory.fire(productId);

    if (inventory.stock !== null && inventory.stock < quantity) {
      return res.status(400).json({ error: 'Insufficient stock' });
    }

    // 2. Charge payment (с circuit breaker)
    const payment = await services.payment.fire({
      userId,
      amount: quantity * 100
    });

    // 3. Send notification (с circuit breaker, не критично)
    services.notification.fire({
      to: userId,
      subject: 'Order confirmed',
      body: `Order #${Date.now()}`
    }).catch(err => console.log('Notification failed (non-critical):', err));

    res.json({
      orderId: Date.now(),
      payment,
      inventory: inventory.cached ? 'Using cached data' : 'Real-time data'
    });

  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// Health check endpoint с circuit breaker stats
app.get('/api/health', (req, res) => {
  const stats = {};

  Object.entries(services).forEach(([name, breaker]) => {
    stats[name] = {
      state: breaker.opened ? 'OPEN' : (breaker.halfOpen ? 'HALF-OPEN' : 'CLOSED'),
      stats: breaker.stats
    };
  });

  res.json({ status: 'ok', circuitBreakers: stats });
});

app.listen(3000, () => {
  console.log('API running on port 3000');
});

🐍 Python: pybreaker Circuit Breaker

Установка

pip install pybreaker requests

Базовый пример

# circuit_breaker.py
from pybreaker import CircuitBreaker, CircuitBreakerError
import requests
import time

# Создаем circuit breaker
payment_breaker = CircuitBreaker(
    fail_max=5,              # Открыть после 5 ошибок подряд
    timeout_duration=30,     # Тестировать восстановление через 30 секунд
    exclude=[requests.HTTPError],  # Не считать HTTP errors за failure
    name='payment-service'
)

# Функция с circuit breaker декоратором
@payment_breaker
def call_payment_service(order_id, amount):
    """Вызов payment service с circuit breaker защитой"""
    response = requests.post(
        'https://payment-service/api/charge',
        json={'order_id': order_id, 'amount': amount},
        timeout=5
    )
    response.raise_for_status()
    return response.json()

# Fallback функция
def payment_fallback(order_id, amount):
    """Fallback когда circuit breaker открыт"""
    print(f"⚠️  Payment service unavailable, queueing order {order_id}")
    return {
        'status': 'queued',
        'order_id': order_id,
        'message': 'Payment will be processed when service recovers'
    }

# Использование с fallback
def process_payment(order_id, amount):
    try:
        result = call_payment_service(order_id, amount)
        print(f"✅ Payment successful: {result}")
        return result
    except CircuitBreakerError:
        # Circuit breaker открыт, используем fallback
        return payment_fallback(order_id, amount)
    except Exception as e:
        print(f"❌ Payment error: {e}")
        raise

# Event listeners
def on_circuit_open(breaker):
    print(f"🔴 Circuit breaker '{breaker.name}' OPEN")

def on_circuit_close(breaker):
    print(f"🟢 Circuit breaker '{breaker.name}' CLOSED")

payment_breaker.add_listener('open', on_circuit_open)
payment_breaker.add_listener('close', on_circuit_close)

Flask API с Circuit Breaker

# app.py
from flask import Flask, jsonify, request
from pybreaker import CircuitBreaker, CircuitBreakerError
import requests

app = Flask(__name__)

# Circuit breakers для разных сервисов
class ServiceBreakers:
    payment = CircuitBreaker(
        fail_max=5,
        timeout_duration=30,
        name='payment-service'
    )

    inventory = CircuitBreaker(
        fail_max=3,
        timeout_duration=20,
        name='inventory-service'
    )

    notification = CircuitBreaker(
        fail_max=10,           # More tolerant для non-critical
        timeout_duration=60,
        name='notification-service'
    )

breakers = ServiceBreakers()

# Service calls с circuit breaker
@breakers.payment
def charge_payment(user_id, amount):
    response = requests.post(
        'https://payment-service/api/charge',
        json={'user_id': user_id, 'amount': amount},
        timeout=5
    )
    response.raise_for_status()
    return response.json()

@breakers.inventory
def check_inventory(product_id):
    response = requests.get(
        f'https://inventory-service/api/stock/{product_id}',
        timeout=3
    )
    response.raise_for_status()
    return response.json()

@breakers.notification
def send_notification(user_id, message):
    response = requests.post(
        'https://notification-service/api/send',
        json={'user_id': user_id, 'message': message},
        timeout=2
    )
    response.raise_for_status()
    return response.json()

# API endpoint
@app.route('/api/orders', methods=['POST'])
def create_order():
    data = request.json
    product_id = data['product_id']
    quantity = data['quantity']
    user_id = data['user_id']

    try:
        # 1. Check inventory
        try:
            inventory = check_inventory(product_id)
            if inventory['stock'] < quantity:
                return jsonify({'error': 'Insufficient stock'}), 400
        except CircuitBreakerError:
            # Circuit breaker открыт, используем кэш или разрешаем заказ
            inventory = {'stock': None, 'cached': True}

        # 2. Charge payment
        try:
            payment = charge_payment(user_id, quantity * 100)
        except CircuitBreakerError:
            return jsonify({
                'error': 'Payment service unavailable',
                'order_status': 'queued'
            }), 503

        # 3. Send notification (non-critical, не ждем ответа)
        try:
            send_notification(user_id, f'Order confirmed: {product_id}')
        except (CircuitBreakerError, Exception) as e:
            # Логируем но не падаем
            app.logger.warning(f'Notification failed: {e}')

        return jsonify({
            'order_id': int(time.time()),
            'payment': payment,
            'inventory_cached': inventory.get('cached', False)
        })

    except Exception as e:
        return jsonify({'error': str(e)}), 500

# Health check с circuit breaker stats
@app.route('/api/health')
def health():
    stats = {
        'payment': {
            'state': breakers.payment.current_state,
            'fail_counter': breakers.payment.fail_counter
        },
        'inventory': {
            'state': breakers.inventory.current_state,
            'fail_counter': breakers.inventory.fail_counter
        },
        'notification': {
            'state': breakers.notification.current_state,
            'fail_counter': breakers.notification.fail_counter
        }
    }
    return jsonify({'status': 'ok', 'circuit_breakers': stats})

if __name__ == '__main__':
    app.run(port=5000)

🔵 Go: gobreaker Circuit Breaker

Установка

go get github.com/sony/gobreaker

Базовый пример

// circuit_breaker.go
package main

import (
    "errors"
    "fmt"
    "github.com/sony/gobreaker"
    "net/http"
    "time"
)

// Создаем circuit breaker
var paymentBreaker *gobreaker.CircuitBreaker

func init() {
    settings := gobreaker.Settings{
        Name:        "payment-service",
        MaxRequests: 3,                    // Max requests в Half-Open state
        Interval:    time.Second * 10,     // Окно для подсчета ошибок
        Timeout:     time.Second * 30,     // Timeout до Half-Open
        ReadyToTrip: func(counts gobreaker.Counts) bool {
            // Открыть circuit breaker если error rate > 50%
            failureRatio := float64(counts.TotalFailures) / float64(counts.Requests)
            return counts.Requests >= 10 && failureRatio >= 0.5
        },
        OnStateChange: func(name string, from gobreaker.State, to gobreaker.State) {
            fmt.Printf("🔄 Circuit breaker '%s': %s -> %s\n", name, from, to)
        },
    }

    paymentBreaker = gobreaker.NewCircuitBreaker(settings)
}

// Функция с circuit breaker
func ChargePayment(orderID string, amount float64) (map[string]interface{}, error) {
    result, err := paymentBreaker.Execute(func() (interface{}, error) {
        // Вызов payment service
        resp, err := http.Post(
            "https://payment-service/api/charge",
            "application/json",
            nil, // request body
        )
        if err != nil {
            return nil, err
        }
        defer resp.Body.Close()

        if resp.StatusCode >= 500 {
            return nil, errors.New("payment service error")
        }

        return map[string]interface{}{
            "status": "success",
            "order_id": orderID,
        }, nil
    })

    if err != nil {
        // Circuit breaker открыт или ошибка вызова
        if err == gobreaker.ErrOpenState {
            fmt.Println("⚠️  Circuit breaker OPEN, using fallback")
            return paymentFallback(orderID, amount), nil
        }
        return nil, err
    }

    return result.(map[string]interface{}), nil
}

// Fallback function
func paymentFallback(orderID string, amount float64) map[string]interface{} {
    return map[string]interface{}{
        "status": "queued",
        "order_id": orderID,
        "message": "Payment queued due to service unavailability",
    }
}

func main() {
    // Тестирование circuit breaker
    for i := 0; i < 20; i++ {
        result, err := ChargePayment(fmt.Sprintf("order-%d", i), 100.0)
        if err != nil {
            fmt.Printf("❌ Payment failed: %v\n", err)
        } else {
            fmt.Printf("✅ Payment result: %v\n", result)
        }
        time.Sleep(time.Millisecond * 100)
    }
}

🔄 Fallback стратегии

1. Cached Response (кэшированные данные)

// Fallback: возврат кэша
breaker.fallback(async (userId) => {
  // Пытаемся получить из Redis cache
  const cached = await redis.get(`user:${userId}`);
  if (cached) {
    return {
      ...JSON.parse(cached),
      cached: true,
      timestamp: new Date()
    };
  }

  // Если нет в кэше — default response
  return {
    id: userId,
    name: 'Unknown',
    cached: false
  };
});

2. Default Response (ответ по умолчанию)

// Fallback: default значения
breaker.fallback((productId) => {
  return {
    id: productId,
    available: true,  // Optimistic default
    stock: null,      // Unknown stock
    message: 'Inventory service unavailable, assuming available'
  };
});

3. Queue for Later Processing

// Fallback: добавить в очередь
breaker.fallback(async (orderId, data) => {
  // Добавляем в RabbitMQ/Redis queue
  await queue.add('payment-retry', {
    orderId,
    data,
    timestamp: Date.now()
  });

  return {
    status: 'queued',
    orderId,
    message: 'Payment queued for processing'
  };
});

4. Backup Service (резервный сервис)

// Fallback: backup сервис
breaker.fallback(async (data) => {
  // Пытаемся вызвать backup payment provider
  try {
    const response = await axios.post('https://backup-payment-service/api/charge', data);
    return response.data;
  } catch (error) {
    // Если и backup упал — queue
    await queue.add('payment-retry', data);
    return { status: 'queued' };
  }
});

5. Graceful Degradation

// Fallback: урезанная функциональность
breaker.fallback((searchQuery) => {
  return {
    results: [],
    message: 'Search unavailable, showing recent items',
    fallback: true,
    recentItems: getRecentItemsFromCache()  // Кэшированные недавние товары
  };
});

📊 Мониторинг Circuit Breaker

Метрики для мониторинга

Prometheus Metrics для Circuit Breaker

// metrics.js
const promClient = require('prom-client');

// Circuit breaker state gauge
const circuitBreakerState = new promClient.Gauge({
  name: 'circuit_breaker_state',
  help: 'Circuit breaker state (0=closed, 1=open, 2=half-open)',
  labelNames: ['service']
});

// Success/failure counters
const circuitBreakerRequests = new promClient.Counter({
  name: 'circuit_breaker_requests_total',
  help: 'Total circuit breaker requests',
  labelNames: ['service', 'result']  // result: success, failure, rejected
});

// Fallback invocations
const circuitBreakerFallbacks = new promClient.Counter({
  name: 'circuit_breaker_fallbacks_total',
  help: 'Total fallback invocations',
  labelNames: ['service']
});

// Attach metrics к circuit breaker
function attachMetrics(breaker, serviceName) {
  // State changes
  breaker.on('open', () => {
    circuitBreakerState.set({ service: serviceName }, 1);
  });

  breaker.on('halfOpen', () => {
    circuitBreakerState.set({ service: serviceName }, 2);
  });

  breaker.on('close', () => {
    circuitBreakerState.set({ service: serviceName }, 0);
  });

  // Success/failure
  breaker.on('success', () => {
    circuitBreakerRequests.inc({ service: serviceName, result: 'success' });
  });

  breaker.on('failure', () => {
    circuitBreakerRequests.inc({ service: serviceName, result: 'failure' });
  });

  breaker.on('reject', () => {
    circuitBreakerRequests.inc({ service: serviceName, result: 'rejected' });
  });

  // Fallback
  breaker.on('fallback', () => {
    circuitBreakerFallbacks.inc({ service: serviceName });
  });
}

module.exports = { attachMetrics };

Grafana Dashboard Queries

# Circuit breaker state (0=closed, 1=open, 2=half-open)
circuit_breaker_state

# Success rate за последние 5 минут
sum(rate(circuit_breaker_requests_total{result="success"}[5m])) by (service)
/
sum(rate(circuit_breaker_requests_total[5m])) by (service)

# Fallback rate
sum(rate(circuit_breaker_fallbacks_total[5m])) by (service)

# Rejected requests (circuit breaker открыт)
sum(rate(circuit_breaker_requests_total{result="rejected"}[5m])) by (service)

✅ Best Practices для Circuit Breaker

🔍 FAQ: Часто задаваемые вопросы

📝 Выводы

В этой статье мы рассмотрели Circuit Breaker Pattern для защиты API от каскадных сбоев:

1. Проблема: Один упавший сервис может убить всю систему (cascading failure)
2. Решение: Circuit Breaker блокирует запросы к падающему сервису
3. Три состояния: Closed (норма) → Open (блокировка) → Half-Open (тест) → Closed
4. Реализация: Opossum (Node.js), pybreaker (Python), gobreaker (Go)
5. Fallback: Кэш, default response, queue, backup service
6. Мониторинг: State, success rate, fallback invocations

Related Articles

• API Horizontal Scaling: масштабирование до 100K RPS
• API Load Testing: Apache JMeter, Gatling, K6
• API Performance: оптимизация производительности