Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /CIRISAI_CIRISAgent /tests /test_metrics_runtime_objects.py
| """ | |
| Comprehensive metric tests for runtime objects - CircuitBreaker and ServiceRegistry. | |
| These objects are core runtime components that provide resilience and service discovery. | |
| They track their own metrics independently of traditional services. | |
| """ | |
| import asyncio | |
| import concurrent.futures | |
| import threading | |
| import time | |
| from typing import Dict, Set | |
| from unittest.mock import AsyncMock, MagicMock | |
| import pytest | |
| from ciris_engine.logic.registries.base import Priority, SelectionStrategy, ServiceRegistry | |
| from ciris_engine.logic.registries.circuit_breaker import CircuitBreaker, CircuitBreakerConfig, CircuitState | |
| from ciris_engine.schemas.runtime.enums import ServiceType | |
| from tests.test_metrics_base import BaseMetricsTest | |
| class TestCircuitBreakerMetrics: | |
| """Test CircuitBreaker metrics collection and state tracking.""" | |
| # Expected CircuitBreaker metrics (10 total) | |
| CIRCUIT_BREAKER_METRICS = { | |
| "cb_test_service_state", | |
| "cb_test_service_total_calls", | |
| "cb_test_service_total_failures", | |
| "cb_test_service_total_successes", | |
| "cb_test_service_success_rate", | |
| "cb_test_service_consecutive_failures", | |
| "cb_test_service_recovery_attempts", | |
| "cb_test_service_state_transitions", | |
| "cb_test_service_time_in_open_state_sec", | |
| "cb_test_service_last_failure_age_sec", | |
| } | |
| def assert_required_metrics(self, metrics: Dict[str, float], required: Set[str]): | |
| """Assert that all required metrics are present and valid.""" | |
| for metric in required: | |
| assert metric in metrics, f"Required metric '{metric}' not found" | |
| assert isinstance( | |
| metrics[metric], (int, float) | |
| ), f"Metric '{metric}' is not numeric: {type(metrics[metric])}" | |
| def assert_all_metrics_are_floats(self, metrics: Dict[str, float]): | |
| """Assert all metric values are floats.""" | |
| for key, value in metrics.items(): | |
| assert isinstance(value, (int, float)), f"Metric {key} is not numeric: {type(value)}" | |
| # Ensure it can be converted to float | |
| float(value) | |
| def circuit_breaker(self): | |
| """Create a circuit breaker for testing.""" | |
| config = CircuitBreakerConfig( | |
| failure_threshold=3, recovery_timeout=5.0, success_threshold=2, timeout_duration=10.0 | |
| ) | |
| return CircuitBreaker("test_service", config) | |
| def fast_recovery_breaker(self): | |
| """Create a circuit breaker with fast recovery for testing.""" | |
| config = CircuitBreakerConfig( | |
| failure_threshold=2, recovery_timeout=0.1, success_threshold=1, timeout_duration=1.0 # Very fast recovery | |
| ) | |
| return CircuitBreaker("fast_service", config) | |
| def test_circuit_breaker_metrics_structure(self, circuit_breaker): | |
| """Test that circuit breaker returns expected metrics.""" | |
| metrics = circuit_breaker.get_metrics() | |
| # Check it's a dict with expected metrics | |
| assert isinstance(metrics, dict) | |
| assert ( | |
| len(metrics) == 14 | |
| ), f"Expected 14 metrics (10 prefixed + 4 v1.4.3), got {len(metrics)}: {list(metrics.keys())}" | |
| # Check all expected metrics are present | |
| self.assert_required_metrics(metrics, self.CIRCUIT_BREAKER_METRICS) | |
| # Check all values are floats | |
| self.assert_all_metrics_are_floats(metrics) | |
| def test_initial_state_metrics(self, circuit_breaker): | |
| """Test initial state metrics are correct.""" | |
| metrics = circuit_breaker.get_metrics() | |
| # State should be CLOSED (0.0) | |
| assert metrics["cb_test_service_state"] == 0.0 | |
| # All counts should be zero initially | |
| assert metrics["cb_test_service_total_calls"] == 0.0 | |
| assert metrics["cb_test_service_total_failures"] == 0.0 | |
| assert metrics["cb_test_service_total_successes"] == 0.0 | |
| assert metrics["cb_test_service_consecutive_failures"] == 0.0 | |
| assert metrics["cb_test_service_recovery_attempts"] == 0.0 | |
| assert metrics["cb_test_service_state_transitions"] == 0.0 | |
| # Success rate should be 1.0 (no failures yet) | |
| assert metrics["cb_test_service_success_rate"] == 1.0 | |
| # Time metrics should be zero | |
| assert metrics["cb_test_service_time_in_open_state_sec"] == 0.0 | |
| assert metrics["cb_test_service_last_failure_age_sec"] == 0.0 | |
| def test_success_recording_metrics(self, circuit_breaker): | |
| """Test that recording successes updates metrics correctly.""" | |
| # Record some successes | |
| circuit_breaker.record_success() | |
| circuit_breaker.record_success() | |
| circuit_breaker.record_success() | |
| metrics = circuit_breaker.get_metrics() | |
| # Check call tracking | |
| assert metrics["cb_test_service_total_calls"] == 3.0 | |
| assert metrics["cb_test_service_total_successes"] == 3.0 | |
| assert metrics["cb_test_service_total_failures"] == 0.0 | |
| # Success rate should be 1.0 | |
| assert metrics["cb_test_service_success_rate"] == 1.0 | |
| # Consecutive failures should be 0 | |
| assert metrics["cb_test_service_consecutive_failures"] == 0.0 | |
| # State should still be CLOSED | |
| assert metrics["cb_test_service_state"] == 0.0 | |
| def test_failure_recording_metrics(self, circuit_breaker): | |
| """Test that recording failures updates metrics correctly.""" | |
| # Record some failures | |
| circuit_breaker.record_failure() | |
| circuit_breaker.record_failure() | |
| metrics = circuit_breaker.get_metrics() | |
| # Check call tracking | |
| assert metrics["cb_test_service_total_calls"] == 2.0 | |
| assert metrics["cb_test_service_total_failures"] == 2.0 | |
| assert metrics["cb_test_service_total_successes"] == 0.0 | |
| # Success rate should be 0.0 | |
| assert metrics["cb_test_service_success_rate"] == 0.0 | |
| # Consecutive failures should match | |
| assert metrics["cb_test_service_consecutive_failures"] == 2.0 | |
| # Last failure age should be very small (recent) | |
| assert 0.0 <= metrics["cb_test_service_last_failure_age_sec"] <= 1.0 | |
| def test_state_transition_to_open_metrics(self, circuit_breaker): | |
| """Test metrics when circuit breaker opens.""" | |
| # Record enough failures to open the circuit (threshold = 3) | |
| circuit_breaker.record_failure() | |
| circuit_breaker.record_failure() | |
| circuit_breaker.record_failure() # This should open the circuit | |
| metrics = circuit_breaker.get_metrics() | |
| # State should be OPEN (1.0) | |
| assert metrics["cb_test_service_state"] == 1.0 | |
| # Should have recorded one state transition | |
| assert metrics["cb_test_service_state_transitions"] == 1.0 | |
| # All calls should be failures | |
| assert metrics["cb_test_service_total_calls"] == 3.0 | |
| assert metrics["cb_test_service_total_failures"] == 3.0 | |
| assert metrics["cb_test_service_success_rate"] == 0.0 | |
| def test_state_transition_to_half_open_metrics(self, fast_recovery_breaker): | |
| """Test metrics when circuit breaker transitions to half-open.""" | |
| # Open the circuit | |
| fast_recovery_breaker.record_failure() | |
| fast_recovery_breaker.record_failure() # threshold = 2 | |
| assert fast_recovery_breaker.state == CircuitState.OPEN | |
| # Wait for recovery timeout | |
| time.sleep(0.15) # recovery_timeout = 0.1 | |
| # Check availability (should trigger transition to HALF_OPEN) | |
| available = fast_recovery_breaker.is_available() | |
| assert available | |
| assert fast_recovery_breaker.state == CircuitState.HALF_OPEN | |
| metrics = fast_recovery_breaker.get_metrics() | |
| # State should be HALF_OPEN (0.5) | |
| assert metrics["cb_fast_service_state"] == 0.5 | |
| # Should have recorded two state transitions (CLOSED->OPEN->HALF_OPEN) | |
| assert metrics["cb_fast_service_state_transitions"] == 2.0 | |
| # Should have recorded one recovery attempt | |
| assert metrics["cb_fast_service_recovery_attempts"] == 1.0 | |
| # Should have some time in open state | |
| assert metrics["cb_fast_service_time_in_open_state_sec"] > 0.0 | |
| def test_recovery_success_metrics(self, fast_recovery_breaker): | |
| """Test metrics when circuit breaker recovers successfully.""" | |
| # Open the circuit | |
| fast_recovery_breaker.record_failure() | |
| fast_recovery_breaker.record_failure() | |
| # Wait and transition to half-open | |
| time.sleep(0.15) | |
| fast_recovery_breaker.is_available() | |
| # Record success to close circuit (success_threshold = 1) | |
| fast_recovery_breaker.record_success() | |
| metrics = fast_recovery_breaker.get_metrics() | |
| # State should be CLOSED (0.0) | |
| assert metrics["cb_fast_service_state"] == 0.0 | |
| # Should have recorded three state transitions (CLOSED->OPEN->HALF_OPEN->CLOSED) | |
| assert metrics["cb_fast_service_state_transitions"] == 3.0 | |
| # Should have one recovery attempt | |
| assert metrics["cb_fast_service_recovery_attempts"] == 1.0 | |
| # Should have accumulated time in open state | |
| assert metrics["cb_fast_service_time_in_open_state_sec"] > 0.0 | |
| def test_mixed_success_failure_metrics(self, circuit_breaker): | |
| """Test metrics with mixed success and failure patterns.""" | |
| # Mixed pattern: success, failure, success, failure, failure | |
| circuit_breaker.record_success() | |
| circuit_breaker.record_failure() | |
| circuit_breaker.record_success() | |
| circuit_breaker.record_failure() | |
| circuit_breaker.record_failure() | |
| metrics = circuit_breaker.get_metrics() | |
| # Check totals | |
| assert metrics["cb_test_service_total_calls"] == 5.0 | |
| assert metrics["cb_test_service_total_successes"] == 2.0 | |
| assert metrics["cb_test_service_total_failures"] == 3.0 | |
| # Success rate should be 2/5 = 0.4 | |
| assert abs(metrics["cb_test_service_success_rate"] - 0.4) < 0.001 | |
| # Consecutive failures should be 2 (last two calls) | |
| assert metrics["cb_test_service_consecutive_failures"] == 2.0 | |
| def test_time_in_open_state_accumulation(self, fast_recovery_breaker): | |
| """Test that time in open state accumulates correctly across multiple opens.""" | |
| # First open cycle | |
| fast_recovery_breaker.record_failure() | |
| fast_recovery_breaker.record_failure() | |
| time.sleep(0.05) # Spend some time open | |
| # Transition to half-open and immediately fail (goes back to open) | |
| time.sleep(0.1) # Wait for recovery timeout | |
| fast_recovery_breaker.is_available() # Triggers HALF_OPEN | |
| fast_recovery_breaker.record_failure() # Back to OPEN | |
| time.sleep(0.05) # More time open | |
| # Close the circuit | |
| time.sleep(0.1) # Wait for recovery timeout | |
| fast_recovery_breaker.is_available() # Triggers HALF_OPEN | |
| fast_recovery_breaker.record_success() # CLOSED | |
| metrics = fast_recovery_breaker.get_metrics() | |
| # Should have accumulated time from both open periods | |
| assert metrics["cb_fast_service_time_in_open_state_sec"] >= 0.1 | |
| # Should have two recovery attempts | |
| assert metrics["cb_fast_service_recovery_attempts"] == 2.0 | |
| def test_circuit_breaker_metrics_validation(self, circuit_breaker): | |
| """Test that all circuit breaker metrics pass validation.""" | |
| # Generate some activity | |
| for i in range(10): | |
| if i % 3 == 0: | |
| circuit_breaker.record_failure() | |
| else: | |
| circuit_breaker.record_success() | |
| metrics = circuit_breaker.get_metrics() | |
| # All values should be floats | |
| self.assert_all_metrics_are_floats(metrics) | |
| # All values should be non-negative | |
| for key, value in metrics.items(): | |
| assert value >= 0, f"Metric {key} should be non-negative: {value}" | |
| # Success rate should be between 0 and 1 | |
| assert 0.0 <= metrics["cb_test_service_success_rate"] <= 1.0 | |
| # State should be 0.0, 0.5, or 1.0 | |
| state_value = metrics["cb_test_service_state"] | |
| assert state_value in [0.0, 0.5, 1.0], f"Invalid state value: {state_value}" | |
| def test_concurrent_transition_thread_safety(self, fast_recovery_breaker): | |
| """Test that concurrent state transitions don't cause race conditions.""" | |
| def worker_open_close_cycle(): | |
| """Worker function that cycles the circuit breaker open/close.""" | |
| try: | |
| # Force open | |
| for _ in range(3): | |
| fast_recovery_breaker.record_failure() | |
| # Wait briefly and attempt recovery | |
| time.sleep(0.01) | |
| fast_recovery_breaker.is_available() # May trigger half-open | |
| fast_recovery_breaker.record_success() # May trigger close | |
| return True | |
| except Exception: | |
| return False | |
| # Run multiple concurrent cycles | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: | |
| futures = [executor.submit(worker_open_close_cycle) for _ in range(20)] | |
| results = [future.result() for future in concurrent.futures.as_completed(futures)] | |
| # All workers should succeed (no exceptions) | |
| assert all(results), "Some workers encountered exceptions" | |
| # Metrics should be consistent (no negative values) | |
| metrics = fast_recovery_breaker.get_stats() | |
| assert metrics["time_in_open_state"] >= 0, f"Negative time_in_open_state: {metrics['time_in_open_state']}" | |
| def test_concurrent_metrics_calculation_thread_safety(self, fast_recovery_breaker): | |
| """Test that concurrent get_metrics() calls don't interfere with time calculations.""" | |
| # Start the circuit breaker in OPEN state | |
| for _ in range(3): | |
| fast_recovery_breaker.record_failure() | |
| assert fast_recovery_breaker.state == CircuitState.OPEN | |
| def worker_get_metrics(): | |
| """Worker function that repeatedly calls get_metrics().""" | |
| metrics_list = [] | |
| for _ in range(10): | |
| metrics = fast_recovery_breaker.get_metrics() | |
| time_in_open = metrics["cb_fast_service_time_in_open_state_sec"] | |
| metrics_list.append(time_in_open) | |
| time.sleep(0.001) # Small delay | |
| return metrics_list | |
| # Run multiple concurrent metrics queries | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: | |
| futures = [executor.submit(worker_get_metrics) for _ in range(6)] | |
| all_metrics = [] | |
| for future in concurrent.futures.as_completed(futures): | |
| all_metrics.extend(future.result()) | |
| # All time values should be non-negative | |
| negative_times = [t for t in all_metrics if t < 0] | |
| assert not negative_times, f"Found negative time values: {negative_times}" | |
| # Times should be monotonically increasing or equal (circuit is open) | |
| for i in range(1, len(all_metrics)): | |
| # Allow small variations due to timing precision, but no large negatives | |
| assert all_metrics[i] >= -0.001, f"Significant negative time at index {i}: {all_metrics[i]}" | |
| def test_rapid_state_transitions_thread_safety(self, fast_recovery_breaker): | |
| """Test rapid state transitions under high concurrency don't corrupt timing data.""" | |
| results = [] | |
| errors = [] | |
| def worker_rapid_transitions(): | |
| """Worker that rapidly triggers state transitions.""" | |
| try: | |
| local_results = [] | |
| for cycle in range(5): | |
| # Open the circuit | |
| fast_recovery_breaker.record_failure() | |
| fast_recovery_breaker.record_failure() | |
| # Try to recover (may go to half-open) | |
| time.sleep(0.01) # Brief wait | |
| fast_recovery_breaker.is_available() | |
| # Get metrics immediately after state change | |
| metrics = fast_recovery_breaker.get_stats() | |
| local_results.append( | |
| {"cycle": cycle, "time_in_open": metrics["time_in_open_state"], "state": metrics["state"]} | |
| ) | |
| # Reset for next cycle | |
| fast_recovery_breaker.record_success() | |
| return local_results | |
| except Exception as e: | |
| errors.append(str(e)) | |
| return [] | |
| # Run multiple workers doing rapid transitions | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: | |
| futures = [executor.submit(worker_rapid_transitions) for _ in range(4)] | |
| for future in concurrent.futures.as_completed(futures): | |
| results.extend(future.result()) | |
| # Should have no errors | |
| assert not errors, f"Workers encountered errors: {errors}" | |
| # All time_in_open_state values should be non-negative | |
| negative_times = [r["time_in_open"] for r in results if r["time_in_open"] < 0] | |
| assert not negative_times, f"Found negative time_in_open_state values: {negative_times}" | |
| def test_thread_safe_time_accumulation(self, fast_recovery_breaker): | |
| """Test that time accumulation works correctly under concurrent access.""" | |
| # Record an initial open period | |
| for _ in range(3): | |
| fast_recovery_breaker.record_failure() | |
| # Let it stay open for a measurable time | |
| time.sleep(0.05) | |
| def worker_trigger_recovery(): | |
| """Worker that attempts to trigger recovery transition.""" | |
| try: | |
| # Multiple workers may call this simultaneously | |
| fast_recovery_breaker.is_available() # May trigger transition to half-open | |
| time.sleep(0.001) | |
| fast_recovery_breaker.record_success() # May trigger transition to closed | |
| return True | |
| except Exception: | |
| return False | |
| # Trigger concurrent recovery attempts | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: | |
| futures = [executor.submit(worker_trigger_recovery) for _ in range(10)] | |
| results = [future.result() for future in concurrent.futures.as_completed(futures)] | |
| # All workers should succeed | |
| assert all(results), "Some recovery workers failed" | |
| # Final metrics should be consistent | |
| final_metrics = fast_recovery_breaker.get_stats() | |
| # Time in open state should be positive and reasonable (at least 0.05s from our sleep) | |
| time_in_open = final_metrics["time_in_open_state"] | |
| assert time_in_open >= 0.04, f"time_in_open_state too small: {time_in_open} (expected >= 0.04)" | |
| assert time_in_open <= 1.0, f"time_in_open_state too large: {time_in_open} (expected <= 1.0)" | |
| def test_lock_contention_performance(self, fast_recovery_breaker): | |
| """Test that locking doesn't create severe performance bottlenecks.""" | |
| start_time = time.time() | |
| def worker_mixed_operations(): | |
| """Worker that performs mixed circuit breaker operations.""" | |
| operations = 0 | |
| for _ in range(50): | |
| # Mix of operations that use locks | |
| fast_recovery_breaker.record_success() | |
| fast_recovery_breaker.get_stats() | |
| fast_recovery_breaker.record_failure() | |
| fast_recovery_breaker.is_available() | |
| operations += 4 | |
| return operations | |
| # Run concurrent mixed operations | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: | |
| futures = [executor.submit(worker_mixed_operations) for _ in range(8)] | |
| total_operations = sum(future.result() for future in concurrent.futures.as_completed(futures)) | |
| elapsed_time = time.time() - start_time | |
| # Performance check: should handle many operations quickly | |
| # 8 workers * 50 iterations * 4 operations = 1600 operations | |
| assert total_operations == 1600, f"Expected 1600 operations, got {total_operations}" | |
| # Should complete in reasonable time (< 2 seconds for 1600 operations) | |
| assert elapsed_time < 2.0, f"Performance too slow: {elapsed_time:.3f}s for {total_operations} operations" | |
| # Final state should be consistent | |
| final_metrics = fast_recovery_breaker.get_stats() | |
| assert final_metrics["time_in_open_state"] >= 0, "Final time_in_open_state is negative" | |
| class TestServiceRegistryMetrics: | |
| """Test ServiceRegistry metrics collection and tracking.""" | |
| # Expected ServiceRegistry metrics (10 total) | |
| SERVICE_REGISTRY_METRICS = { | |
| "registry_total_services", | |
| "registry_service_types", | |
| "registry_circuit_breakers", | |
| "registry_open_breakers", | |
| "registry_service_lookups", | |
| "registry_service_hits", | |
| "registry_service_misses", | |
| "registry_hit_rate", | |
| "registry_health_check_failures", | |
| "registry_max_open_breakers", | |
| } | |
| def assert_required_metrics(self, metrics: Dict[str, float], required: Set[str]): | |
| """Assert that all required metrics are present and valid.""" | |
| for metric in required: | |
| assert metric in metrics, f"Required metric '{metric}' not found" | |
| assert isinstance( | |
| metrics[metric], (int, float) | |
| ), f"Metric '{metric}' is not numeric: {type(metrics[metric])}" | |
| def assert_all_metrics_are_floats(self, metrics: Dict[str, float]): | |
| """Assert all metric values are floats.""" | |
| for key, value in metrics.items(): | |
| assert isinstance(value, (int, float)), f"Metric {key} is not numeric: {type(value)}" | |
| # Ensure it can be converted to float | |
| float(value) | |
| def service_registry(self): | |
| """Create a service registry for testing.""" | |
| return ServiceRegistry() | |
| def mock_service(self): | |
| """Create a mock service for registration.""" | |
| service = MagicMock() | |
| service.is_healthy = AsyncMock(return_value=True) | |
| return service | |
| def unhealthy_service(self): | |
| """Create a mock service that fails health checks.""" | |
| service = MagicMock() | |
| service.is_healthy = AsyncMock(return_value=False) | |
| return service | |
| def test_service_registry_metrics_structure(self, service_registry): | |
| """Test that service registry returns expected metrics.""" | |
| metrics = service_registry.get_metrics() | |
| # Check it's a dict with expected metrics | |
| assert isinstance(metrics, dict) | |
| # Registry no longer tracks uptime_seconds (processor handles that) | |
| assert ( | |
| len(metrics) == 15 | |
| ), f"Expected 15 metrics (10 detailed + 4 v1.4.3 + 1 health), got {len(metrics)}: {list(metrics.keys())}" | |
| # Check all expected metrics are present | |
| self.assert_required_metrics(metrics, self.SERVICE_REGISTRY_METRICS) | |
| # Check all values are floats | |
| self.assert_all_metrics_are_floats(metrics) | |
| def test_initial_registry_metrics(self, service_registry): | |
| """Test initial registry metrics are correct.""" | |
| metrics = service_registry.get_metrics() | |
| # Should start with zero services | |
| assert metrics["registry_total_services"] == 0.0 | |
| assert metrics["registry_service_types"] == 0.0 | |
| assert metrics["registry_circuit_breakers"] == 0.0 | |
| assert metrics["registry_open_breakers"] == 0.0 | |
| # No lookups initially | |
| assert metrics["registry_service_lookups"] == 0.0 | |
| assert metrics["registry_service_hits"] == 0.0 | |
| assert metrics["registry_service_misses"] == 0.0 | |
| assert metrics["registry_hit_rate"] == 0.0 | |
| # No health check failures initially | |
| assert metrics["registry_health_check_failures"] == 0.0 | |
| assert metrics["registry_max_open_breakers"] == 0.0 | |
| def test_service_registration_metrics(self, service_registry, mock_service): | |
| """Test that service registration updates metrics.""" | |
| # Create separate service instances to avoid same object registration issues | |
| service1 = MagicMock() | |
| service1.is_healthy = AsyncMock(return_value=True) | |
| service2 = MagicMock() | |
| service2.is_healthy = AsyncMock(return_value=True) | |
| # Register services | |
| service_registry.register_service(ServiceType.LLM, service1, Priority.NORMAL) | |
| service_registry.register_service(ServiceType.MEMORY, service2, Priority.HIGH) | |
| metrics = service_registry.get_metrics() | |
| # Should have 2 services | |
| assert metrics["registry_total_services"] == 2.0 | |
| # Should have 2 service types | |
| assert metrics["registry_service_types"] == 2.0 | |
| # Should have 2 circuit breakers (one per service) | |
| assert metrics["registry_circuit_breakers"] == 2.0 | |
| # No open breakers initially | |
| assert metrics["registry_open_breakers"] == 0.0 | |
| async def test_service_lookup_hit_metrics(self, service_registry, mock_service): | |
| """Test that successful service lookups update hit metrics.""" | |
| # Register a service | |
| service_registry.register_service(ServiceType.LLM, mock_service, Priority.NORMAL) | |
| # Perform successful lookups | |
| result1 = await service_registry.get_service("test_handler", ServiceType.LLM) | |
| result2 = await service_registry.get_service("test_handler", ServiceType.LLM) | |
| assert result1 is not None | |
| assert result2 is not None | |
| metrics = service_registry.get_metrics() | |
| # Should have 2 lookups and 2 hits | |
| assert metrics["registry_service_lookups"] == 2.0 | |
| assert metrics["registry_service_hits"] == 2.0 | |
| assert metrics["registry_service_misses"] == 0.0 | |
| # Hit rate should be 1.0 (100%) | |
| assert metrics["registry_hit_rate"] == 1.0 | |
| async def test_service_lookup_miss_metrics(self, service_registry): | |
| """Test that failed service lookups update miss metrics.""" | |
| # Try to get a service that doesn't exist | |
| result1 = await service_registry.get_service("test_handler", ServiceType.LLM) | |
| result2 = await service_registry.get_service("test_handler", ServiceType.MEMORY) | |
| assert result1 is None | |
| assert result2 is None | |
| metrics = service_registry.get_metrics() | |
| # Should have 2 lookups and 2 misses | |
| assert metrics["registry_service_lookups"] == 2.0 | |
| assert metrics["registry_service_hits"] == 0.0 | |
| assert metrics["registry_service_misses"] == 2.0 | |
| # Hit rate should be 0.0 (0%) | |
| assert metrics["registry_hit_rate"] == 0.0 | |
| async def test_mixed_lookup_metrics(self, service_registry, mock_service): | |
| """Test hit rate calculation with mixed lookup results.""" | |
| # Register one service | |
| service_registry.register_service(ServiceType.LLM, mock_service, Priority.NORMAL) | |
| # Mix of successful and failed lookups | |
| await service_registry.get_service("test", ServiceType.LLM) # hit | |
| await service_registry.get_service("test", ServiceType.MEMORY) # miss | |
| await service_registry.get_service("test", ServiceType.LLM) # hit | |
| await service_registry.get_service("test", ServiceType.AUDIT) # miss | |
| metrics = service_registry.get_metrics() | |
| # Should have 4 lookups, 2 hits, 2 misses | |
| assert metrics["registry_service_lookups"] == 4.0 | |
| assert metrics["registry_service_hits"] == 2.0 | |
| assert metrics["registry_service_misses"] == 2.0 | |
| # Hit rate should be 0.5 (50%) | |
| assert abs(metrics["registry_hit_rate"] - 0.5) < 0.001 | |
| async def test_health_check_failure_metrics(self, service_registry, unhealthy_service): | |
| """Test that health check failures are tracked.""" | |
| # Register an unhealthy service | |
| service_registry.register_service(ServiceType.LLM, unhealthy_service, Priority.NORMAL) | |
| # Try to get the unhealthy service (should fail health check) | |
| result = await service_registry.get_service("test", ServiceType.LLM) | |
| assert result is None # Should fail due to health check | |
| metrics = service_registry.get_metrics() | |
| # Should have recorded a health check failure | |
| assert metrics["registry_health_check_failures"] >= 1.0 | |
| # Should be a miss since service was unhealthy | |
| assert metrics["registry_service_misses"] == 1.0 | |
| def test_circuit_breaker_open_tracking(self, service_registry): | |
| """Test tracking of open circuit breakers.""" | |
| # Create separate service instances | |
| service1 = MagicMock() | |
| service1.is_healthy = AsyncMock(return_value=True) | |
| service2 = MagicMock() | |
| service2.is_healthy = AsyncMock(return_value=True) | |
| # Register services | |
| provider1 = service_registry.register_service(ServiceType.LLM, service1, Priority.NORMAL) | |
| provider2 = service_registry.register_service(ServiceType.MEMORY, service2, Priority.HIGH) | |
| # Open circuit breakers by recording failures | |
| cb1 = service_registry._circuit_breakers[provider1] | |
| cb2 = service_registry._circuit_breakers[provider2] | |
| # Open first circuit breaker | |
| for _ in range(5): # Default threshold is 5 | |
| cb1.record_failure() | |
| metrics = service_registry.get_metrics() | |
| # Should have 1 open breaker | |
| assert metrics["registry_open_breakers"] == 1.0 | |
| assert metrics["registry_max_open_breakers"] == 1.0 | |
| # Open second circuit breaker | |
| for _ in range(5): | |
| cb2.record_failure() | |
| metrics = service_registry.get_metrics() | |
| # Should have 2 open breakers | |
| assert metrics["registry_open_breakers"] == 2.0 | |
| assert metrics["registry_max_open_breakers"] == 2.0 | |
| def test_multiple_service_registration_metrics(self, service_registry): | |
| """Test metrics with multiple services of same and different types.""" | |
| # Create multiple mock services | |
| service1 = MagicMock() | |
| service2 = MagicMock() | |
| service3 = MagicMock() | |
| # Register multiple services | |
| service_registry.register_service(ServiceType.LLM, service1, Priority.HIGH) | |
| service_registry.register_service(ServiceType.LLM, service2, Priority.NORMAL) # Same type | |
| service_registry.register_service(ServiceType.MEMORY, service3, Priority.HIGH) | |
| metrics = service_registry.get_metrics() | |
| # Should have 3 total services | |
| assert metrics["registry_total_services"] == 3.0 | |
| # Should have 2 service types (LLM and MEMORY) | |
| assert metrics["registry_service_types"] == 2.0 | |
| # Should have 3 circuit breakers | |
| assert metrics["registry_circuit_breakers"] == 3.0 | |
| def test_service_unregistration_metrics(self, service_registry): | |
| """Test that unregistering services updates metrics.""" | |
| # Create separate service instances | |
| service1 = MagicMock() | |
| service1.is_healthy = AsyncMock(return_value=True) | |
| service2 = MagicMock() | |
| service2.is_healthy = AsyncMock(return_value=True) | |
| # Register services | |
| provider1 = service_registry.register_service(ServiceType.LLM, service1, Priority.NORMAL) | |
| provider2 = service_registry.register_service(ServiceType.MEMORY, service2, Priority.HIGH) | |
| initial_metrics = service_registry.get_metrics() | |
| assert initial_metrics["registry_total_services"] == 2.0 | |
| assert initial_metrics["registry_circuit_breakers"] == 2.0 | |
| # Unregister one service | |
| success = service_registry.unregister(provider1) | |
| assert success | |
| metrics = service_registry.get_metrics() | |
| # Should have 1 service and 1 circuit breaker remaining | |
| assert metrics["registry_total_services"] == 1.0 | |
| assert metrics["registry_circuit_breakers"] == 1.0 | |
| def test_registry_metrics_validation(self, service_registry, mock_service): | |
| """Test that all registry metrics pass validation.""" | |
| # Register some services and generate activity | |
| service_registry.register_service(ServiceType.LLM, mock_service, Priority.NORMAL) | |
| service_registry.register_service(ServiceType.MEMORY, mock_service, Priority.HIGH) | |
| # Generate some lookup activity | |
| asyncio.run(self._generate_lookup_activity(service_registry)) | |
| metrics = service_registry.get_metrics() | |
| # All values should be floats | |
| self.assert_all_metrics_are_floats(metrics) | |
| # All values should be non-negative | |
| for key, value in metrics.items(): | |
| assert value >= 0, f"Metric {key} should be non-negative: {value}" | |
| # Hit rate should be between 0 and 1 | |
| assert 0.0 <= metrics["registry_hit_rate"] <= 1.0 | |
| # Consistency checks | |
| total_lookups = metrics["registry_service_lookups"] | |
| hits = metrics["registry_service_hits"] | |
| misses = metrics["registry_service_misses"] | |
| # Hits + misses should equal total lookups | |
| assert abs((hits + misses) - total_lookups) < 0.001 | |
| # If there are lookups, hit rate should match calculation | |
| if total_lookups > 0: | |
| expected_hit_rate = hits / total_lookups | |
| assert abs(metrics["registry_hit_rate"] - expected_hit_rate) < 0.001 | |
| async def _generate_lookup_activity(self, service_registry): | |
| """Generate various lookup activity for testing.""" | |
| # Mix of hits and misses | |
| await service_registry.get_service("test", ServiceType.LLM) # hit | |
| await service_registry.get_service("test", ServiceType.MEMORY) # hit | |
| await service_registry.get_service("test", ServiceType.AUDIT) # miss | |
| await service_registry.get_service("test", ServiceType.RUNTIME_CONTROL) # miss | |
| def test_circuit_breaker_max_tracking(self, service_registry): | |
| """Test that max open breakers is tracked correctly.""" | |
| # Create separate service instances | |
| service1 = MagicMock() | |
| service1.is_healthy = AsyncMock(return_value=True) | |
| service2 = MagicMock() | |
| service2.is_healthy = AsyncMock(return_value=True) | |
| service3 = MagicMock() | |
| service3.is_healthy = AsyncMock(return_value=True) | |
| # Register multiple services | |
| provider1 = service_registry.register_service(ServiceType.LLM, service1, Priority.NORMAL) | |
| provider2 = service_registry.register_service(ServiceType.MEMORY, service2, Priority.HIGH) | |
| provider3 = service_registry.register_service(ServiceType.AUDIT, service3, Priority.LOW) | |
| # Open breakers sequentially and check max tracking | |
| cb1 = service_registry._circuit_breakers[provider1] | |
| cb2 = service_registry._circuit_breakers[provider2] | |
| cb3 = service_registry._circuit_breakers[provider3] | |
| # Open first breaker | |
| for _ in range(5): | |
| cb1.record_failure() | |
| metrics = service_registry.get_metrics() | |
| assert metrics["registry_max_open_breakers"] == 1.0 | |
| # Open second breaker | |
| for _ in range(5): | |
| cb2.record_failure() | |
| metrics = service_registry.get_metrics() | |
| assert metrics["registry_max_open_breakers"] == 2.0 | |
| # Close first breaker (should still track max) | |
| cb1.reset() | |
| metrics = service_registry.get_metrics() | |
| assert metrics["registry_open_breakers"] == 1.0 # Only cb2 still open | |
| assert metrics["registry_max_open_breakers"] == 2.0 # Max should persist | |
| async def test_registry_comprehensive_scenario(self, service_registry): | |
| """Test comprehensive scenario with multiple services and activities.""" | |
| # Create separate service instances to avoid circuit breaker sharing | |
| healthy_service1 = MagicMock() | |
| healthy_service1.is_healthy = AsyncMock(return_value=True) | |
| unhealthy_service = MagicMock() | |
| unhealthy_service.is_healthy = AsyncMock(return_value=False) | |
| healthy_service2 = MagicMock() | |
| healthy_service2.is_healthy = AsyncMock(return_value=True) | |
| # Register services | |
| provider1 = service_registry.register_service(ServiceType.LLM, healthy_service1, Priority.HIGH) | |
| provider2 = service_registry.register_service(ServiceType.MEMORY, unhealthy_service, Priority.NORMAL) | |
| provider3 = service_registry.register_service(ServiceType.AUDIT, healthy_service2, Priority.LOW) | |
| # Generate mixed activity | |
| await service_registry.get_service("test", ServiceType.LLM) # hit (healthy) | |
| await service_registry.get_service("test", ServiceType.MEMORY) # miss (unhealthy) | |
| await service_registry.get_service("test", ServiceType.AUDIT) # hit (healthy) | |
| await service_registry.get_service("test", ServiceType.COMMUNICATION) # miss (not registered) | |
| # Break one circuit breaker | |
| cb1 = service_registry._circuit_breakers[provider1] | |
| for _ in range(5): | |
| cb1.record_failure() | |
| metrics = service_registry.get_metrics() | |
| # Verify comprehensive metrics | |
| assert metrics["registry_total_services"] == 3.0 | |
| assert metrics["registry_service_types"] == 3.0 | |
| assert metrics["registry_circuit_breakers"] == 3.0 | |
| assert metrics["registry_open_breakers"] == 1.0 | |
| assert metrics["registry_service_lookups"] == 4.0 | |
| assert metrics["registry_service_hits"] == 2.0 | |
| assert metrics["registry_service_misses"] == 2.0 | |
| assert abs(metrics["registry_hit_rate"] - 0.5) < 0.001 | |
| assert metrics["registry_health_check_failures"] >= 1.0 | |
| assert metrics["registry_max_open_breakers"] == 1.0 | |
Xet Storage Details
- Size:
- 37.9 kB
- Xet hash:
- 7fd24fcbffd5cca604a43677506611fb4946f573d8b2c21df63c8a63331c5946
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.