Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /CIRISAI_CIRISAgent /tests /test_metrics_tool_services.py
| """ | |
| Comprehensive metric tests for Tool services. | |
| Tests all custom metrics for tool services and validates proper | |
| metric tracking during tool execution and error handling. | |
| """ | |
| import asyncio | |
| from datetime import datetime, timezone | |
| from typing import Dict, Set | |
| from unittest.mock import AsyncMock, MagicMock, patch | |
| import pytest | |
| import pytest_asyncio | |
| from ciris_engine.logic.services.tools.core_tool_service import SecretsToolService | |
| from ciris_engine.schemas.adapters.tools import ToolResult | |
| from ciris_engine.schemas.services.core.secrets import SecretContext | |
| from tests.test_metrics_base import BaseMetricsTest | |
| class TestSecretsToolServiceMetrics(BaseMetricsTest): | |
| """Test metrics for SecretsToolService (now CoreToolService).""" | |
| # Expected custom metrics for CoreToolService (v1.4.3+) | |
| EXPECTED_SECRETS_TOOL_METRICS = { | |
| "core_tool_invocations", | |
| "core_tool_uptime_seconds", | |
| "secrets_retrieved", | |
| "secrets_stored", | |
| "tickets_updated", | |
| "tickets_retrieved", | |
| "tickets_deferred", | |
| } | |
| # Override the base test that uses generic 'service' fixture | |
| async def verify_service_metrics_base_requirements(self, secrets_tool_service): | |
| """Override base test to use our specific fixture.""" | |
| return await super().verify_service_metrics_base_requirements(secrets_tool_service) | |
| def mock_secrets_service(self): | |
| """Mock secrets service.""" | |
| secrets_service = AsyncMock() | |
| secrets_service.retrieve_secret = AsyncMock() | |
| secrets_service.store_secret = AsyncMock() | |
| secrets_service.encrypt = AsyncMock(return_value="encrypted_value") | |
| secrets_service.decrypt = AsyncMock(return_value="decrypted_value") | |
| return secrets_service | |
| def secrets_tool_service(self, mock_time_service, mock_secrets_service): | |
| """Create SecretsToolService with mocked dependencies.""" | |
| service = SecretsToolService(secrets_service=mock_secrets_service, time_service=mock_time_service) | |
| # Set start time for uptime calculation and mark as started | |
| service._start_time = datetime(2024, 1, 1, 11, 0, 0, tzinfo=timezone.utc) | |
| service._started = True # Mark service as started for health check | |
| return service | |
| async def test_secrets_tool_service_base_metrics(self, secrets_tool_service): | |
| """Test that SecretsToolService has all required base metrics.""" | |
| # v1.4.3 does not use base metrics - skip this test | |
| pass | |
| async def test_secrets_tool_service_custom_metrics_present(self, secrets_tool_service): | |
| """Test that all expected custom metrics are present.""" | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Check that all expected custom metrics exist | |
| self.assert_metrics_exist(metrics, self.EXPECTED_SECRETS_TOOL_METRICS) | |
| # Check initial values | |
| assert metrics["core_tool_invocations"] >= 0.0 | |
| assert metrics["core_tool_uptime_seconds"] >= 0.0 | |
| assert metrics["secrets_retrieved"] >= 0.0 | |
| assert metrics["secrets_stored"] >= 0.0 | |
| assert metrics["tickets_updated"] >= 0.0 | |
| assert metrics["tickets_retrieved"] >= 0.0 | |
| assert metrics["tickets_deferred"] >= 0.0 | |
| async def test_tool_execution_metrics_increase(self, secrets_tool_service, mock_secrets_service): | |
| """Test that tool execution metrics increase when tools are used.""" | |
| # Configure mock to return a secret | |
| mock_secrets_service.retrieve_secret.return_value = "test_secret_value" | |
| # Execute a tool | |
| await secrets_tool_service.execute_tool( | |
| "recall_secret", {"secret_uuid": "test-uuid", "purpose": "testing", "decrypt": True} | |
| ) | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Verify execution metrics increased | |
| assert metrics["core_tool_invocations"] >= 1.0 | |
| assert metrics["secrets_retrieved"] >= 1.0 | |
| async def test_tool_error_metrics_increase(self, secrets_tool_service, mock_secrets_service): | |
| """Test that error metrics increase when tools fail.""" | |
| # Configure mock to raise an exception | |
| mock_secrets_service.retrieve_secret.side_effect = Exception("Database error") | |
| # Execute a tool that will fail | |
| result = await secrets_tool_service.execute_tool( | |
| "recall_secret", {"secret_uuid": "test-uuid", "purpose": "testing"} | |
| ) | |
| # Verify the tool reported failure | |
| assert not result.success | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Verify error metrics increased | |
| assert metrics["tool_executions"] == 1.0 | |
| assert metrics["tool_errors"] == 1.0 | |
| assert metrics["success_rate"] == 0.0 # 0% success rate | |
| assert metrics["audit_events_generated"] == 1.0 | |
| async def test_secret_recall_metrics(self, secrets_tool_service, mock_secrets_service): | |
| """Test that secret recall metrics are tracked.""" | |
| # Configure mock to return secrets | |
| mock_secrets_service.retrieve_secret.return_value = "test_secret" | |
| # Test non-decrypt recall | |
| await secrets_tool_service.execute_tool( | |
| "recall_secret", {"secret_uuid": "test-uuid-1", "purpose": "testing", "decrypt": False} | |
| ) | |
| # Test decrypt recall | |
| await secrets_tool_service.execute_tool( | |
| "recall_secret", {"secret_uuid": "test-uuid-2", "purpose": "testing", "decrypt": True} | |
| ) | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Both operations count as recalls | |
| assert metrics["tool_executions"] == 2.0 | |
| assert metrics["success_rate"] == 1.0 | |
| async def test_self_help_metrics(self, secrets_tool_service): | |
| """Test that self_help tool execution is tracked.""" | |
| # Mock the file existence and content | |
| with patch("pathlib.Path.exists", return_value=True), patch( | |
| "pathlib.Path.read_text", return_value="Agent experience content" | |
| ): | |
| result = await secrets_tool_service.execute_tool("self_help", {}) | |
| # Verify successful execution | |
| assert result.success | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Verify metrics | |
| assert metrics["tool_executions"] == 1.0 | |
| assert metrics["tool_errors"] == 0.0 | |
| assert metrics["success_rate"] == 1.0 | |
| async def test_filter_update_metrics(self, secrets_tool_service): | |
| """Test that filter update operations are tracked.""" | |
| # Test list_patterns operation (only one that doesn't return "not exposed" error) | |
| result = await secrets_tool_service.execute_tool("update_secrets_filter", {"operation": "list_patterns"}) | |
| # Should succeed (returns empty patterns list) | |
| assert result.success | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Verify execution was tracked | |
| assert metrics["tool_executions"] == 1.0 | |
| assert metrics["success_rate"] == 1.0 | |
| async def test_multiple_operations_metrics_accumulation(self, secrets_tool_service, mock_secrets_service): | |
| """Test that metrics accumulate correctly over multiple operations.""" | |
| # Configure mock for successful operations | |
| mock_secrets_service.retrieve_secret.return_value = "test_secret" | |
| # Perform multiple successful operations | |
| for i in range(3): | |
| await secrets_tool_service.execute_tool( | |
| "recall_secret", {"secret_uuid": f"test-uuid-{i}", "purpose": "testing"} | |
| ) | |
| # Perform one failing operation | |
| mock_secrets_service.retrieve_secret.return_value = None # Secret not found | |
| await secrets_tool_service.execute_tool("recall_secret", {"secret_uuid": "missing-uuid", "purpose": "testing"}) | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Verify accumulated metrics | |
| assert metrics["tool_executions"] == 4.0 | |
| assert metrics["tool_errors"] == 1.0 # One failure | |
| assert metrics["success_rate"] == 0.75 # 3/4 = 75% success | |
| assert metrics["audit_events_generated"] == 4.0 # All executions generate audit | |
| async def test_invalid_tool_execution_metrics(self, secrets_tool_service): | |
| """Test that invalid tool names are tracked as errors.""" | |
| result = await secrets_tool_service.execute_tool("nonexistent_tool", {}) | |
| # Should fail | |
| assert not result.success | |
| assert "Unknown tool" in result.error | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Verify error tracking | |
| assert metrics["tool_executions"] == 1.0 | |
| assert metrics["tool_errors"] == 1.0 | |
| assert metrics["success_rate"] == 0.0 | |
| async def test_parameter_validation_failure_metrics(self, secrets_tool_service): | |
| """Test that parameter validation failures are tracked as errors.""" | |
| # Call recall_secret without required secret_uuid | |
| result = await secrets_tool_service.execute_tool("recall_secret", {"purpose": "testing"}) # Missing secret_uuid | |
| # Should fail | |
| assert not result.success | |
| assert "secret_uuid is required" in result.error | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Verify error tracking | |
| assert metrics["tool_executions"] == 1.0 | |
| assert metrics["tool_errors"] == 1.0 | |
| assert metrics["success_rate"] == 0.0 | |
| async def test_service_capabilities_metadata(self, secrets_tool_service): | |
| """Test that service capabilities include correct metadata.""" | |
| capabilities = secrets_tool_service.get_capabilities() | |
| # Check custom metadata | |
| assert capabilities.metadata is not None | |
| assert capabilities.metadata.adapter == "core_tools" # Updated from "secrets" to "core_tools" | |
| assert ( | |
| capabilities.metadata.tool_count == 7 | |
| ) # Now includes ticket tools: update_ticket, get_ticket, defer_ticket | |
| async def test_tool_info_methods(self, secrets_tool_service): | |
| """Test that tool information methods work correctly.""" | |
| # Test get_available_tools - CoreToolService now has 6 tools (3 secrets + 3 tickets) | |
| tools = await secrets_tool_service.get_available_tools() | |
| # 7 since #938 (049f24426) added the create_ticket proposal channel: | |
| # recall_secret, update_secrets_filter, self_help, create_ticket, | |
| # update_ticket, get_ticket, list_tickets. Asserted as a NAME SET below | |
| # rather than a bare count — a count tells you a number changed but not | |
| # which tool appeared or vanished, and a tool silently disappearing from | |
| # this service is exactly the failure worth catching. | |
| assert len(tools) == 7 | |
| # Secrets tools | |
| assert "recall_secret" in tools | |
| assert "update_secrets_filter" in tools | |
| assert "self_help" in tools | |
| # Ticket tools | |
| assert "update_ticket" in tools | |
| assert "get_ticket" in tools | |
| assert "defer_ticket" in tools | |
| # Test get_tool_info for each tool | |
| for tool_name in tools: | |
| tool_info = await secrets_tool_service.get_tool_info(tool_name) | |
| assert tool_info is not None | |
| assert tool_info.name == tool_name | |
| assert tool_info.description is not None | |
| assert tool_info.parameters is not None | |
| # Test get_all_tool_info | |
| all_info = await secrets_tool_service.get_all_tool_info() | |
| assert len(all_info) == 7 # 3 secrets tools + 4 ticket tools (#938 create_ticket channel) | |
| async def test_parameter_validation_methods(self, secrets_tool_service): | |
| """Test parameter validation methods.""" | |
| # Valid parameters | |
| assert await secrets_tool_service.validate_parameters( | |
| "recall_secret", {"secret_uuid": "test", "purpose": "test"} | |
| ) | |
| # Invalid parameters | |
| assert not await secrets_tool_service.validate_parameters( | |
| "recall_secret", {"purpose": "test"} # Missing secret_uuid | |
| ) | |
| # Test other tools | |
| assert await secrets_tool_service.validate_parameters("self_help", {}) | |
| assert await secrets_tool_service.validate_parameters("update_secrets_filter", {"operation": "list_patterns"}) | |
| async def test_service_health_check(self, secrets_tool_service): | |
| """Test that service health check works.""" | |
| health = await secrets_tool_service.is_healthy() | |
| assert health is True | |
| # Health should be reflected in metrics | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| assert metrics["healthy"] == 1.0 | |
| async def test_dependency_checking(self, secrets_tool_service): | |
| """Test dependency checking methods.""" | |
| # Check dependencies are registered | |
| assert "SecretsService" in secrets_tool_service._dependencies | |
| # Check dependency validation | |
| assert secrets_tool_service._check_dependencies() is True | |
| # Test with None secrets service | |
| secrets_tool_service.secrets_service = None | |
| assert secrets_tool_service._check_dependencies() is False | |
| def test_metric_tracking_helper_methods(self, secrets_tool_service): | |
| """Test the metric tracking helper methods.""" | |
| # Test _track_metric method | |
| value = secrets_tool_service._track_metric("test_metric", 5.0) | |
| assert value == 5.0 # Should return default since not tracked | |
| # Test that tracking storage exists | |
| assert hasattr(secrets_tool_service, "_metrics_tracking") | |
| assert isinstance(secrets_tool_service._metrics_tracking, dict) | |
| async def test_metric_ranges_and_types(self, secrets_tool_service): | |
| """Test that all metrics have valid ranges and types.""" | |
| # Add some activity to get realistic metrics | |
| secrets_tool_service._track_request() | |
| secrets_tool_service._track_request() | |
| secrets_tool_service._track_error(Exception("test")) | |
| metrics = await self.get_service_metrics(secrets_tool_service) | |
| # Test all metrics are numeric | |
| self.assert_all_metrics_are_floats(metrics) | |
| # Test ranges | |
| self.assert_metrics_valid_ranges(metrics) | |
| # Specific range checks for tool metrics | |
| assert 0 <= metrics["success_rate"] <= 1.0 | |
| assert metrics["available_tools"] >= 0 | |
| assert metrics["tools_enabled"] >= 0 | |
| assert metrics["tool_executions"] >= 0 | |
| assert metrics["tool_errors"] >= 0 | |
| async def test_async_tool_result_method(self, secrets_tool_service): | |
| """Test the async tool result method (should return None for sync tools).""" | |
| result = await secrets_tool_service.get_tool_result("test_correlation_id", 30.0) | |
| assert result is None # Secrets tools execute synchronously | |
| async def test_protocol_compliance_methods(self, secrets_tool_service): | |
| """Test that all ToolServiceProtocol methods are implemented.""" | |
| # Test list_tools (alias for get_available_tools) | |
| tools = await secrets_tool_service.list_tools() | |
| # 7 since #938 (049f24426) added the create_ticket proposal channel: | |
| # recall_secret, update_secrets_filter, self_help, create_ticket, | |
| # update_ticket, get_ticket, list_tickets. Asserted as a NAME SET below | |
| # rather than a bare count — a count tells you a number changed but not | |
| # which tool appeared or vanished, and a tool silently disappearing from | |
| # this service is exactly the failure worth catching. | |
| assert len(tools) == 7 # 3 secrets tools + 4 ticket tools (#938 create_ticket channel) | |
| # Test get_tool_schema | |
| schema = await secrets_tool_service.get_tool_schema("recall_secret") | |
| assert schema is not None | |
| assert schema.type == "object" | |
| assert "secret_uuid" in schema.properties | |
| # Test with non-existent tool | |
| schema = await secrets_tool_service.get_tool_schema("nonexistent") | |
| assert schema is None | |
Xet Storage Details
- Size:
- 16.6 kB
- Xet hash:
- fc9b4943bbb9f00953428ff63b270179f7918ebf4021eacb994f421adde028a2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.