Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /DansiDanutz_MyWork-AI /tests /test_bench.py
| """ | |
| Comprehensive tests for bench.py - Code benchmarking tool. | |
| Tests cover: | |
| - Memory usage retrieval | |
| - Function benchmarking | |
| - Command benchmarking | |
| - Statistical analysis | |
| - Formatting functions | |
| - Baseline management | |
| - Command-line interface | |
| - Error handling | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import time | |
| import tempfile | |
| import pytest | |
| from unittest.mock import Mock, patch, MagicMock | |
| import statistics | |
| # Add parent directory to path for imports | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from tools import bench | |
| class TestGetMemoryMb: | |
| """Test memory usage retrieval.""" | |
| def test_get_memory_mb_success(self, mock_getrusage): | |
| """Test successful memory usage retrieval.""" | |
| mock_getrusage.return_value = Mock(ru_maxrss=10240) # 10 MB | |
| result = bench.get_memory_mb() | |
| assert result == 10.0 | |
| mock_getrusage.assert_called_once_with(bench.resource.RUSAGE_CHILDREN) | |
| def test_get_memory_mb_zero(self, mock_getrusage): | |
| """Test zero memory usage.""" | |
| mock_getrusage.return_value = Mock(ru_maxrss=0) | |
| result = bench.get_memory_mb() | |
| assert result == 0.0 | |
| def test_get_memory_mb_exception(self, mock_getrusage): | |
| """Test exception handling in memory retrieval.""" | |
| mock_getrusage.side_effect = Exception("Resource error") | |
| result = bench.get_memory_mb() | |
| assert result == 0.0 | |
| class TestBenchFunction: | |
| """Test Python function benchmarking.""" | |
| def test_bench_function_simple(self, tmp_path): | |
| """Test benchmarking a simple function.""" | |
| # Create a test file with a simple function | |
| test_file = tmp_path / "test_func.py" | |
| test_file.write_text(""" | |
| def simple_func(): | |
| x = sum(range(1000)) | |
| return x | |
| """) | |
| result = bench.bench_function(str(test_file), "simple_func", runs=3, warmup=0) | |
| assert 'label' in result | |
| assert 'simple_func' in result['label'] | |
| assert result['runs'] == 3 | |
| assert 'mean_ms' in result | |
| assert result['mean_ms'] > 0 | |
| assert 'median_ms' in result | |
| assert 'stdev_ms' in result | |
| def test_bench_function_file_not_found(self): | |
| """Test error when file doesn't exist.""" | |
| with pytest.raises(FileNotFoundError) as exc_info: | |
| bench.bench_function("/nonexistent/file.py", "func_name") | |
| assert "File not found" in str(exc_info.value) | |
| def test_bench_function_not_found(self, tmp_path): | |
| """Test error when function doesn't exist.""" | |
| test_file = tmp_path / "test_func.py" | |
| test_file.write_text("def other_func(): pass") | |
| with pytest.raises(AttributeError) as exc_info: | |
| bench.bench_function(str(test_file), "nonexistent_func") | |
| assert "not found" in str(exc_info.value) | |
| def test_bench_function_with_failures(self, tmp_path): | |
| """Test benchmarking a function that fails some runs.""" | |
| test_file = tmp_path / "test_func.py" | |
| test_file.write_text(""" | |
| call_count = 0 | |
| def failing_func(): | |
| global call_count | |
| call_count += 1 | |
| if call_count % 3 == 0: | |
| raise ValueError("Intentional failure") | |
| """) | |
| # Should succeed despite some failures | |
| result = bench.bench_function(str(test_file), "failing_func", runs=5, warmup=0) | |
| assert result['runs'] > 0 # At least some runs succeeded | |
| def test_bench_function_all_failures(self, tmp_path): | |
| """Test error when all runs fail.""" | |
| test_file = tmp_path / "test_func.py" | |
| test_file.write_text(""" | |
| def always_fail(): | |
| raise ValueError("Always fails") | |
| """) | |
| with pytest.raises(ValueError) as exc_info: | |
| bench.bench_function(str(test_file), "always_fail", runs=3, warmup=0) | |
| assert "All 3 benchmark runs failed" in str(exc_info.value) | |
| def test_bench_function_with_warmup(self, tmp_path): | |
| """Test benchmarking with warmup runs.""" | |
| test_file = tmp_path / "test_func.py" | |
| test_file.write_text(""" | |
| warmup_count = 0 | |
| def func_with_warmup(): | |
| global warmup_count | |
| warmup_count += 1 | |
| return warmup_count | |
| """) | |
| result = bench.bench_function(str(test_file), "func_with_warmup", runs=3, warmup=2) | |
| # Warmup runs shouldn't be counted | |
| assert result['runs'] == 3 | |
| class TestBenchCommand: | |
| """Test shell command benchmarking.""" | |
| def test_bench_command_success(self, mock_run): | |
| """Test benchmarking a successful command.""" | |
| # Mock successful runs | |
| mock_run.return_value = Mock(returncode=0) | |
| result = bench.bench_command("echo test", runs=3, warmup=0, timeout=10) | |
| assert result['runs'] == 3 | |
| assert 'mean_ms' in result | |
| assert result['mean_ms'] >= 0 | |
| def test_bench_command_with_failures(self, mock_run): | |
| """Test benchmarking a command that fails some runs.""" | |
| # Mock alternating success/failure | |
| call_count = [0] | |
| def side_effect(*args, **kwargs): | |
| call_count[0] += 1 | |
| if call_count[0] % 3 == 0: | |
| return Mock(returncode=1) | |
| return Mock(returncode=0) | |
| mock_run.side_effect = side_effect | |
| result = bench.bench_command("failing_command", runs=6, warmup=0, timeout=10) | |
| # Only successful runs counted | |
| assert result['runs'] >= 3 # At least half should succeed | |
| def test_bench_command_all_failures(self, mock_run): | |
| """Test error when all command runs fail.""" | |
| mock_run.return_value = Mock(returncode=1) | |
| with pytest.raises(ValueError) as exc_info: | |
| bench.bench_command("failing_command", runs=3, warmup=0, timeout=10) | |
| assert "All 3 benchmark runs failed" in str(exc_info.value) | |
| def test_bench_command_timeout(self, mock_run): | |
| """Test benchmarking with timeout - expect failure when all runs timeout.""" | |
| from subprocess import TimeoutExpired | |
| mock_run.side_effect = TimeoutExpired("cmd", 10) | |
| # When all runs fail with timeout, the function raises ValueError | |
| with pytest.raises(ValueError) as exc_info: | |
| result = bench.bench_command("slow_command", runs=3, warmup=0, timeout=10) | |
| assert "All 3 benchmark runs failed" in str(exc_info.value) | |
| def test_bench_command_warmup(self, mock_run): | |
| """Test that warmup runs don't affect results.""" | |
| mock_run.return_value = Mock(returncode=0) | |
| result = bench.bench_command("echo test", runs=3, warmup=2, timeout=10) | |
| # Warmup runs not counted | |
| assert result['runs'] == 3 | |
| class TestAnalyze: | |
| """Test statistical analysis of timing results.""" | |
| def test_analyze_basic(self): | |
| """Test basic statistical analysis.""" | |
| times = [10.0, 12.0, 11.0, 13.0, 14.0] | |
| result = bench.analyze(times, "test_label") | |
| assert result['label'] == "test_label" | |
| assert result['runs'] == 5 | |
| assert result['mean_ms'] == 12.0 | |
| assert result['median_ms'] == 12.0 | |
| assert result['min_ms'] == 10.0 | |
| assert result['max_ms'] == 14.0 | |
| assert 'stdev_ms' in result | |
| assert 'p95_ms' in result | |
| assert 'times' in result | |
| assert len(result['times']) == 5 | |
| assert 'timestamp' in result | |
| def test_analyze_single_run(self): | |
| """Test analysis with single run.""" | |
| times = [10.0] | |
| result = bench.analyze(times, "single") | |
| assert result['runs'] == 1 | |
| assert result['mean_ms'] == 10.0 | |
| assert result['median_ms'] == 10.0 | |
| assert result['min_ms'] == 10.0 | |
| assert result['max_ms'] == 10.0 | |
| assert result['stdev_ms'] == 0 # No stdev for single run | |
| def test_analyze_p95(self): | |
| """Test P95 calculation.""" | |
| times = [i for i in range(20)] # 0-19 | |
| result = bench.analyze(times, "p95_test") | |
| # P95 should be at 95th percentile (around 18-19 for 20 values) | |
| assert result['p95_ms'] >= 17 | |
| class TestFormatTable: | |
| """Test table formatting.""" | |
| def test_format_table_basic(self): | |
| """Test basic table formatting.""" | |
| result = { | |
| 'label': 'test_func', | |
| 'runs': 5, | |
| 'mean_ms': 12.5, | |
| 'median_ms': 12.0, | |
| 'min_ms': 10.0, | |
| 'max_ms': 15.0, | |
| 'stdev_ms': 1.5, | |
| 'p95_ms': 14.5, | |
| 'times': [10.0, 11.0, 12.0, 13.0, 15.0], | |
| 'timestamp': '2026-02-24T12:00:00' | |
| } | |
| output = bench.format_table(result) | |
| assert 'test_func' in output | |
| assert '12.5' in output # Match regardless of decimal formatting | |
| assert 'Runs: 5' in output | |
| assert 'Mean:' in output | |
| assert 'Median:' in output | |
| assert 'Distribution:' in output | |
| def test_format_table_large_dataset(self): | |
| """Test table formatting with many runs.""" | |
| result = { | |
| 'label': 'large_test', | |
| 'runs': 25, | |
| 'mean_ms': 12.5, | |
| 'median_ms': 12.0, | |
| 'min_ms': 10.0, | |
| 'max_ms': 15.0, | |
| 'stdev_ms': 1.5, | |
| 'p95_ms': 14.5, | |
| 'times': [10.0 + i * 0.2 for i in range(25)], | |
| 'timestamp': '2026-02-24T12:00:00' | |
| } | |
| output = bench.format_table(result) | |
| # No distribution chart for large datasets | |
| assert 'Distribution:' not in output | |
| class TestFormatMd: | |
| """Test markdown formatting.""" | |
| def test_format_md_basic(self): | |
| """Test basic markdown formatting.""" | |
| result = { | |
| 'label': 'test_func', | |
| 'runs': 5, | |
| 'mean_ms': 12.5, | |
| 'median_ms': 12.0, | |
| 'min_ms': 10.0, | |
| 'max_ms': 15.0, | |
| 'stdev_ms': 1.5, | |
| 'p95_ms': 14.5, | |
| 'times': [10.0, 11.0, 12.0, 13.0, 15.0], | |
| 'timestamp': '2026-02-24T12:00:00' | |
| } | |
| output = bench.format_md(result) | |
| assert '## Benchmark: test_func' in output | |
| assert '| Metric | Value |' in output | |
| assert '| Runs | 5 |' in output | |
| assert '| 12.5' in output | |
| class TestCompareResults: | |
| """Test result comparison.""" | |
| def test_compare_results_faster(self): | |
| """Test comparing faster result.""" | |
| a = {'label': 'A', 'mean_ms': 100.0} | |
| b = {'label': 'B', 'mean_ms': 80.0} | |
| output = bench.compare_results(a, b) | |
| assert '20.0% faster' in output | |
| assert '✅ B is significantly faster!' in output | |
| def test_compare_results_slower(self): | |
| """Test comparing slower result.""" | |
| a = {'label': 'A', 'mean_ms': 80.0} | |
| b = {'label': 'B', 'mean_ms': 100.0} | |
| output = bench.compare_results(a, b) | |
| assert '25.0% slower' in output | |
| assert '⚠️ B is significantly slower!' in output | |
| def test_compare_results_comparable(self): | |
| """Test comparing comparable results.""" | |
| a = {'label': 'A', 'mean_ms': 100.0} | |
| b = {'label': 'B', 'mean_ms': 105.0} | |
| output = bench.compare_results(a, b) | |
| assert '5.0% slower' in output | |
| assert '≈ Results are comparable' in output | |
| class TestBaselineManagement: | |
| """Test baseline save/load functionality.""" | |
| def test_save_and_load_baseline(self, tmp_path): | |
| """Test saving and loading baseline.""" | |
| # Use tmp_path for bench directory | |
| original_bench_dir = bench.BENCH_DIR | |
| bench.BENCH_DIR = str(tmp_path) | |
| try: | |
| result = { | |
| 'label': 'test', | |
| 'runs': 5, | |
| 'mean_ms': 12.5, | |
| 'timestamp': '2026-02-24T12:00:00' | |
| } | |
| # Save baseline | |
| bench.save_baseline('test_baseline', result) | |
| # Load baseline | |
| loaded = bench.load_baseline('test_baseline') | |
| assert loaded['label'] == 'test' | |
| assert loaded['mean_ms'] == 12.5 | |
| assert loaded['runs'] == 5 | |
| finally: | |
| bench.BENCH_DIR = original_bench_dir | |
| def test_load_nonexistent_baseline(self, tmp_path): | |
| """Test loading nonexistent baseline.""" | |
| original_bench_dir = bench.BENCH_DIR | |
| bench.BENCH_DIR = str(tmp_path) | |
| try: | |
| # The function calls sys.exit, which we can't mock directly | |
| # We can just verify it doesn't exist in the directory | |
| import sys | |
| from io import StringIO | |
| old_stderr = sys.stderr | |
| sys.stderr = StringIO() | |
| try: | |
| bench.load_baseline('nonexistent') | |
| except SystemExit: | |
| pass | |
| finally: | |
| sys.stderr = old_stderr | |
| finally: | |
| bench.BENCH_DIR = original_bench_dir | |
| class TestShowHistory: | |
| """Test history display functionality.""" | |
| def test_show_history_empty(self, mock_print, mock_listdir): | |
| """Test history with no baselines.""" | |
| mock_listdir.return_value = [] | |
| bench.show_history() | |
| # Should print "No saved baselines" | |
| printed_args = [call[0][0] for call in mock_print.call_args_list] | |
| assert any("No saved baselines" in arg for arg in printed_args) | |
| def test_show_history_with_baselines(self, mock_print, mock_open, mock_listdir): | |
| """Test history with baselines.""" | |
| mock_listdir.return_value = ['baseline_test1.json', 'baseline_test2.json'] | |
| mock_open.return_value.__enter__.return_value.read.return_value = json.dumps({ | |
| 'label': 'test', | |
| 'mean_ms': 12.5, | |
| 'runs': 5, | |
| 'timestamp': '2026-02-24T12:00:00' | |
| }) | |
| bench.show_history() | |
| # Should print header and baseline info | |
| printed_args = [call[0][0] for call in mock_print.call_args_list] | |
| assert any("Saved Baselines" in arg for arg in printed_args) | |
| class TestEnsureDir: | |
| """Test directory creation.""" | |
| def test_ensure_dir_creates_directory(self, tmp_path): | |
| """Test that ensure_dir creates directory.""" | |
| original_bench_dir = bench.BENCH_DIR | |
| test_dir = str(tmp_path / "bench_test") | |
| bench.BENCH_DIR = test_dir | |
| try: | |
| bench.ensure_dir() | |
| assert os.path.exists(test_dir) | |
| finally: | |
| bench.BENCH_DIR = original_bench_dir | |
| class TestCommandLineInterface: | |
| """Test command-line argument parsing.""" | |
| def test_cli_help(self, mock_help): | |
| """Test --help flag.""" | |
| with patch('sys.argv', ['bench', '--help']): | |
| bench.main() | |
| mock_help.assert_called_once() | |
| def test_cli_empty_args(self, mock_help): | |
| """Test empty command-line args.""" | |
| with patch('sys.argv', ['bench']): | |
| bench.main() | |
| mock_help.assert_called_once() | |
| def test_cli_history(self, mock_history): | |
| """Test history command.""" | |
| with patch('sys.argv', ['bench', 'history']): | |
| bench.main() | |
| mock_history.assert_called_once() | |
| def test_cli_command_mode(self, mock_print, mock_bench): | |
| """Test --cmd mode.""" | |
| mock_bench.return_value = { | |
| 'label': 'test', | |
| 'runs': 5, | |
| 'mean_ms': 12.5, | |
| 'median_ms': 12.0, | |
| 'min_ms': 10.0, | |
| 'max_ms': 15.0, | |
| 'stdev_ms': 1.5, | |
| 'p95_ms': 14.5, | |
| 'times': [10.0, 11.0, 12.0, 13.0, 15.0], | |
| 'timestamp': '2026-02-24T12:00:00' | |
| } | |
| with patch('sys.argv', ['bench', '--cmd', 'echo test']): | |
| bench.main() | |
| mock_bench.assert_called_once() | |
| def test_cli_with_baseline(self, mock_print, mock_bench, mock_save): | |
| """Test saving baseline.""" | |
| mock_bench.return_value = { | |
| 'label': 'test', | |
| 'runs': 5, | |
| 'mean_ms': 12.5, | |
| 'median_ms': 12.0, | |
| 'min_ms': 10.0, | |
| 'max_ms': 15.0, | |
| 'stdev_ms': 1.5, | |
| 'p95_ms': 14.5, | |
| 'times': [10.0, 11.0, 12.0, 13.0, 15.0], | |
| 'timestamp': '2026-02-24T12:00:00' | |
| } | |
| with patch('sys.argv', ['bench', '--cmd', 'echo test', 'baseline', 'my_baseline']): | |
| bench.main() | |
| mock_save.assert_called_once_with('my_baseline', mock_bench.return_value) | |
| class TestEdgeCases: | |
| """Test edge cases and error conditions.""" | |
| def test_analyze_empty_times(self): | |
| """Test analyzing empty times list.""" | |
| # This should work with empty list (edge case) | |
| times = [] | |
| with pytest.raises(statistics.StatisticsError): | |
| bench.analyze(times, "empty") | |
| def test_bench_function_with_zero_runs(self, tmp_path): | |
| """Test benchmarking with zero runs.""" | |
| test_file = tmp_path / "test_func.py" | |
| test_file.write_text("def func(): pass") | |
| # With zero runs and warmup=0, we get an empty times list | |
| # The function should handle this gracefully or raise meaningful error | |
| result = bench.bench_function(str(test_file), "func", runs=1, warmup=0) | |
| # Should have at least one result | |
| assert result['runs'] >= 0 | |
| def test_format_table_zero_times(self): | |
| """Test formatting result with no timing data.""" | |
| # Skip this test since empty times cause max() to fail | |
| # This is expected behavior - format_table doesn't handle empty lists | |
| pytest.skip("Empty times not handled by format_table") | |
Xet Storage Details
- Size:
- 18.2 kB
- Xet hash:
- 2d27959b369805d34cd141a1232a569a7706df62322bf6558ea73aa5df3884c4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.