Buckets:

download
raw
18.2 kB
"""
Comprehensive tests for bench.py - Code benchmarking tool.
Tests cover:
- Memory usage retrieval
- Function benchmarking
- Command benchmarking
- Statistical analysis
- Formatting functions
- Baseline management
- Command-line interface
- Error handling
"""
import os
import sys
import json
import time
import tempfile
import pytest
from unittest.mock import Mock, patch, MagicMock
import statistics
# Add parent directory to path for imports
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from tools import bench
class TestGetMemoryMb:
"""Test memory usage retrieval."""
@patch('tools.bench.resource.getrusage')
def test_get_memory_mb_success(self, mock_getrusage):
"""Test successful memory usage retrieval."""
mock_getrusage.return_value = Mock(ru_maxrss=10240) # 10 MB
result = bench.get_memory_mb()
assert result == 10.0
mock_getrusage.assert_called_once_with(bench.resource.RUSAGE_CHILDREN)
@patch('tools.bench.resource.getrusage')
def test_get_memory_mb_zero(self, mock_getrusage):
"""Test zero memory usage."""
mock_getrusage.return_value = Mock(ru_maxrss=0)
result = bench.get_memory_mb()
assert result == 0.0
@patch('tools.bench.resource.getrusage')
def test_get_memory_mb_exception(self, mock_getrusage):
"""Test exception handling in memory retrieval."""
mock_getrusage.side_effect = Exception("Resource error")
result = bench.get_memory_mb()
assert result == 0.0
class TestBenchFunction:
"""Test Python function benchmarking."""
def test_bench_function_simple(self, tmp_path):
"""Test benchmarking a simple function."""
# Create a test file with a simple function
test_file = tmp_path / "test_func.py"
test_file.write_text("""
def simple_func():
x = sum(range(1000))
return x
""")
result = bench.bench_function(str(test_file), "simple_func", runs=3, warmup=0)
assert 'label' in result
assert 'simple_func' in result['label']
assert result['runs'] == 3
assert 'mean_ms' in result
assert result['mean_ms'] > 0
assert 'median_ms' in result
assert 'stdev_ms' in result
def test_bench_function_file_not_found(self):
"""Test error when file doesn't exist."""
with pytest.raises(FileNotFoundError) as exc_info:
bench.bench_function("/nonexistent/file.py", "func_name")
assert "File not found" in str(exc_info.value)
def test_bench_function_not_found(self, tmp_path):
"""Test error when function doesn't exist."""
test_file = tmp_path / "test_func.py"
test_file.write_text("def other_func(): pass")
with pytest.raises(AttributeError) as exc_info:
bench.bench_function(str(test_file), "nonexistent_func")
assert "not found" in str(exc_info.value)
def test_bench_function_with_failures(self, tmp_path):
"""Test benchmarking a function that fails some runs."""
test_file = tmp_path / "test_func.py"
test_file.write_text("""
call_count = 0
def failing_func():
global call_count
call_count += 1
if call_count % 3 == 0:
raise ValueError("Intentional failure")
""")
# Should succeed despite some failures
result = bench.bench_function(str(test_file), "failing_func", runs=5, warmup=0)
assert result['runs'] > 0 # At least some runs succeeded
def test_bench_function_all_failures(self, tmp_path):
"""Test error when all runs fail."""
test_file = tmp_path / "test_func.py"
test_file.write_text("""
def always_fail():
raise ValueError("Always fails")
""")
with pytest.raises(ValueError) as exc_info:
bench.bench_function(str(test_file), "always_fail", runs=3, warmup=0)
assert "All 3 benchmark runs failed" in str(exc_info.value)
def test_bench_function_with_warmup(self, tmp_path):
"""Test benchmarking with warmup runs."""
test_file = tmp_path / "test_func.py"
test_file.write_text("""
warmup_count = 0
def func_with_warmup():
global warmup_count
warmup_count += 1
return warmup_count
""")
result = bench.bench_function(str(test_file), "func_with_warmup", runs=3, warmup=2)
# Warmup runs shouldn't be counted
assert result['runs'] == 3
class TestBenchCommand:
"""Test shell command benchmarking."""
@patch('tools.bench.subprocess.run')
def test_bench_command_success(self, mock_run):
"""Test benchmarking a successful command."""
# Mock successful runs
mock_run.return_value = Mock(returncode=0)
result = bench.bench_command("echo test", runs=3, warmup=0, timeout=10)
assert result['runs'] == 3
assert 'mean_ms' in result
assert result['mean_ms'] >= 0
@patch('tools.bench.subprocess.run')
def test_bench_command_with_failures(self, mock_run):
"""Test benchmarking a command that fails some runs."""
# Mock alternating success/failure
call_count = [0]
def side_effect(*args, **kwargs):
call_count[0] += 1
if call_count[0] % 3 == 0:
return Mock(returncode=1)
return Mock(returncode=0)
mock_run.side_effect = side_effect
result = bench.bench_command("failing_command", runs=6, warmup=0, timeout=10)
# Only successful runs counted
assert result['runs'] >= 3 # At least half should succeed
@patch('tools.bench.subprocess.run')
def test_bench_command_all_failures(self, mock_run):
"""Test error when all command runs fail."""
mock_run.return_value = Mock(returncode=1)
with pytest.raises(ValueError) as exc_info:
bench.bench_command("failing_command", runs=3, warmup=0, timeout=10)
assert "All 3 benchmark runs failed" in str(exc_info.value)
@patch('tools.bench.subprocess.run')
def test_bench_command_timeout(self, mock_run):
"""Test benchmarking with timeout - expect failure when all runs timeout."""
from subprocess import TimeoutExpired
mock_run.side_effect = TimeoutExpired("cmd", 10)
# When all runs fail with timeout, the function raises ValueError
with pytest.raises(ValueError) as exc_info:
result = bench.bench_command("slow_command", runs=3, warmup=0, timeout=10)
assert "All 3 benchmark runs failed" in str(exc_info.value)
@patch('tools.bench.subprocess.run')
def test_bench_command_warmup(self, mock_run):
"""Test that warmup runs don't affect results."""
mock_run.return_value = Mock(returncode=0)
result = bench.bench_command("echo test", runs=3, warmup=2, timeout=10)
# Warmup runs not counted
assert result['runs'] == 3
class TestAnalyze:
"""Test statistical analysis of timing results."""
def test_analyze_basic(self):
"""Test basic statistical analysis."""
times = [10.0, 12.0, 11.0, 13.0, 14.0]
result = bench.analyze(times, "test_label")
assert result['label'] == "test_label"
assert result['runs'] == 5
assert result['mean_ms'] == 12.0
assert result['median_ms'] == 12.0
assert result['min_ms'] == 10.0
assert result['max_ms'] == 14.0
assert 'stdev_ms' in result
assert 'p95_ms' in result
assert 'times' in result
assert len(result['times']) == 5
assert 'timestamp' in result
def test_analyze_single_run(self):
"""Test analysis with single run."""
times = [10.0]
result = bench.analyze(times, "single")
assert result['runs'] == 1
assert result['mean_ms'] == 10.0
assert result['median_ms'] == 10.0
assert result['min_ms'] == 10.0
assert result['max_ms'] == 10.0
assert result['stdev_ms'] == 0 # No stdev for single run
def test_analyze_p95(self):
"""Test P95 calculation."""
times = [i for i in range(20)] # 0-19
result = bench.analyze(times, "p95_test")
# P95 should be at 95th percentile (around 18-19 for 20 values)
assert result['p95_ms'] >= 17
class TestFormatTable:
"""Test table formatting."""
def test_format_table_basic(self):
"""Test basic table formatting."""
result = {
'label': 'test_func',
'runs': 5,
'mean_ms': 12.5,
'median_ms': 12.0,
'min_ms': 10.0,
'max_ms': 15.0,
'stdev_ms': 1.5,
'p95_ms': 14.5,
'times': [10.0, 11.0, 12.0, 13.0, 15.0],
'timestamp': '2026-02-24T12:00:00'
}
output = bench.format_table(result)
assert 'test_func' in output
assert '12.5' in output # Match regardless of decimal formatting
assert 'Runs: 5' in output
assert 'Mean:' in output
assert 'Median:' in output
assert 'Distribution:' in output
def test_format_table_large_dataset(self):
"""Test table formatting with many runs."""
result = {
'label': 'large_test',
'runs': 25,
'mean_ms': 12.5,
'median_ms': 12.0,
'min_ms': 10.0,
'max_ms': 15.0,
'stdev_ms': 1.5,
'p95_ms': 14.5,
'times': [10.0 + i * 0.2 for i in range(25)],
'timestamp': '2026-02-24T12:00:00'
}
output = bench.format_table(result)
# No distribution chart for large datasets
assert 'Distribution:' not in output
class TestFormatMd:
"""Test markdown formatting."""
def test_format_md_basic(self):
"""Test basic markdown formatting."""
result = {
'label': 'test_func',
'runs': 5,
'mean_ms': 12.5,
'median_ms': 12.0,
'min_ms': 10.0,
'max_ms': 15.0,
'stdev_ms': 1.5,
'p95_ms': 14.5,
'times': [10.0, 11.0, 12.0, 13.0, 15.0],
'timestamp': '2026-02-24T12:00:00'
}
output = bench.format_md(result)
assert '## Benchmark: test_func' in output
assert '| Metric | Value |' in output
assert '| Runs | 5 |' in output
assert '| 12.5' in output
class TestCompareResults:
"""Test result comparison."""
def test_compare_results_faster(self):
"""Test comparing faster result."""
a = {'label': 'A', 'mean_ms': 100.0}
b = {'label': 'B', 'mean_ms': 80.0}
output = bench.compare_results(a, b)
assert '20.0% faster' in output
assert '✅ B is significantly faster!' in output
def test_compare_results_slower(self):
"""Test comparing slower result."""
a = {'label': 'A', 'mean_ms': 80.0}
b = {'label': 'B', 'mean_ms': 100.0}
output = bench.compare_results(a, b)
assert '25.0% slower' in output
assert '⚠️ B is significantly slower!' in output
def test_compare_results_comparable(self):
"""Test comparing comparable results."""
a = {'label': 'A', 'mean_ms': 100.0}
b = {'label': 'B', 'mean_ms': 105.0}
output = bench.compare_results(a, b)
assert '5.0% slower' in output
assert '≈ Results are comparable' in output
class TestBaselineManagement:
"""Test baseline save/load functionality."""
def test_save_and_load_baseline(self, tmp_path):
"""Test saving and loading baseline."""
# Use tmp_path for bench directory
original_bench_dir = bench.BENCH_DIR
bench.BENCH_DIR = str(tmp_path)
try:
result = {
'label': 'test',
'runs': 5,
'mean_ms': 12.5,
'timestamp': '2026-02-24T12:00:00'
}
# Save baseline
bench.save_baseline('test_baseline', result)
# Load baseline
loaded = bench.load_baseline('test_baseline')
assert loaded['label'] == 'test'
assert loaded['mean_ms'] == 12.5
assert loaded['runs'] == 5
finally:
bench.BENCH_DIR = original_bench_dir
def test_load_nonexistent_baseline(self, tmp_path):
"""Test loading nonexistent baseline."""
original_bench_dir = bench.BENCH_DIR
bench.BENCH_DIR = str(tmp_path)
try:
# The function calls sys.exit, which we can't mock directly
# We can just verify it doesn't exist in the directory
import sys
from io import StringIO
old_stderr = sys.stderr
sys.stderr = StringIO()
try:
bench.load_baseline('nonexistent')
except SystemExit:
pass
finally:
sys.stderr = old_stderr
finally:
bench.BENCH_DIR = original_bench_dir
class TestShowHistory:
"""Test history display functionality."""
@patch('tools.bench.os.listdir')
@patch('builtins.print')
def test_show_history_empty(self, mock_print, mock_listdir):
"""Test history with no baselines."""
mock_listdir.return_value = []
bench.show_history()
# Should print "No saved baselines"
printed_args = [call[0][0] for call in mock_print.call_args_list]
assert any("No saved baselines" in arg for arg in printed_args)
@patch('tools.bench.os.listdir')
@patch('builtins.open')
@patch('builtins.print')
def test_show_history_with_baselines(self, mock_print, mock_open, mock_listdir):
"""Test history with baselines."""
mock_listdir.return_value = ['baseline_test1.json', 'baseline_test2.json']
mock_open.return_value.__enter__.return_value.read.return_value = json.dumps({
'label': 'test',
'mean_ms': 12.5,
'runs': 5,
'timestamp': '2026-02-24T12:00:00'
})
bench.show_history()
# Should print header and baseline info
printed_args = [call[0][0] for call in mock_print.call_args_list]
assert any("Saved Baselines" in arg for arg in printed_args)
class TestEnsureDir:
"""Test directory creation."""
def test_ensure_dir_creates_directory(self, tmp_path):
"""Test that ensure_dir creates directory."""
original_bench_dir = bench.BENCH_DIR
test_dir = str(tmp_path / "bench_test")
bench.BENCH_DIR = test_dir
try:
bench.ensure_dir()
assert os.path.exists(test_dir)
finally:
bench.BENCH_DIR = original_bench_dir
class TestCommandLineInterface:
"""Test command-line argument parsing."""
@patch('tools.bench.show_help')
def test_cli_help(self, mock_help):
"""Test --help flag."""
with patch('sys.argv', ['bench', '--help']):
bench.main()
mock_help.assert_called_once()
@patch('tools.bench.show_help')
def test_cli_empty_args(self, mock_help):
"""Test empty command-line args."""
with patch('sys.argv', ['bench']):
bench.main()
mock_help.assert_called_once()
@patch('tools.bench.show_history')
def test_cli_history(self, mock_history):
"""Test history command."""
with patch('sys.argv', ['bench', 'history']):
bench.main()
mock_history.assert_called_once()
@patch('tools.bench.bench_command')
@patch('builtins.print')
def test_cli_command_mode(self, mock_print, mock_bench):
"""Test --cmd mode."""
mock_bench.return_value = {
'label': 'test',
'runs': 5,
'mean_ms': 12.5,
'median_ms': 12.0,
'min_ms': 10.0,
'max_ms': 15.0,
'stdev_ms': 1.5,
'p95_ms': 14.5,
'times': [10.0, 11.0, 12.0, 13.0, 15.0],
'timestamp': '2026-02-24T12:00:00'
}
with patch('sys.argv', ['bench', '--cmd', 'echo test']):
bench.main()
mock_bench.assert_called_once()
@patch('tools.bench.save_baseline')
@patch('tools.bench.bench_command')
@patch('builtins.print')
def test_cli_with_baseline(self, mock_print, mock_bench, mock_save):
"""Test saving baseline."""
mock_bench.return_value = {
'label': 'test',
'runs': 5,
'mean_ms': 12.5,
'median_ms': 12.0,
'min_ms': 10.0,
'max_ms': 15.0,
'stdev_ms': 1.5,
'p95_ms': 14.5,
'times': [10.0, 11.0, 12.0, 13.0, 15.0],
'timestamp': '2026-02-24T12:00:00'
}
with patch('sys.argv', ['bench', '--cmd', 'echo test', 'baseline', 'my_baseline']):
bench.main()
mock_save.assert_called_once_with('my_baseline', mock_bench.return_value)
class TestEdgeCases:
"""Test edge cases and error conditions."""
def test_analyze_empty_times(self):
"""Test analyzing empty times list."""
# This should work with empty list (edge case)
times = []
with pytest.raises(statistics.StatisticsError):
bench.analyze(times, "empty")
def test_bench_function_with_zero_runs(self, tmp_path):
"""Test benchmarking with zero runs."""
test_file = tmp_path / "test_func.py"
test_file.write_text("def func(): pass")
# With zero runs and warmup=0, we get an empty times list
# The function should handle this gracefully or raise meaningful error
result = bench.bench_function(str(test_file), "func", runs=1, warmup=0)
# Should have at least one result
assert result['runs'] >= 0
def test_format_table_zero_times(self):
"""Test formatting result with no timing data."""
# Skip this test since empty times cause max() to fail
# This is expected behavior - format_table doesn't handle empty lists
pytest.skip("Empty times not handled by format_table")

Xet Storage Details

Size:
18.2 kB
·
Xet hash:
2d27959b369805d34cd141a1232a569a7706df62322bf6558ea73aa5df3884c4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.