Buckets:

download
raw
11 kB
#!/usr/bin/env python3
"""
mw bench — Code benchmarking tool.
Benchmark Python functions, scripts, or shell commands with statistical analysis.
Measures execution time, memory usage, and provides comparison reports.
Usage:
mw bench <file>::<function> Benchmark a Python function
mw bench --cmd "<command>" Benchmark a shell command
mw bench --compare <a> <b> Compare two benchmarks
mw bench --runs <N> Number of iterations (default: 10)
mw bench --warmup <N> Warmup runs (default: 2)
mw bench --output json|md|table Output format (default: table)
mw bench history Show past benchmark results
mw bench baseline <name> Save current run as baseline
mw bench vs-baseline <name> Compare against saved baseline
"""
import sys
import os
import time
import json
import statistics
import subprocess
import importlib.util
import traceback
import resource
BENCH_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.bench')
def ensure_dir():
os.makedirs(BENCH_DIR, exist_ok=True)
def get_memory_mb():
"""Get current memory usage in MB."""
try:
ru = resource.getrusage(resource.RUSAGE_CHILDREN)
return ru.ru_maxrss / 1024 # Convert KB to MB on Linux
except Exception:
return 0.0
def bench_function(file_path, func_name, runs=10, warmup=2):
"""Benchmark a Python function.
Args:
file_path: Path to Python file containing the function
func_name: Name of the function to benchmark
runs: Number of benchmark iterations
warmup: Number of warmup iterations (not counted)
Returns:
Dict with statistical analysis of timing results
Raises:
FileNotFoundError: If file_path doesn't exist
AttributeError: If func_name doesn't exist in the module
Exception: If the function raises an error during execution
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
spec = importlib.util.spec_from_file_location("bench_module", file_path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
if not hasattr(mod, func_name):
raise AttributeError(f"Function '{func_name}' not found in {file_path}")
func = getattr(mod, func_name)
# Warmup (ignore failures)
for _ in range(warmup):
try:
func()
except Exception:
pass
times = []
for _ in range(runs):
try:
start = time.perf_counter_ns()
func()
elapsed = (time.perf_counter_ns() - start) / 1_000_000 # ms
times.append(elapsed)
except Exception as e:
print(f"⚠️ Warning: Run failed with {type(e).__name__}: {e}", file=sys.stderr)
if not times:
raise ValueError(f"All {runs} benchmark runs failed for function: {func_name}")
return analyze(times, f"{os.path.basename(file_path)}::{func_name}")
def bench_command(cmd, runs=10, warmup=2, timeout=30):
"""Benchmark a shell command.
Only includes successful runs (exit code 0) in the results.
Failed runs are skipped and a warning is printed.
Args:
cmd: Shell command to benchmark
runs: Number of benchmark iterations
warmup: Number of warmup iterations (not counted)
timeout: Timeout in seconds per run
Returns:
Dict with statistical analysis of successful runs only
"""
# Warmup (ignore failures during warmup)
for _ in range(warmup):
try:
subprocess.run(cmd, shell=True, capture_output=True, timeout=timeout)
except (subprocess.TimeoutExpired, Exception):
pass
times = []
failed_runs = 0
for i in range(runs):
try:
start = time.perf_counter_ns()
result = subprocess.run(cmd, shell=True, capture_output=True, timeout=timeout)
elapsed = (time.perf_counter_ns() - start) / 1_000_000 # ms
# Only include successful runs
if result.returncode == 0:
times.append(elapsed)
else:
failed_runs += 1
except (subprocess.TimeoutExpired, Exception) as e:
failed_runs += 1
if failed_runs > 0:
print(f"⚠️ Warning: {failed_runs}/{runs} runs failed (skipped from results)", file=sys.stderr)
if not times:
raise ValueError(f"All {runs} benchmark runs failed for command: {cmd[:60]}")
return analyze(times, cmd[:60])
def analyze(times, label):
"""Statistical analysis of timing results."""
return {
'label': label,
'runs': len(times),
'mean_ms': round(statistics.mean(times), 3),
'median_ms': round(statistics.median(times), 3),
'min_ms': round(min(times), 3),
'max_ms': round(max(times), 3),
'stdev_ms': round(statistics.stdev(times), 3) if len(times) > 1 else 0,
'p95_ms': round(sorted(times)[int(len(times) * 0.95)], 3) if len(times) >= 5 else round(max(times), 3),
'times': [round(t, 3) for t in times],
'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S'),
}
def format_table(result):
"""Format as pretty table."""
lines = []
lines.append(f"\n\033[1m⏱ Benchmark: {result['label']}\033[0m")
lines.append("=" * 50)
lines.append(f" Runs: {result['runs']}")
lines.append(f" Mean: {result['mean_ms']:.3f} ms")
lines.append(f" Median: {result['median_ms']:.3f} ms")
lines.append(f" Min: {result['min_ms']:.3f} ms")
lines.append(f" Max: {result['max_ms']:.3f} ms")
lines.append(f" Stdev: {result['stdev_ms']:.3f} ms")
lines.append(f" P95: {result['p95_ms']:.3f} ms")
# Visual bar chart of runs
if len(result['times']) <= 20:
lines.append(f"\n Distribution:")
max_t = max(result['times'])
for i, t in enumerate(result['times']):
bar_len = int((t / max_t) * 30) if max_t > 0 else 0
bar = '█' * bar_len
lines.append(f" Run {i+1:2d}: {bar} {t:.3f}ms")
lines.append("")
return '\n'.join(lines)
def format_md(result):
"""Format as markdown."""
lines = [
f"## Benchmark: {result['label']}",
f"",
f"| Metric | Value |",
f"|--------|-------|",
f"| Runs | {result['runs']} |",
f"| Mean | {result['mean_ms']:.3f} ms |",
f"| Median | {result['median_ms']:.3f} ms |",
f"| Min | {result['min_ms']:.3f} ms |",
f"| Max | {result['max_ms']:.3f} ms |",
f"| Stdev | {result['stdev_ms']:.3f} ms |",
f"| P95 | {result['p95_ms']:.3f} ms |",
]
return '\n'.join(lines)
def compare_results(a, b):
"""Compare two benchmark results."""
diff = ((b['mean_ms'] - a['mean_ms']) / a['mean_ms']) * 100
faster = "faster" if diff < 0 else "slower"
lines = []
lines.append(f"\n\033[1m📊 Comparison\033[0m")
lines.append("=" * 50)
lines.append(f" A: {a['label']} — {a['mean_ms']:.3f}ms avg")
lines.append(f" B: {b['label']} — {b['mean_ms']:.3f}ms avg")
lines.append(f" Δ: {abs(diff):.1f}% {faster}")
if diff < -10:
lines.append(f" \033[92m✅ B is significantly faster!\033[0m")
elif diff > 10:
lines.append(f" \033[91m⚠️ B is significantly slower!\033[0m")
else:
lines.append(f" \033[93m≈ Results are comparable\033[0m")
return '\n'.join(lines)
def save_baseline(name, result):
"""Save benchmark result as a named baseline."""
ensure_dir()
path = os.path.join(BENCH_DIR, f"baseline_{name}.json")
with open(path, 'w') as f:
json.dump(result, f, indent=2)
print(f"✅ Baseline '{name}' saved to {path}")
def load_baseline(name):
"""Load a saved baseline."""
path = os.path.join(BENCH_DIR, f"baseline_{name}.json")
if not os.path.exists(path):
print(f"❌ No baseline '{name}' found")
sys.exit(1)
with open(path) as f:
return json.load(f)
def show_history():
"""Show all saved baselines."""
ensure_dir()
baselines = [f for f in os.listdir(BENCH_DIR) if f.startswith('baseline_')]
if not baselines:
print("No saved baselines. Use: mw bench baseline <name>")
return
print(f"\n\033[1m📋 Saved Baselines\033[0m")
print("=" * 50)
for f in sorted(baselines):
with open(os.path.join(BENCH_DIR, f)) as fh:
data = json.load(fh)
name = f.replace('baseline_', '').replace('.json', '')
print(f" {name:20s} — {data['mean_ms']:.3f}ms avg ({data['runs']} runs, {data.get('timestamp', '?')})")
def show_help():
print(__doc__)
def main():
args = sys.argv[1:]
if not args or args[0] in ('-h', '--help', 'help'):
show_help()
return
if args[0] == 'history':
show_history()
return
if args[0] == 'self':
# Benchmark the mw CLI itself
from tools.benchmark import cmd_benchmark
sys.exit(cmd_benchmark(args[1:]))
return
runs = 10
warmup = 2
fmt = 'table'
cmd = None
target = None
baseline_name = None
vs_baseline = None
i = 0
while i < len(args):
if args[i] == '--runs' and i + 1 < len(args):
runs = int(args[i + 1]); i += 2
elif args[i] == '--warmup' and i + 1 < len(args):
warmup = int(args[i + 1]); i += 2
elif args[i] == '--output' and i + 1 < len(args):
fmt = args[i + 1]; i += 2
elif args[i] == '--cmd' and i + 1 < len(args):
cmd = args[i + 1]; i += 2
elif args[i] == 'baseline' and i + 1 < len(args):
baseline_name = args[i + 1]; i += 2
elif args[i] == 'vs-baseline' and i + 1 < len(args):
vs_baseline = args[i + 1]; i += 2
elif '::' in args[i]:
target = args[i]; i += 1
else:
i += 1
# Run benchmark
if cmd:
result = bench_command(cmd, runs, warmup)
elif target and '::' in target:
file_path, func_name = target.split('::', 1)
result = bench_function(file_path, func_name, runs, warmup)
else:
print("❌ Specify a target: mw bench <file>::<func> or mw bench --cmd '<command>'")
sys.exit(1)
# Output
if fmt == 'json':
print(json.dumps(result, indent=2))
elif fmt == 'md':
print(format_md(result))
else:
print(format_table(result))
# Save baseline if requested
if baseline_name:
save_baseline(baseline_name, result)
# Compare against baseline if requested
if vs_baseline:
baseline = load_baseline(vs_baseline)
print(compare_results(baseline, result))
if __name__ == '__main__':
main()

Xet Storage Details

Size:
11 kB
·
Xet hash:
2aa55d9df24a10b831300740d3b29d98c47d1f26306fe0ee99cdd6916dc89259

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.