Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /olaflaitinen_Promethium /benchmarks /run_all.py
| """ | |
| Promethium Benchmark Runner | |
| Executes benchmark configurations and aggregates results across | |
| multiple pipelines and datasets for algorithm comparison. | |
| Usage: | |
| python -m benchmarks.run_all configs/batch/classical_vs_ml.yaml | |
| or programmatically: | |
| from benchmarks.run_all import run_benchmark | |
| results = run_benchmark("configs/batch/classical_vs_ml.yaml") | |
| """ | |
| import json | |
| import os | |
| import sys | |
| from datetime import datetime | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional | |
| import yaml | |
| def load_benchmark_config(config_path: Path) -> Dict[str, Any]: | |
| """Load benchmark configuration from YAML file.""" | |
| with open(config_path, "r") as f: | |
| return yaml.safe_load(f) | |
| def run_single_pipeline( | |
| pipeline_config: Dict[str, Any], | |
| dataset_path: str, | |
| reference_path: Optional[str] = None, | |
| output_dir: Path = Path("results"), | |
| ) -> Dict[str, Any]: | |
| """ | |
| Run a single pipeline and collect metrics. | |
| Args: | |
| pipeline_config: Pipeline configuration dictionary. | |
| dataset_path: Path to input dataset. | |
| reference_path: Optional path to reference data. | |
| output_dir: Directory for outputs. | |
| Returns: | |
| Dictionary with run results and metrics. | |
| """ | |
| from promethium.io.readers import load_seismic_data | |
| from promethium.pipelines.recovery import SeismicRecoveryPipeline | |
| from promethium.evaluation.metrics import ( | |
| signal_to_noise_ratio, | |
| mean_squared_error, | |
| structural_similarity_index, | |
| ) | |
| import numpy as np | |
| pipeline_name = pipeline_config["name"] | |
| start_time = datetime.now() | |
| try: | |
| # Load data | |
| dataset = load_seismic_data(dataset_path) | |
| # Create and run pipeline | |
| pipe = SeismicRecoveryPipeline(pipeline_name, { | |
| "model": pipeline_config.get("config", {}), | |
| }) | |
| reconstructed = pipe.run(dataset) | |
| end_time = datetime.now() | |
| duration = (end_time - start_time).total_seconds() | |
| # Compute metrics if reference available | |
| metrics = {} | |
| if reference_path and Path(reference_path).exists(): | |
| reference = np.load(reference_path) | |
| recon_data = reconstructed.traces if hasattr(reconstructed, "traces") else np.array(reconstructed) | |
| metrics = { | |
| "snr": float(signal_to_noise_ratio(reference, recon_data)), | |
| "mse": float(mean_squared_error(reference, recon_data)), | |
| "ssim": float(structural_similarity_index(reference, recon_data)), | |
| } | |
| return { | |
| "pipeline": pipeline_name, | |
| "status": "success", | |
| "duration_seconds": duration, | |
| "metrics": metrics, | |
| } | |
| except Exception as e: | |
| return { | |
| "pipeline": pipeline_name, | |
| "status": "failed", | |
| "error": str(e), | |
| "metrics": {}, | |
| } | |
| def run_benchmark( | |
| config_path: str, | |
| output_dir: Optional[str] = None, | |
| verbose: bool = True, | |
| ) -> Dict[str, Any]: | |
| """ | |
| Run a complete benchmark suite. | |
| Args: | |
| config_path: Path to benchmark configuration file. | |
| output_dir: Optional output directory override. | |
| verbose: Whether to print progress. | |
| Returns: | |
| Benchmark results dictionary. | |
| """ | |
| config_path = Path(config_path) | |
| config = load_benchmark_config(config_path) | |
| benchmark_name = config["benchmark"]["name"] | |
| timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") | |
| if verbose: | |
| print(f"Running benchmark: {benchmark_name}") | |
| print("=" * 60) | |
| # Setup output directory | |
| output_base = Path(output_dir) if output_dir else Path(config["output"]["results_dir"]) | |
| output_base.mkdir(parents=True, exist_ok=True) | |
| # Collect results | |
| all_results = [] | |
| datasets = config.get("datasets", []) | |
| pipelines = config.get("pipelines", []) | |
| for dataset_info in datasets: | |
| dataset_id = dataset_info["id"] | |
| dataset_path = dataset_info["path"] | |
| reference_path = dataset_info.get("reference") | |
| if verbose: | |
| print(f"\nDataset: {dataset_id}") | |
| print("-" * 40) | |
| for pipeline_config in pipelines: | |
| pipeline_name = pipeline_config["name"] | |
| if verbose: | |
| print(f" Running: {pipeline_name}...", end=" ") | |
| result = run_single_pipeline( | |
| pipeline_config, | |
| dataset_path, | |
| reference_path, | |
| output_base, | |
| ) | |
| result["dataset"] = dataset_id | |
| result["timestamp"] = datetime.now().isoformat() | |
| all_results.append(result) | |
| if verbose: | |
| if result["status"] == "success": | |
| metrics = result.get("metrics", {}) | |
| snr = metrics.get("snr", "N/A") | |
| print(f"OK (SNR: {snr:.2f} dB)" if isinstance(snr, float) else "OK") | |
| else: | |
| print(f"FAILED: {result.get('error', 'Unknown error')}") | |
| # Aggregate results | |
| summary = generate_summary(all_results, config) | |
| # Save results | |
| results_file = output_base / f"benchmark_{benchmark_name}_{timestamp}.json" | |
| with open(results_file, "w") as f: | |
| json.dump({ | |
| "benchmark": benchmark_name, | |
| "timestamp": timestamp, | |
| "config_path": str(config_path), | |
| "summary": summary, | |
| "results": all_results, | |
| }, f, indent=2) | |
| if verbose: | |
| print(f"\nResults saved to: {results_file}") | |
| # Generate CSV | |
| if config["output"].get("save_csv", True): | |
| csv_file = output_base / f"benchmark_{benchmark_name}_{timestamp}.csv" | |
| save_results_csv(all_results, csv_file) | |
| if verbose: | |
| print(f"CSV saved to: {csv_file}") | |
| return { | |
| "benchmark": benchmark_name, | |
| "summary": summary, | |
| "results": all_results, | |
| "output_path": str(results_file), | |
| } | |
| def generate_summary(results: List[Dict], config: Dict) -> Dict[str, Any]: | |
| """Generate summary statistics from results.""" | |
| import statistics | |
| # Group by pipeline | |
| by_pipeline = {} | |
| for result in results: | |
| if result["status"] != "success": | |
| continue | |
| name = result["pipeline"] | |
| if name not in by_pipeline: | |
| by_pipeline[name] = {"metrics": {}} | |
| for metric, value in result.get("metrics", {}).items(): | |
| if metric not in by_pipeline[name]["metrics"]: | |
| by_pipeline[name]["metrics"][metric] = [] | |
| by_pipeline[name]["metrics"][metric].append(value) | |
| # Compute statistics | |
| summary = {} | |
| for pipeline, data in by_pipeline.items(): | |
| summary[pipeline] = {} | |
| for metric, values in data["metrics"].items(): | |
| if values: | |
| summary[pipeline][metric] = { | |
| "mean": statistics.mean(values), | |
| "std": statistics.stdev(values) if len(values) > 1 else 0.0, | |
| "min": min(values), | |
| "max": max(values), | |
| } | |
| return summary | |
| def save_results_csv(results: List[Dict], output_path: Path) -> None: | |
| """Save results to CSV format.""" | |
| import csv | |
| if not results: | |
| return | |
| # Collect all metric keys | |
| all_metrics = set() | |
| for r in results: | |
| all_metrics.update(r.get("metrics", {}).keys()) | |
| fieldnames = ["dataset", "pipeline", "status", "duration_seconds"] + sorted(all_metrics) | |
| with open(output_path, "w", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=fieldnames) | |
| writer.writeheader() | |
| for result in results: | |
| row = { | |
| "dataset": result.get("dataset", ""), | |
| "pipeline": result.get("pipeline", ""), | |
| "status": result.get("status", ""), | |
| "duration_seconds": result.get("duration_seconds", ""), | |
| } | |
| for metric in all_metrics: | |
| row[metric] = result.get("metrics", {}).get(metric, "") | |
| writer.writerow(row) | |
| def print_comparison_table(summary: Dict[str, Any]) -> None: | |
| """Print a formatted comparison table.""" | |
| try: | |
| from rich.console import Console | |
| from rich.table import Table | |
| console = Console() | |
| table = Table(title="Benchmark Results Summary") | |
| # Add columns | |
| table.add_column("Pipeline", style="cyan") | |
| # Get all metrics | |
| all_metrics = set() | |
| for data in summary.values(): | |
| all_metrics.update(data.keys()) | |
| for metric in sorted(all_metrics): | |
| table.add_column(metric.upper(), style="green") | |
| # Add rows | |
| for pipeline, metrics in summary.items(): | |
| row = [pipeline] | |
| for metric in sorted(all_metrics): | |
| if metric in metrics: | |
| val = metrics[metric]["mean"] | |
| std = metrics[metric]["std"] | |
| row.append(f"{val:.4f} +/- {std:.4f}") | |
| else: | |
| row.append("N/A") | |
| table.add_row(*row) | |
| console.print(table) | |
| except ImportError: | |
| # Fallback without rich | |
| print("\nBenchmark Summary:") | |
| print("-" * 60) | |
| for pipeline, metrics in summary.items(): | |
| print(f"\n{pipeline}:") | |
| for metric, stats in metrics.items(): | |
| print(f" {metric}: {stats['mean']:.4f} +/- {stats['std']:.4f}") | |
| if __name__ == "__main__": | |
| if len(sys.argv) < 2: | |
| print("Usage: python run_all.py <benchmark_config.yaml>") | |
| print("\nExample:") | |
| print(" python run_all.py configs/batch/classical_vs_ml.yaml") | |
| sys.exit(1) | |
| config_path = sys.argv[1] | |
| output_dir = sys.argv[2] if len(sys.argv) > 2 else None | |
| results = run_benchmark(config_path, output_dir) | |
| print("\n" + "=" * 60) | |
| print_comparison_table(results["summary"]) | |
Xet Storage Details
- Size:
- 10.3 kB
- Xet hash:
- aa1ef5f9bc3801c41349b089045e17d87d5d7c3f4112754eddab36c3df7ad932
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.