Spaces:
Sleeping
Sleeping
Download src/evaluate.py from amitaddi/btxrd-workstation: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/evaluate.py
- Command line
-
hf download hf://spaces/amitaddi/btxrd-workstation/src/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/evaluate.py
11.6 kB
| """ | |
| Unified Evaluation Script. | |
| Loads saved results from all experiments and produces: | |
| - Comparison tables (LaTeX + CSV) | |
| - Summary statistics | |
| - Statistical significance tests | |
| Usage: | |
| python -m src.evaluate --project_root . --results_dir results/runs | |
| """ | |
| import os | |
| import json | |
| import glob | |
| import argparse | |
| import numpy as np | |
| import pandas as pd | |
| from pathlib import Path | |
| from typing import Dict, List | |
| from src.metrics import compare_models_statistical | |
| CLASS_NAMES = [ | |
| 'Giant Cell Tumor', 'Multiple Osteochondromas', 'Osteochondroma', | |
| 'Osteofibroma', 'Osteosarcoma', 'Other BT', 'Other MT', | |
| 'Simple Bone Cyst', 'Synovial Osteochondroma' | |
| ] | |
| def get_clean_model_name(run_name: str, subkey: str = None) -> str: | |
| """Map a raw run folder name and optional subkey to a clean, publication-ready name.""" | |
| if run_name.startswith('unet_'): | |
| return 'UNet Baseline' | |
| elif run_name.startswith('sam2_zeroshot_'): | |
| if subkey == 'box': | |
| return 'SAM2 (Zero-Shot Box)' | |
| elif subkey == 'center_point': | |
| return 'SAM2 (Zero-Shot Center Point)' | |
| elif subkey == 'random_3_points': | |
| return 'SAM2 (Zero-Shot 3 Points)' | |
| return f'SAM2 Zero-Shot ({subkey})' | |
| elif run_name.startswith('sam2_fewshot_'): | |
| shot_num = subkey.replace('_shot', '') if subkey else 'Few' | |
| return f'SAM2 ({shot_num}-Shot Box)' | |
| elif run_name.startswith('sam2_finetune_'): | |
| return 'SAM2 (Fine-Tuned Box)' | |
| elif run_name.startswith('medsam_zeroshot_'): | |
| return 'MedSAM (Zero-Shot Box)' | |
| elif run_name.startswith('sam2_ablation_'): | |
| if subkey == 'baseline_padded': | |
| return 'SAM2 (Padded Box)' | |
| elif subkey == 'iterative_3pass': | |
| return 'SAM2 (Iterative Box)' | |
| elif subkey == 'iterative_postprocess': | |
| return 'SAM2 (Iterative + Morph Box)' | |
| elif subkey == 'full_pipeline': | |
| return 'SAM2 (Full Pipeline)' | |
| return f'SAM2 Ablation ({subkey})' | |
| elif run_name.startswith('sam2_enhanced_'): | |
| if subkey == 'enhanced': | |
| return 'SAM2 (Enhanced Weak-Label)' | |
| return f'SAM2 Enhanced ({subkey})' | |
| else: | |
| name = run_name | |
| if subkey: | |
| name += f" ({subkey})" | |
| return name | |
| def filter_latest_runs(run_dirs: List[str]) -> List[str]: | |
| """Filter run directories to keep only the latest run for each experiment type.""" | |
| prefixes = ['unet', 'sam2_zeroshot', 'sam2_fewshot', 'sam2_finetune', 'medsam_zeroshot', 'sam2_enhanced', 'sam2_ablation'] | |
| latest_by_prefix = {} | |
| for d in run_dirs: | |
| name = Path(d).name | |
| matched_prefix = None | |
| for p in prefixes: | |
| if name.startswith(p + '_'): | |
| matched_prefix = p | |
| break | |
| if matched_prefix: | |
| if matched_prefix not in latest_by_prefix or name > Path(latest_by_prefix[matched_prefix]).name: | |
| latest_by_prefix[matched_prefix] = d | |
| else: | |
| latest_by_prefix[name] = d | |
| return list(latest_by_prefix.values()) | |
| def load_run_results(run_dir: str) -> Dict: | |
| """Load results and config from a run directory.""" | |
| results_path = os.path.join(run_dir, 'results.json') | |
| config_path = os.path.join(run_dir, 'config.json') | |
| data = {} | |
| if os.path.exists(results_path): | |
| with open(results_path, 'r') as f: | |
| data['results'] = json.load(f) | |
| if os.path.exists(config_path): | |
| with open(config_path, 'r') as f: | |
| data['config'] = json.load(f) | |
| data['run_name'] = Path(run_dir).name | |
| return data | |
| def create_comparison_table(all_results: List[Dict]) -> pd.DataFrame: | |
| """ | |
| Create a comparison table across all experiments. | |
| Returns: | |
| DataFrame with models as rows, metrics as columns | |
| """ | |
| rows = [] | |
| for run_data in all_results: | |
| name = run_data['run_name'] | |
| if 'results' not in run_data: | |
| continue | |
| results = run_data['results'] | |
| # Handle different result structures | |
| if isinstance(results, dict): | |
| # Could be per-prompt-type results (SAM2 zero-shot) | |
| if any(k in results for k in ['box', 'center_point', 'random_3_points']): | |
| for prompt_type, pt_results in results.items(): | |
| if isinstance(pt_results, dict) and 'overall' in pt_results: | |
| overall = pt_results['overall'] | |
| row = { | |
| 'Model': get_clean_model_name(name, prompt_type), | |
| 'Dice (mean)': overall.get('dice', {}).get('mean', np.nan), | |
| 'Dice (std)': overall.get('dice', {}).get('std', np.nan), | |
| 'IoU (mean)': overall.get('iou', {}).get('mean', np.nan), | |
| 'IoU (std)': overall.get('iou', {}).get('std', np.nan), | |
| 'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan), | |
| 'Precision': overall.get('precision', {}).get('mean', np.nan), | |
| 'Recall': overall.get('recall', {}).get('mean', np.nan), | |
| 'N': pt_results.get('n_samples', 0), | |
| } | |
| rows.append(row) | |
| # Could be few-shot results | |
| elif any(k.endswith('_shot') for k in results.keys()): | |
| for shot_type, shot_results in results.items(): | |
| if isinstance(shot_results, dict) and 'overall' in shot_results: | |
| overall = shot_results['overall'] | |
| row = { | |
| 'Model': get_clean_model_name(name, shot_type), | |
| 'Dice (mean)': overall.get('dice', {}).get('mean', np.nan), | |
| 'Dice (std)': overall.get('dice', {}).get('std', np.nan), | |
| 'IoU (mean)': overall.get('iou', {}).get('mean', np.nan), | |
| 'IoU (std)': overall.get('iou', {}).get('std', np.nan), | |
| 'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan), | |
| 'Precision': overall.get('precision', {}).get('mean', np.nan), | |
| 'Recall': overall.get('recall', {}).get('mean', np.nan), | |
| 'N': shot_results.get('n_samples', 0), | |
| } | |
| rows.append(row) | |
| # Could be ablation or enhanced results | |
| elif any(k in results for k in ['baseline_padded', 'iterative_3pass', 'iterative_postprocess', 'full_pipeline', 'enhanced']): | |
| for config_name, config_results in results.items(): | |
| if isinstance(config_results, dict) and 'overall' in config_results: | |
| overall = config_results['overall'] | |
| row = { | |
| 'Model': get_clean_model_name(name, config_name), | |
| 'Dice (mean)': overall.get('dice', {}).get('mean', np.nan), | |
| 'Dice (std)': overall.get('dice', {}).get('std', np.nan), | |
| 'IoU (mean)': overall.get('iou', {}).get('mean', np.nan), | |
| 'IoU (std)': overall.get('iou', {}).get('std', np.nan), | |
| 'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan), | |
| 'Precision': overall.get('precision', {}).get('mean', np.nan), | |
| 'Recall': overall.get('recall', {}).get('mean', np.nan), | |
| 'N': config_results.get('n_samples', 0), | |
| } | |
| rows.append(row) | |
| elif 'overall' in results: | |
| overall = results['overall'] | |
| row = { | |
| 'Model': get_clean_model_name(name), | |
| 'Dice (mean)': overall.get('dice', {}).get('mean', np.nan), | |
| 'Dice (std)': overall.get('dice', {}).get('std', np.nan), | |
| 'IoU (mean)': overall.get('iou', {}).get('mean', np.nan), | |
| 'IoU (std)': overall.get('iou', {}).get('std', np.nan), | |
| 'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan), | |
| 'Precision': overall.get('precision', {}).get('mean', np.nan), | |
| 'Recall': overall.get('recall', {}).get('mean', np.nan), | |
| 'N': results.get('n_samples', 0), | |
| } | |
| rows.append(row) | |
| return pd.DataFrame(rows) | |
| def generate_latex_table(df: pd.DataFrame, caption: str = "Segmentation Performance Comparison") -> str: | |
| """Generate a LaTeX table from comparison DataFrame.""" | |
| latex = "\\begin{table}[htbp]\n" | |
| latex += f"\\caption{{{caption}}}\n" | |
| latex += "\\centering\n" | |
| latex += "\\begin{tabular}{l c c c c c}\n" | |
| latex += "\\toprule\n" | |
| latex += "Model & Dice & IoU & HD95 & Precision & Recall \\\\\n" | |
| latex += "\\midrule\n" | |
| for _, row in df.iterrows(): | |
| dice_str = f"{row['Dice (mean)']:.4f} \\pm {row['Dice (std)']:.4f}" | |
| iou_str = f"{row['IoU (mean)']:.4f}" | |
| hd95_str = f"{row['HD95 (mean)']:.2f}" if not np.isnan(row['HD95 (mean)']) else "—" | |
| prec_str = f"{row['Precision']:.4f}" | |
| rec_str = f"{row['Recall']:.4f}" | |
| latex += f"{row['Model']} & {dice_str} & {iou_str} & {hd95_str} & {prec_str} & {rec_str} \\\\\n" | |
| latex += "\\bottomrule\n" | |
| latex += "\\end{tabular}\n" | |
| latex += "\\label{tab:comparison}\n" | |
| latex += "\\end{table}\n" | |
| return latex | |
| def main(): | |
| parser = argparse.ArgumentParser(description='Unified evaluation') | |
| parser.add_argument('--project_root', type=str, default='.') | |
| parser.add_argument('--results_dir', type=str, default=None) | |
| args = parser.parse_args() | |
| if args.results_dir is None: | |
| args.results_dir = os.path.join(args.project_root, 'results', 'runs') | |
| # Find all run directories | |
| run_dirs = sorted(glob.glob(os.path.join(args.results_dir, '*'))) | |
| run_dirs = [d for d in run_dirs if os.path.isdir(d)] | |
| if not run_dirs: | |
| print(f"No runs found in {args.results_dir}") | |
| return | |
| # Chronologically filter run directories to keep only the latest of each type | |
| run_dirs = filter_latest_runs(run_dirs) | |
| print(f"Comparing the following {len(run_dirs)} latest runs:") | |
| for d in sorted(run_dirs): | |
| print(f" - {Path(d).name}") | |
| # Load all results | |
| all_results = [load_run_results(d) for d in run_dirs] | |
| # Create comparison table | |
| df = create_comparison_table(all_results) | |
| if len(df) == 0: | |
| print("\nNo results to compare (run experiments first).") | |
| return | |
| # Sort by Dice | |
| df = df.sort_values('Dice (mean)', ascending=False) | |
| print(f"\n{'='*80}") | |
| print("COMPARISON TABLE (LATEST SUCCESSFUL RUNS)") | |
| print(f"{'='*80}") | |
| print(df.to_string(index=False)) | |
| # Save CSV | |
| tables_dir = os.path.join(args.project_root, 'results', 'tables') | |
| os.makedirs(tables_dir, exist_ok=True) | |
| csv_path = os.path.join(tables_dir, 'comparison.csv') | |
| df.to_csv(csv_path, index=False) | |
| print(f"\nCSV saved: {csv_path}") | |
| # Save LaTeX | |
| latex = generate_latex_table(df) | |
| latex_path = os.path.join(tables_dir, 'comparison.tex') | |
| with open(latex_path, 'w') as f: | |
| f.write(latex) | |
| print(f"LaTeX saved: {latex_path}") | |
| if __name__ == '__main__': | |
| main() | |