File size: 11,631 Bytes
1bd13a5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
"""
Unified Evaluation Script.

Loads saved results from all experiments and produces:
- Comparison tables (LaTeX + CSV)
- Summary statistics
- Statistical significance tests

Usage:
    python -m src.evaluate --project_root . --results_dir results/runs
"""

import os
import json
import glob
import argparse
import numpy as np
import pandas as pd
from pathlib import Path
from typing import Dict, List

from src.metrics import compare_models_statistical


CLASS_NAMES = [
    'Giant Cell Tumor', 'Multiple Osteochondromas', 'Osteochondroma',
    'Osteofibroma', 'Osteosarcoma', 'Other BT', 'Other MT',
    'Simple Bone Cyst', 'Synovial Osteochondroma'
]


def get_clean_model_name(run_name: str, subkey: str = None) -> str:
    """Map a raw run folder name and optional subkey to a clean, publication-ready name."""
    if run_name.startswith('unet_'):
        return 'UNet Baseline'
    elif run_name.startswith('sam2_zeroshot_'):
        if subkey == 'box':
            return 'SAM2 (Zero-Shot Box)'
        elif subkey == 'center_point':
            return 'SAM2 (Zero-Shot Center Point)'
        elif subkey == 'random_3_points':
            return 'SAM2 (Zero-Shot 3 Points)'
        return f'SAM2 Zero-Shot ({subkey})'
    elif run_name.startswith('sam2_fewshot_'):
        shot_num = subkey.replace('_shot', '') if subkey else 'Few'
        return f'SAM2 ({shot_num}-Shot Box)'
    elif run_name.startswith('sam2_finetune_'):
        return 'SAM2 (Fine-Tuned Box)'
    elif run_name.startswith('medsam_zeroshot_'):
        return 'MedSAM (Zero-Shot Box)'
    elif run_name.startswith('sam2_ablation_'):
        if subkey == 'baseline_padded':
            return 'SAM2 (Padded Box)'
        elif subkey == 'iterative_3pass':
            return 'SAM2 (Iterative Box)'
        elif subkey == 'iterative_postprocess':
            return 'SAM2 (Iterative + Morph Box)'
        elif subkey == 'full_pipeline':
            return 'SAM2 (Full Pipeline)'
        return f'SAM2 Ablation ({subkey})'
    elif run_name.startswith('sam2_enhanced_'):
        if subkey == 'enhanced':
            return 'SAM2 (Enhanced Weak-Label)'
        return f'SAM2 Enhanced ({subkey})'
    else:
        name = run_name
        if subkey:
            name += f" ({subkey})"
        return name


def filter_latest_runs(run_dirs: List[str]) -> List[str]:
    """Filter run directories to keep only the latest run for each experiment type."""
    prefixes = ['unet', 'sam2_zeroshot', 'sam2_fewshot', 'sam2_finetune', 'medsam_zeroshot', 'sam2_enhanced', 'sam2_ablation']
    latest_by_prefix = {}
    
    for d in run_dirs:
        name = Path(d).name
        matched_prefix = None
        for p in prefixes:
            if name.startswith(p + '_'):
                matched_prefix = p
                break
        
        if matched_prefix:
            if matched_prefix not in latest_by_prefix or name > Path(latest_by_prefix[matched_prefix]).name:
                latest_by_prefix[matched_prefix] = d
        else:
            latest_by_prefix[name] = d
            
    return list(latest_by_prefix.values())


def load_run_results(run_dir: str) -> Dict:
    """Load results and config from a run directory."""
    results_path = os.path.join(run_dir, 'results.json')
    config_path = os.path.join(run_dir, 'config.json')
    
    data = {}
    
    if os.path.exists(results_path):
        with open(results_path, 'r') as f:
            data['results'] = json.load(f)
    
    if os.path.exists(config_path):
        with open(config_path, 'r') as f:
            data['config'] = json.load(f)
    
    data['run_name'] = Path(run_dir).name
    
    return data


def create_comparison_table(all_results: List[Dict]) -> pd.DataFrame:
    """
    Create a comparison table across all experiments.
    
    Returns:
        DataFrame with models as rows, metrics as columns
    """
    rows = []
    
    for run_data in all_results:
        name = run_data['run_name']
        
        if 'results' not in run_data:
            continue
        
        results = run_data['results']
        
        # Handle different result structures
        if isinstance(results, dict):
            # Could be per-prompt-type results (SAM2 zero-shot)
            if any(k in results for k in ['box', 'center_point', 'random_3_points']):
                for prompt_type, pt_results in results.items():
                    if isinstance(pt_results, dict) and 'overall' in pt_results:
                        overall = pt_results['overall']
                        row = {
                            'Model': get_clean_model_name(name, prompt_type),
                            'Dice (mean)': overall.get('dice', {}).get('mean', np.nan),
                            'Dice (std)': overall.get('dice', {}).get('std', np.nan),
                            'IoU (mean)': overall.get('iou', {}).get('mean', np.nan),
                            'IoU (std)': overall.get('iou', {}).get('std', np.nan),
                            'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan),
                            'Precision': overall.get('precision', {}).get('mean', np.nan),
                            'Recall': overall.get('recall', {}).get('mean', np.nan),
                            'N': pt_results.get('n_samples', 0),
                        }
                        rows.append(row)
            # Could be few-shot results
            elif any(k.endswith('_shot') for k in results.keys()):
                for shot_type, shot_results in results.items():
                    if isinstance(shot_results, dict) and 'overall' in shot_results:
                        overall = shot_results['overall']
                        row = {
                            'Model': get_clean_model_name(name, shot_type),
                            'Dice (mean)': overall.get('dice', {}).get('mean', np.nan),
                            'Dice (std)': overall.get('dice', {}).get('std', np.nan),
                            'IoU (mean)': overall.get('iou', {}).get('mean', np.nan),
                            'IoU (std)': overall.get('iou', {}).get('std', np.nan),
                            'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan),
                            'Precision': overall.get('precision', {}).get('mean', np.nan),
                            'Recall': overall.get('recall', {}).get('mean', np.nan),
                            'N': shot_results.get('n_samples', 0),
                        }
                        rows.append(row)
            # Could be ablation or enhanced results
            elif any(k in results for k in ['baseline_padded', 'iterative_3pass', 'iterative_postprocess', 'full_pipeline', 'enhanced']):
                for config_name, config_results in results.items():
                    if isinstance(config_results, dict) and 'overall' in config_results:
                        overall = config_results['overall']
                        row = {
                            'Model': get_clean_model_name(name, config_name),
                            'Dice (mean)': overall.get('dice', {}).get('mean', np.nan),
                            'Dice (std)': overall.get('dice', {}).get('std', np.nan),
                            'IoU (mean)': overall.get('iou', {}).get('mean', np.nan),
                            'IoU (std)': overall.get('iou', {}).get('std', np.nan),
                            'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan),
                            'Precision': overall.get('precision', {}).get('mean', np.nan),
                            'Recall': overall.get('recall', {}).get('mean', np.nan),
                            'N': config_results.get('n_samples', 0),
                        }
                        rows.append(row)
            elif 'overall' in results:
                overall = results['overall']
                row = {
                    'Model': get_clean_model_name(name),
                    'Dice (mean)': overall.get('dice', {}).get('mean', np.nan),
                    'Dice (std)': overall.get('dice', {}).get('std', np.nan),
                    'IoU (mean)': overall.get('iou', {}).get('mean', np.nan),
                    'IoU (std)': overall.get('iou', {}).get('std', np.nan),
                    'HD95 (mean)': overall.get('hd95', {}).get('mean', np.nan),
                    'Precision': overall.get('precision', {}).get('mean', np.nan),
                    'Recall': overall.get('recall', {}).get('mean', np.nan),
                    'N': results.get('n_samples', 0),
                }
                rows.append(row)
    
    return pd.DataFrame(rows)


def generate_latex_table(df: pd.DataFrame, caption: str = "Segmentation Performance Comparison") -> str:
    """Generate a LaTeX table from comparison DataFrame."""
    
    latex = "\\begin{table}[htbp]\n"
    latex += f"\\caption{{{caption}}}\n"
    latex += "\\centering\n"
    latex += "\\begin{tabular}{l c c c c c}\n"
    latex += "\\toprule\n"
    latex += "Model & Dice & IoU & HD95 & Precision & Recall \\\\\n"
    latex += "\\midrule\n"
    
    for _, row in df.iterrows():
        dice_str = f"{row['Dice (mean)']:.4f} \\pm {row['Dice (std)']:.4f}"
        iou_str = f"{row['IoU (mean)']:.4f}"
        hd95_str = f"{row['HD95 (mean)']:.2f}" if not np.isnan(row['HD95 (mean)']) else "—"
        prec_str = f"{row['Precision']:.4f}"
        rec_str = f"{row['Recall']:.4f}"
        
        latex += f"{row['Model']} & {dice_str} & {iou_str} & {hd95_str} & {prec_str} & {rec_str} \\\\\n"
    
    latex += "\\bottomrule\n"
    latex += "\\end{tabular}\n"
    latex += "\\label{tab:comparison}\n"
    latex += "\\end{table}\n"
    
    return latex


def main():
    parser = argparse.ArgumentParser(description='Unified evaluation')
    parser.add_argument('--project_root', type=str, default='.')
    parser.add_argument('--results_dir', type=str, default=None)
    args = parser.parse_args()
    
    if args.results_dir is None:
        args.results_dir = os.path.join(args.project_root, 'results', 'runs')
    
    # Find all run directories
    run_dirs = sorted(glob.glob(os.path.join(args.results_dir, '*')))
    run_dirs = [d for d in run_dirs if os.path.isdir(d)]
    
    if not run_dirs:
        print(f"No runs found in {args.results_dir}")
        return
        
    # Chronologically filter run directories to keep only the latest of each type
    run_dirs = filter_latest_runs(run_dirs)
    
    print(f"Comparing the following {len(run_dirs)} latest runs:")
    for d in sorted(run_dirs):
        print(f"  - {Path(d).name}")
    
    # Load all results
    all_results = [load_run_results(d) for d in run_dirs]
    
    # Create comparison table
    df = create_comparison_table(all_results)
    
    if len(df) == 0:
        print("\nNo results to compare (run experiments first).")
        return
    
    # Sort by Dice
    df = df.sort_values('Dice (mean)', ascending=False)
    
    print(f"\n{'='*80}")
    print("COMPARISON TABLE (LATEST SUCCESSFUL RUNS)")
    print(f"{'='*80}")
    print(df.to_string(index=False))
    
    # Save CSV
    tables_dir = os.path.join(args.project_root, 'results', 'tables')
    os.makedirs(tables_dir, exist_ok=True)
    
    csv_path = os.path.join(tables_dir, 'comparison.csv')
    df.to_csv(csv_path, index=False)
    print(f"\nCSV saved: {csv_path}")
    
    # Save LaTeX
    latex = generate_latex_table(df)
    latex_path = os.path.join(tables_dir, 'comparison.tex')
    with open(latex_path, 'w') as f:
        f.write(latex)
    print(f"LaTeX saved: {latex_path}")


if __name__ == '__main__':
    main()