""" Failure Mode Analysis for SAM2 on Bone Tumor X-rays. Categorizes and analyzes where SAM2 fails: - Per tumor type analysis - Image characteristic correlation (size, contrast, location) - Prompt sensitivity analysis - Edge case identification This analysis is critical for the "why foundation models fail" narrative. Usage: python -m src.analysis.failure_modes --project_root . --results results/runs/sam2_zeroshot_*/results.json """ import os import json import argparse import numpy as np import pandas as pd from collections import defaultdict from typing import Dict, List, Tuple import matplotlib.pyplot as plt import seaborn as sns CLASS_NAMES = [ 'giant cell tumor', 'multiple osteochondromas', 'osteochondroma', 'osteofibroma', 'osteosarcoma', 'other bt', 'other mt', 'simple bone cyst', 'synovial osteochondroma' ] def analyze_failure_modes(per_sample_metrics: List[Dict], dice_threshold: float = 0.5) -> Dict: """ Categorize predictions into success/failure and analyze patterns. Args: per_sample_metrics: Per-sample metrics with 'stem', 'dice', etc. dice_threshold: Below this → failure case Returns: Analysis dict with failure categories """ failures = [m for m in per_sample_metrics if m['dice'] < dice_threshold] successes = [m for m in per_sample_metrics if m['dice'] >= dice_threshold] analysis = { 'total': len(per_sample_metrics), 'n_failures': len(failures), 'n_successes': len(successes), 'failure_rate': len(failures) / max(len(per_sample_metrics), 1), 'mean_dice_failures': np.mean([m['dice'] for m in failures]) if failures else 0, 'mean_dice_successes': np.mean([m['dice'] for m in successes]) if successes else 0, } # Failure categories categories = defaultdict(list) for m in failures: stem = m.get('stem', '') dice = m['dice'] # Over-segmentation (high recall, low precision) if m.get('recall', 0) > 0.7 and m.get('precision', 0) < 0.4: categories['over_segmentation'].append(stem) # Under-segmentation (low recall, high precision) elif m.get('recall', 0) < 0.4 and m.get('precision', 0) > 0.7: categories['under_segmentation'].append(stem) # Complete miss (very low IoU) elif m.get('iou', 0) < 0.1: categories['complete_miss'].append(stem) # Boundary errors (decent overlap but poor HD95) elif dice > 0.3 and m.get('hd95', 0) > 50: categories['boundary_error'].append(stem) # Other else: categories['other'].append(stem) analysis['categories'] = {k: len(v) for k, v in categories.items()} analysis['category_stems'] = dict(categories) # Size-related analysis if all('gt_area' in m for m in per_sample_metrics): areas = np.array([m['gt_area'] for m in per_sample_metrics]) dices = np.array([m['dice'] for m in per_sample_metrics]) # Bin by area small = areas < np.percentile(areas, 33) medium = (areas >= np.percentile(areas, 33)) & (areas < np.percentile(areas, 66)) large = areas >= np.percentile(areas, 66) analysis['by_size'] = { 'small': { 'mean_dice': float(np.mean(dices[small])) if small.sum() > 0 else 0, 'n': int(small.sum()), }, 'medium': { 'mean_dice': float(np.mean(dices[medium])) if medium.sum() > 0 else 0, 'n': int(medium.sum()), }, 'large': { 'mean_dice': float(np.mean(dices[large])) if large.sum() > 0 else 0, 'n': int(large.sum()), }, } return analysis def generate_failure_report(analysis: Dict, output_dir: str): """Generate a text report of failure analysis.""" report = [] report.append("=" * 70) report.append("FAILURE MODE ANALYSIS REPORT") report.append("=" * 70) report.append("") report.append(f"Total samples: {analysis['total']}") report.append(f"Failures (Dice < 0.5): {analysis['n_failures']} ({analysis['failure_rate']*100:.1f}%)") report.append(f"Successes: {analysis['n_successes']}") report.append(f"Mean Dice (failures): {analysis['mean_dice_failures']:.4f}") report.append(f"Mean Dice (successes): {analysis['mean_dice_successes']:.4f}") report.append("") report.append("FAILURE CATEGORIES:") for category, count in sorted(analysis.get('categories', {}).items()): report.append(f" {category:25s}: {count}") report.append("") if 'by_size' in analysis: report.append("PERFORMANCE BY TUMOR SIZE:") for size, stats in analysis['by_size'].items(): report.append(f" {size:10s}: Dice = {stats['mean_dice']:.4f} (n={stats['n']})") report.append("") # Key findings for paper report.append("KEY FINDINGS FOR MANUSCRIPT:") report.append("-" * 40) cats = analysis.get('categories', {}) if cats.get('over_segmentation', 0) > cats.get('under_segmentation', 0): report.append("• SAM2 tends to OVER-SEGMENT bone tumors on X-rays") report.append(" → Model captures surrounding anatomy as tumor") elif cats.get('under_segmentation', 0) > cats.get('over_segmentation', 0): report.append("• SAM2 tends to UNDER-SEGMENT bone tumors on X-rays") report.append(" → Model misses subtle boundaries / low-contrast regions") if cats.get('complete_miss', 0) > 0: miss_rate = cats['complete_miss'] / max(analysis['total'], 1) report.append(f"• Complete segmentation failures: {miss_rate*100:.1f}% of cases") report.append(" → Foundation model has NO prior for these pathologies") if 'by_size' in analysis: sizes = analysis['by_size'] if sizes['small']['mean_dice'] < sizes['large']['mean_dice'] - 0.1: report.append("• STRONG size bias: small tumors segmented much worse") report.append(" → SAM2's resolution may be insufficient for small lesions") report_text = "\n".join(report) os.makedirs(output_dir, exist_ok=True) with open(os.path.join(output_dir, 'failure_analysis.txt'), 'w', encoding='utf-8') as f: f.write(report_text) try: print(report_text) except UnicodeEncodeError: print(report_text.encode('ascii', errors='replace').decode('ascii')) return report_text def plot_failure_distribution(per_sample_metrics: List[Dict], save_path: str = None): """Plot distribution of Dice scores to show failure patterns.""" dices = [m['dice'] for m in per_sample_metrics] fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5)) # Histogram ax1.hist(dices, bins=30, color='#2196F3', alpha=0.7, edgecolor='white') ax1.axvline(x=0.5, color='red', linestyle='--', linewidth=2, label='Failure threshold') ax1.axvline(x=np.mean(dices), color='green', linestyle='--', linewidth=2, label=f'Mean: {np.mean(dices):.3f}') ax1.set_xlabel('Dice Score') ax1.set_ylabel('Count') ax1.set_title('Distribution of Dice Scores', fontweight='bold') ax1.legend() # Dice vs predicted area if all('pred_area' in m and 'gt_area' in m for m in per_sample_metrics): gt_areas = [m['gt_area'] for m in per_sample_metrics] ax2.scatter(gt_areas, dices, alpha=0.5, c='#2196F3', s=20) ax2.set_xlabel('Ground Truth Area (pixels)') ax2.set_ylabel('Dice Score') ax2.set_title('Performance vs. Tumor Size', fontweight='bold') ax2.axhline(y=0.5, color='red', linestyle='--', alpha=0.5) # Fit trend line valid = np.isfinite(gt_areas) & np.isfinite(dices) if np.sum(valid) > 10: z = np.polyfit(np.array(gt_areas)[valid], np.array(dices)[valid], 1) p = np.poly1d(z) x_range = np.linspace(min(gt_areas), max(gt_areas), 100) ax2.plot(x_range, p(x_range), 'r-', linewidth=2, alpha=0.7, label='Trend') ax2.legend() plt.tight_layout() if save_path: os.makedirs(os.path.dirname(save_path), exist_ok=True) plt.savefig(save_path, dpi=300, bbox_inches='tight') plt.close() else: plt.show() def main(): parser = argparse.ArgumentParser(description='Failure mode analysis') parser.add_argument('--project_root', type=str, default='.') parser.add_argument('--results', type=str, required=True, help='Path to results.json file') parser.add_argument('--threshold', type=float, default=0.5) args = parser.parse_args() with open(args.results, 'r') as f: results = json.load(f) # Handle multi-prompt results if 'box' in results: per_sample = results['box']['per_sample'] elif 'per_sample' in results: per_sample = results['per_sample'] else: print("Could not find per_sample metrics in results file.") return output_dir = os.path.join(args.project_root, 'results', 'analysis') analysis = analyze_failure_modes(per_sample, dice_threshold=args.threshold) generate_failure_report(analysis, output_dir) plot_failure_distribution( per_sample, save_path=os.path.join(output_dir, 'failure_distribution.pdf') ) print(f"\nAnalysis saved to: {output_dir}") if __name__ == '__main__': main()