Spaces:
Sleeping
Sleeping
Download src/analysis/failure_modes.py from amitaddi/btxrd-workstation: direct link, hf CLI and curl.
- Browser
- Download file 9.7 kB
-
https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/analysis/failure_modes.py
- Command line
-
hf download hf://spaces/amitaddi/btxrd-workstation/src/analysis/failure_modes.py
-
curl -L -o failure_modes.py https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/analysis/failure_modes.py
9.7 kB
| """ | |
| Failure Mode Analysis for SAM2 on Bone Tumor X-rays. | |
| Categorizes and analyzes where SAM2 fails: | |
| - Per tumor type analysis | |
| - Image characteristic correlation (size, contrast, location) | |
| - Prompt sensitivity analysis | |
| - Edge case identification | |
| This analysis is critical for the "why foundation models fail" narrative. | |
| Usage: | |
| python -m src.analysis.failure_modes --project_root . --results results/runs/sam2_zeroshot_*/results.json | |
| """ | |
| import os | |
| import json | |
| import argparse | |
| import numpy as np | |
| import pandas as pd | |
| from collections import defaultdict | |
| from typing import Dict, List, Tuple | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| CLASS_NAMES = [ | |
| 'giant cell tumor', 'multiple osteochondromas', 'osteochondroma', | |
| 'osteofibroma', 'osteosarcoma', 'other bt', 'other mt', | |
| 'simple bone cyst', 'synovial osteochondroma' | |
| ] | |
| def analyze_failure_modes(per_sample_metrics: List[Dict], | |
| dice_threshold: float = 0.5) -> Dict: | |
| """ | |
| Categorize predictions into success/failure and analyze patterns. | |
| Args: | |
| per_sample_metrics: Per-sample metrics with 'stem', 'dice', etc. | |
| dice_threshold: Below this → failure case | |
| Returns: | |
| Analysis dict with failure categories | |
| """ | |
| failures = [m for m in per_sample_metrics if m['dice'] < dice_threshold] | |
| successes = [m for m in per_sample_metrics if m['dice'] >= dice_threshold] | |
| analysis = { | |
| 'total': len(per_sample_metrics), | |
| 'n_failures': len(failures), | |
| 'n_successes': len(successes), | |
| 'failure_rate': len(failures) / max(len(per_sample_metrics), 1), | |
| 'mean_dice_failures': np.mean([m['dice'] for m in failures]) if failures else 0, | |
| 'mean_dice_successes': np.mean([m['dice'] for m in successes]) if successes else 0, | |
| } | |
| # Failure categories | |
| categories = defaultdict(list) | |
| for m in failures: | |
| stem = m.get('stem', '') | |
| dice = m['dice'] | |
| # Over-segmentation (high recall, low precision) | |
| if m.get('recall', 0) > 0.7 and m.get('precision', 0) < 0.4: | |
| categories['over_segmentation'].append(stem) | |
| # Under-segmentation (low recall, high precision) | |
| elif m.get('recall', 0) < 0.4 and m.get('precision', 0) > 0.7: | |
| categories['under_segmentation'].append(stem) | |
| # Complete miss (very low IoU) | |
| elif m.get('iou', 0) < 0.1: | |
| categories['complete_miss'].append(stem) | |
| # Boundary errors (decent overlap but poor HD95) | |
| elif dice > 0.3 and m.get('hd95', 0) > 50: | |
| categories['boundary_error'].append(stem) | |
| # Other | |
| else: | |
| categories['other'].append(stem) | |
| analysis['categories'] = {k: len(v) for k, v in categories.items()} | |
| analysis['category_stems'] = dict(categories) | |
| # Size-related analysis | |
| if all('gt_area' in m for m in per_sample_metrics): | |
| areas = np.array([m['gt_area'] for m in per_sample_metrics]) | |
| dices = np.array([m['dice'] for m in per_sample_metrics]) | |
| # Bin by area | |
| small = areas < np.percentile(areas, 33) | |
| medium = (areas >= np.percentile(areas, 33)) & (areas < np.percentile(areas, 66)) | |
| large = areas >= np.percentile(areas, 66) | |
| analysis['by_size'] = { | |
| 'small': { | |
| 'mean_dice': float(np.mean(dices[small])) if small.sum() > 0 else 0, | |
| 'n': int(small.sum()), | |
| }, | |
| 'medium': { | |
| 'mean_dice': float(np.mean(dices[medium])) if medium.sum() > 0 else 0, | |
| 'n': int(medium.sum()), | |
| }, | |
| 'large': { | |
| 'mean_dice': float(np.mean(dices[large])) if large.sum() > 0 else 0, | |
| 'n': int(large.sum()), | |
| }, | |
| } | |
| return analysis | |
| def generate_failure_report(analysis: Dict, output_dir: str): | |
| """Generate a text report of failure analysis.""" | |
| report = [] | |
| report.append("=" * 70) | |
| report.append("FAILURE MODE ANALYSIS REPORT") | |
| report.append("=" * 70) | |
| report.append("") | |
| report.append(f"Total samples: {analysis['total']}") | |
| report.append(f"Failures (Dice < 0.5): {analysis['n_failures']} ({analysis['failure_rate']*100:.1f}%)") | |
| report.append(f"Successes: {analysis['n_successes']}") | |
| report.append(f"Mean Dice (failures): {analysis['mean_dice_failures']:.4f}") | |
| report.append(f"Mean Dice (successes): {analysis['mean_dice_successes']:.4f}") | |
| report.append("") | |
| report.append("FAILURE CATEGORIES:") | |
| for category, count in sorted(analysis.get('categories', {}).items()): | |
| report.append(f" {category:25s}: {count}") | |
| report.append("") | |
| if 'by_size' in analysis: | |
| report.append("PERFORMANCE BY TUMOR SIZE:") | |
| for size, stats in analysis['by_size'].items(): | |
| report.append(f" {size:10s}: Dice = {stats['mean_dice']:.4f} (n={stats['n']})") | |
| report.append("") | |
| # Key findings for paper | |
| report.append("KEY FINDINGS FOR MANUSCRIPT:") | |
| report.append("-" * 40) | |
| cats = analysis.get('categories', {}) | |
| if cats.get('over_segmentation', 0) > cats.get('under_segmentation', 0): | |
| report.append("• SAM2 tends to OVER-SEGMENT bone tumors on X-rays") | |
| report.append(" → Model captures surrounding anatomy as tumor") | |
| elif cats.get('under_segmentation', 0) > cats.get('over_segmentation', 0): | |
| report.append("• SAM2 tends to UNDER-SEGMENT bone tumors on X-rays") | |
| report.append(" → Model misses subtle boundaries / low-contrast regions") | |
| if cats.get('complete_miss', 0) > 0: | |
| miss_rate = cats['complete_miss'] / max(analysis['total'], 1) | |
| report.append(f"• Complete segmentation failures: {miss_rate*100:.1f}% of cases") | |
| report.append(" → Foundation model has NO prior for these pathologies") | |
| if 'by_size' in analysis: | |
| sizes = analysis['by_size'] | |
| if sizes['small']['mean_dice'] < sizes['large']['mean_dice'] - 0.1: | |
| report.append("• STRONG size bias: small tumors segmented much worse") | |
| report.append(" → SAM2's resolution may be insufficient for small lesions") | |
| report_text = "\n".join(report) | |
| os.makedirs(output_dir, exist_ok=True) | |
| with open(os.path.join(output_dir, 'failure_analysis.txt'), 'w', encoding='utf-8') as f: | |
| f.write(report_text) | |
| try: | |
| print(report_text) | |
| except UnicodeEncodeError: | |
| print(report_text.encode('ascii', errors='replace').decode('ascii')) | |
| return report_text | |
| def plot_failure_distribution(per_sample_metrics: List[Dict], | |
| save_path: str = None): | |
| """Plot distribution of Dice scores to show failure patterns.""" | |
| dices = [m['dice'] for m in per_sample_metrics] | |
| fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5)) | |
| # Histogram | |
| ax1.hist(dices, bins=30, color='#2196F3', alpha=0.7, edgecolor='white') | |
| ax1.axvline(x=0.5, color='red', linestyle='--', linewidth=2, label='Failure threshold') | |
| ax1.axvline(x=np.mean(dices), color='green', linestyle='--', linewidth=2, | |
| label=f'Mean: {np.mean(dices):.3f}') | |
| ax1.set_xlabel('Dice Score') | |
| ax1.set_ylabel('Count') | |
| ax1.set_title('Distribution of Dice Scores', fontweight='bold') | |
| ax1.legend() | |
| # Dice vs predicted area | |
| if all('pred_area' in m and 'gt_area' in m for m in per_sample_metrics): | |
| gt_areas = [m['gt_area'] for m in per_sample_metrics] | |
| ax2.scatter(gt_areas, dices, alpha=0.5, c='#2196F3', s=20) | |
| ax2.set_xlabel('Ground Truth Area (pixels)') | |
| ax2.set_ylabel('Dice Score') | |
| ax2.set_title('Performance vs. Tumor Size', fontweight='bold') | |
| ax2.axhline(y=0.5, color='red', linestyle='--', alpha=0.5) | |
| # Fit trend line | |
| valid = np.isfinite(gt_areas) & np.isfinite(dices) | |
| if np.sum(valid) > 10: | |
| z = np.polyfit(np.array(gt_areas)[valid], np.array(dices)[valid], 1) | |
| p = np.poly1d(z) | |
| x_range = np.linspace(min(gt_areas), max(gt_areas), 100) | |
| ax2.plot(x_range, p(x_range), 'r-', linewidth=2, alpha=0.7, label='Trend') | |
| ax2.legend() | |
| plt.tight_layout() | |
| if save_path: | |
| os.makedirs(os.path.dirname(save_path), exist_ok=True) | |
| plt.savefig(save_path, dpi=300, bbox_inches='tight') | |
| plt.close() | |
| else: | |
| plt.show() | |
| def main(): | |
| parser = argparse.ArgumentParser(description='Failure mode analysis') | |
| parser.add_argument('--project_root', type=str, default='.') | |
| parser.add_argument('--results', type=str, required=True, | |
| help='Path to results.json file') | |
| parser.add_argument('--threshold', type=float, default=0.5) | |
| args = parser.parse_args() | |
| with open(args.results, 'r') as f: | |
| results = json.load(f) | |
| # Handle multi-prompt results | |
| if 'box' in results: | |
| per_sample = results['box']['per_sample'] | |
| elif 'per_sample' in results: | |
| per_sample = results['per_sample'] | |
| else: | |
| print("Could not find per_sample metrics in results file.") | |
| return | |
| output_dir = os.path.join(args.project_root, 'results', 'analysis') | |
| analysis = analyze_failure_modes(per_sample, dice_threshold=args.threshold) | |
| generate_failure_report(analysis, output_dir) | |
| plot_failure_distribution( | |
| per_sample, | |
| save_path=os.path.join(output_dir, 'failure_distribution.pdf') | |
| ) | |
| print(f"\nAnalysis saved to: {output_dir}") | |
| if __name__ == '__main__': | |
| main() | |