btxrd-workstation / src /analysis /failure_modes.py
amitaddi
feat: deploy bone tumor segmentation workstation
1bd13a5
Raw History Blame Contribute Delete
9.7 kB
"""
Failure Mode Analysis for SAM2 on Bone Tumor X-rays.
Categorizes and analyzes where SAM2 fails:
- Per tumor type analysis
- Image characteristic correlation (size, contrast, location)
- Prompt sensitivity analysis
- Edge case identification
This analysis is critical for the "why foundation models fail" narrative.
Usage:
python -m src.analysis.failure_modes --project_root . --results results/runs/sam2_zeroshot_*/results.json
"""
import os
import json
import argparse
import numpy as np
import pandas as pd
from collections import defaultdict
from typing import Dict, List, Tuple
import matplotlib.pyplot as plt
import seaborn as sns
CLASS_NAMES = [
'giant cell tumor', 'multiple osteochondromas', 'osteochondroma',
'osteofibroma', 'osteosarcoma', 'other bt', 'other mt',
'simple bone cyst', 'synovial osteochondroma'
]
def analyze_failure_modes(per_sample_metrics: List[Dict],
dice_threshold: float = 0.5) -> Dict:
"""
Categorize predictions into success/failure and analyze patterns.
Args:
per_sample_metrics: Per-sample metrics with 'stem', 'dice', etc.
dice_threshold: Below this → failure case
Returns:
Analysis dict with failure categories
"""
failures = [m for m in per_sample_metrics if m['dice'] < dice_threshold]
successes = [m for m in per_sample_metrics if m['dice'] >= dice_threshold]
analysis = {
'total': len(per_sample_metrics),
'n_failures': len(failures),
'n_successes': len(successes),
'failure_rate': len(failures) / max(len(per_sample_metrics), 1),
'mean_dice_failures': np.mean([m['dice'] for m in failures]) if failures else 0,
'mean_dice_successes': np.mean([m['dice'] for m in successes]) if successes else 0,
}
# Failure categories
categories = defaultdict(list)
for m in failures:
stem = m.get('stem', '')
dice = m['dice']
# Over-segmentation (high recall, low precision)
if m.get('recall', 0) > 0.7 and m.get('precision', 0) < 0.4:
categories['over_segmentation'].append(stem)
# Under-segmentation (low recall, high precision)
elif m.get('recall', 0) < 0.4 and m.get('precision', 0) > 0.7:
categories['under_segmentation'].append(stem)
# Complete miss (very low IoU)
elif m.get('iou', 0) < 0.1:
categories['complete_miss'].append(stem)
# Boundary errors (decent overlap but poor HD95)
elif dice > 0.3 and m.get('hd95', 0) > 50:
categories['boundary_error'].append(stem)
# Other
else:
categories['other'].append(stem)
analysis['categories'] = {k: len(v) for k, v in categories.items()}
analysis['category_stems'] = dict(categories)
# Size-related analysis
if all('gt_area' in m for m in per_sample_metrics):
areas = np.array([m['gt_area'] for m in per_sample_metrics])
dices = np.array([m['dice'] for m in per_sample_metrics])
# Bin by area
small = areas < np.percentile(areas, 33)
medium = (areas >= np.percentile(areas, 33)) & (areas < np.percentile(areas, 66))
large = areas >= np.percentile(areas, 66)
analysis['by_size'] = {
'small': {
'mean_dice': float(np.mean(dices[small])) if small.sum() > 0 else 0,
'n': int(small.sum()),
},
'medium': {
'mean_dice': float(np.mean(dices[medium])) if medium.sum() > 0 else 0,
'n': int(medium.sum()),
},
'large': {
'mean_dice': float(np.mean(dices[large])) if large.sum() > 0 else 0,
'n': int(large.sum()),
},
}
return analysis
def generate_failure_report(analysis: Dict, output_dir: str):
"""Generate a text report of failure analysis."""
report = []
report.append("=" * 70)
report.append("FAILURE MODE ANALYSIS REPORT")
report.append("=" * 70)
report.append("")
report.append(f"Total samples: {analysis['total']}")
report.append(f"Failures (Dice < 0.5): {analysis['n_failures']} ({analysis['failure_rate']*100:.1f}%)")
report.append(f"Successes: {analysis['n_successes']}")
report.append(f"Mean Dice (failures): {analysis['mean_dice_failures']:.4f}")
report.append(f"Mean Dice (successes): {analysis['mean_dice_successes']:.4f}")
report.append("")
report.append("FAILURE CATEGORIES:")
for category, count in sorted(analysis.get('categories', {}).items()):
report.append(f" {category:25s}: {count}")
report.append("")
if 'by_size' in analysis:
report.append("PERFORMANCE BY TUMOR SIZE:")
for size, stats in analysis['by_size'].items():
report.append(f" {size:10s}: Dice = {stats['mean_dice']:.4f} (n={stats['n']})")
report.append("")
# Key findings for paper
report.append("KEY FINDINGS FOR MANUSCRIPT:")
report.append("-" * 40)
cats = analysis.get('categories', {})
if cats.get('over_segmentation', 0) > cats.get('under_segmentation', 0):
report.append("• SAM2 tends to OVER-SEGMENT bone tumors on X-rays")
report.append(" → Model captures surrounding anatomy as tumor")
elif cats.get('under_segmentation', 0) > cats.get('over_segmentation', 0):
report.append("• SAM2 tends to UNDER-SEGMENT bone tumors on X-rays")
report.append(" → Model misses subtle boundaries / low-contrast regions")
if cats.get('complete_miss', 0) > 0:
miss_rate = cats['complete_miss'] / max(analysis['total'], 1)
report.append(f"• Complete segmentation failures: {miss_rate*100:.1f}% of cases")
report.append(" → Foundation model has NO prior for these pathologies")
if 'by_size' in analysis:
sizes = analysis['by_size']
if sizes['small']['mean_dice'] < sizes['large']['mean_dice'] - 0.1:
report.append("• STRONG size bias: small tumors segmented much worse")
report.append(" → SAM2's resolution may be insufficient for small lesions")
report_text = "\n".join(report)
os.makedirs(output_dir, exist_ok=True)
with open(os.path.join(output_dir, 'failure_analysis.txt'), 'w', encoding='utf-8') as f:
f.write(report_text)
try:
print(report_text)
except UnicodeEncodeError:
print(report_text.encode('ascii', errors='replace').decode('ascii'))
return report_text
def plot_failure_distribution(per_sample_metrics: List[Dict],
save_path: str = None):
"""Plot distribution of Dice scores to show failure patterns."""
dices = [m['dice'] for m in per_sample_metrics]
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
# Histogram
ax1.hist(dices, bins=30, color='#2196F3', alpha=0.7, edgecolor='white')
ax1.axvline(x=0.5, color='red', linestyle='--', linewidth=2, label='Failure threshold')
ax1.axvline(x=np.mean(dices), color='green', linestyle='--', linewidth=2,
label=f'Mean: {np.mean(dices):.3f}')
ax1.set_xlabel('Dice Score')
ax1.set_ylabel('Count')
ax1.set_title('Distribution of Dice Scores', fontweight='bold')
ax1.legend()
# Dice vs predicted area
if all('pred_area' in m and 'gt_area' in m for m in per_sample_metrics):
gt_areas = [m['gt_area'] for m in per_sample_metrics]
ax2.scatter(gt_areas, dices, alpha=0.5, c='#2196F3', s=20)
ax2.set_xlabel('Ground Truth Area (pixels)')
ax2.set_ylabel('Dice Score')
ax2.set_title('Performance vs. Tumor Size', fontweight='bold')
ax2.axhline(y=0.5, color='red', linestyle='--', alpha=0.5)
# Fit trend line
valid = np.isfinite(gt_areas) & np.isfinite(dices)
if np.sum(valid) > 10:
z = np.polyfit(np.array(gt_areas)[valid], np.array(dices)[valid], 1)
p = np.poly1d(z)
x_range = np.linspace(min(gt_areas), max(gt_areas), 100)
ax2.plot(x_range, p(x_range), 'r-', linewidth=2, alpha=0.7, label='Trend')
ax2.legend()
plt.tight_layout()
if save_path:
os.makedirs(os.path.dirname(save_path), exist_ok=True)
plt.savefig(save_path, dpi=300, bbox_inches='tight')
plt.close()
else:
plt.show()
def main():
parser = argparse.ArgumentParser(description='Failure mode analysis')
parser.add_argument('--project_root', type=str, default='.')
parser.add_argument('--results', type=str, required=True,
help='Path to results.json file')
parser.add_argument('--threshold', type=float, default=0.5)
args = parser.parse_args()
with open(args.results, 'r') as f:
results = json.load(f)
# Handle multi-prompt results
if 'box' in results:
per_sample = results['box']['per_sample']
elif 'per_sample' in results:
per_sample = results['per_sample']
else:
print("Could not find per_sample metrics in results file.")
return
output_dir = os.path.join(args.project_root, 'results', 'analysis')
analysis = analyze_failure_modes(per_sample, dice_threshold=args.threshold)
generate_failure_report(analysis, output_dir)
plot_failure_distribution(
per_sample,
save_path=os.path.join(output_dir, 'failure_distribution.pdf')
)
print(f"\nAnalysis saved to: {output_dir}")
if __name__ == '__main__':
main()