btxrd-workstation / src /models /sam2_wrapper.py
amitaddi
feat: deploy bone tumor segmentation workstation
1bd13a5
Raw History Blame Contribute Delete
17.3 kB
"""
SAM2 Wrapper for Bone Tumor Segmentation.
Provides a unified interface for:
- Zero-shot inference with box/point/mask prompts
- Few-shot evaluation
- Fine-tuning support
Supports all SAM2 model variants:
sam2_hiera_tiny, sam2_hiera_small, sam2_hiera_base_plus, sam2_hiera_large
"""
import os
import numpy as np
import torch
from typing import Optional, List, Dict, Tuple, Union
# SAM2 imports — these come from Meta's official sam2 package
# Install: pip install git+https://github.com/facebookresearch/sam2.git
try:
from sam2.build_sam import build_sam2
from sam2.sam2_image_predictor import SAM2ImagePredictor
SAM2_AVAILABLE = True
except ImportError:
SAM2_AVAILABLE = False
print("WARNING: sam2 not installed. Install with: pip install git+https://github.com/facebookresearch/sam2.git")
# Model config paths (relative to sam2 package)
SAM2_CONFIGS = {
'sam2_hiera_tiny': 'configs/sam2/sam2_hiera_t.yaml',
'sam2_hiera_small': 'configs/sam2/sam2_hiera_s.yaml',
'sam2_hiera_base_plus': 'configs/sam2/sam2_hiera_b+.yaml',
'sam2_hiera_large': 'configs/sam2/sam2_hiera_l.yaml',
}
# Checkpoint URLs for auto-download
SAM2_CHECKPOINTS = {
'sam2_hiera_tiny': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_tiny.pt',
'sam2_hiera_small': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_small.pt',
'sam2_hiera_base_plus': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_base_plus.pt',
'sam2_hiera_large': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt',
}
class SAM2Wrapper:
"""
Wrapper around SAM2 for bone tumor segmentation.
Provides clean interface for box-prompted, point-prompted,
and combined prompt inference.
"""
def __init__(self,
variant: str = 'sam2_hiera_large',
checkpoint_path: Optional[str] = None,
device: str = 'cuda'):
"""
Args:
variant: SAM2 model variant name
checkpoint_path: Path to checkpoint (auto-downloads if None)
device: 'cuda' or 'cpu'
"""
if not SAM2_AVAILABLE:
raise RuntimeError("sam2 package not installed")
self.variant = variant
self.device = device
# Resolve checkpoint
if checkpoint_path is None:
checkpoint_path = self._get_or_download_checkpoint(variant)
# Build model
config = SAM2_CONFIGS[variant]
self.model = build_sam2(config, checkpoint_path, device=device)
self.predictor = SAM2ImagePredictor(self.model)
print(f"SAM2 ({variant}) loaded on {device}")
def _get_or_download_checkpoint(self, variant: str) -> str:
"""Download checkpoint if not present."""
cache_dir = os.path.join(os.path.expanduser('~'), '.cache', 'sam2')
os.makedirs(cache_dir, exist_ok=True)
filename = f"{variant}.pt"
filepath = os.path.join(cache_dir, filename)
if not os.path.exists(filepath):
url = SAM2_CHECKPOINTS[variant]
print(f"Downloading {variant} checkpoint...")
print(f" URL: {url}")
print(f" Saving to: {filepath}")
import urllib.request
urllib.request.urlretrieve(url, filepath)
print(" Download complete!")
return filepath
def set_image(self, image: np.ndarray):
"""
Set the image for prediction. Must be called before predict_*.
Args:
image: (H, W, 3) RGB numpy array
"""
self.predictor.set_image(image)
self._current_image = image
def predict_with_box(self,
box: np.ndarray,
multimask_output: bool = False) -> Tuple[np.ndarray, float]:
"""
Predict segmentation mask given a bounding box prompt.
Args:
box: (4,) array [x1, y1, x2, y2]
multimask_output: If True, returns 3 masks with scores
Returns:
mask: (H, W) binary mask
score: Predicted IoU score from SAM2
"""
masks, scores, logits = self.predictor.predict(
point_coords=None,
point_labels=None,
box=box[None, :], # (1, 4)
multimask_output=multimask_output,
)
# Take the best mask
best_idx = scores.argmax()
mask = masks[best_idx] # (H, W)
score = float(scores[best_idx])
return mask.astype(np.uint8), score
def predict_with_points(self,
points: np.ndarray,
labels: np.ndarray,
multimask_output: bool = True) -> Tuple[np.ndarray, float]:
"""
Predict with point prompts.
Args:
points: (N, 2) array of (x, y) coordinates
labels: (N,) array of 1 (foreground) or 0 (background)
multimask_output: Return 3 mask candidates
Returns:
mask: (H, W) binary mask
score: Predicted IoU score
"""
masks, scores, logits = self.predictor.predict(
point_coords=points,
point_labels=labels,
box=None,
multimask_output=multimask_output,
)
best_idx = scores.argmax()
return masks[best_idx].astype(np.uint8), float(scores[best_idx])
def predict_with_box_and_points(self,
box: np.ndarray,
points: np.ndarray,
labels: np.ndarray) -> Tuple[np.ndarray, float]:
"""
Predict with combined box + point prompts.
Args:
box: (4,) array [x1, y1, x2, y2]
points: (N, 2) point coordinates
labels: (N,) point labels (1=fg, 0=bg)
Returns:
mask: (H, W) binary mask
score: Predicted IoU
"""
masks, scores, logits = self.predictor.predict(
point_coords=points,
point_labels=labels,
box=box[None, :],
multimask_output=False,
)
best_idx = scores.argmax()
return masks[best_idx].astype(np.uint8), float(scores[best_idx])
def predict_all_objects(self,
image: np.ndarray,
prompts: List[Dict],
prompt_type: str = 'box') -> List[Dict]:
"""
Predict masks for all objects in an image.
Args:
image: (H, W, 3) RGB image
prompts: List of prompt dicts with 'box', 'center_point', 'class_id'
prompt_type: 'box', 'center_point', 'random_3_points', 'box_and_center'
Returns:
List of result dicts with 'mask', 'score', 'class_id'
"""
self.set_image(image)
results = []
for prompt in prompts:
if prompt_type == 'box':
mask, score = self.predict_with_box(prompt['box'])
elif prompt_type == 'center_point':
points = prompt['center_point'].reshape(1, 2)
labels = np.array([1])
mask, score = self.predict_with_points(points, labels)
elif prompt_type == 'random_3_points':
# Generate 3 random points within the bounding box
box = prompt['box']
rng = np.random.RandomState(42)
pts_x = rng.uniform(box[0], box[2], 3)
pts_y = rng.uniform(box[1], box[3], 3)
points = np.stack([pts_x, pts_y], axis=1)
labels = np.ones(3, dtype=int)
mask, score = self.predict_with_points(points, labels)
elif prompt_type == 'box_and_center':
mask, score = self.predict_with_box_and_points(
prompt['box'],
prompt['center_point'].reshape(1, 2),
np.array([1])
)
else:
raise ValueError(f"Unknown prompt_type: {prompt_type}")
results.append({
'mask': mask,
'score': score,
'class_id': prompt['class_id'],
'prompt_type': prompt_type,
})
return results
def predict_iterative_box(self,
box: np.ndarray,
iterations: int = 3) -> Tuple[np.ndarray, float]:
"""
Iterative mask refinement using SAM2's logit feedback loop.
Each iteration feeds the previous mask logits back into SAM2
as a mask prior, progressively refining boundaries.
Args:
box: (4,) array [x1, y1, x2, y2]
iterations: Number of refinement iterations (default 3)
Returns:
mask: (H, W) binary mask (refined)
score: Final predicted IoU score
"""
# First pass: box prompt only, get initial logits
masks, scores, logits = self.predictor.predict(
point_coords=None,
point_labels=None,
box=box[None, :],
multimask_output=True, # Get 3 candidates on first pass
)
best_idx = scores.argmax()
best_mask = masks[best_idx]
best_score = float(scores[best_idx])
best_logits = logits[best_idx:best_idx+1, :, :] # (1, H, W)
# Iterative refinement passes
for i in range(iterations - 1):
masks, scores, logits = self.predictor.predict(
point_coords=None,
point_labels=None,
box=box[None, :],
mask_input=best_logits,
multimask_output=False, # Single mask on refinement passes
)
best_idx = scores.argmax()
candidate_mask = masks[best_idx]
candidate_score = float(scores[best_idx])
# Accept refinement only if score improves or stays close
if candidate_score >= best_score - 0.02:
best_mask = candidate_mask
best_score = candidate_score
best_logits = logits[best_idx:best_idx+1, :, :]
return best_mask.astype(np.uint8), best_score
def predict_multiscale_box(self,
image: np.ndarray,
box: np.ndarray,
scales: List[int] = None,
iterations: int = 3) -> Tuple[np.ndarray, float]:
"""
Multi-scale inference with majority-vote fusion.
Runs SAM2 at multiple resolutions and fuses the predictions
using soft averaging, capturing both fine detail and global context.
Args:
image: (H, W, 3) RGB image at original resolution
box: (4,) array [x1, y1, x2, y2] at original resolution
scales: List of target sizes (default [1024, 1536, 2048])
iterations: Iterative refinement passes per scale
Returns:
mask: (H, W) binary mask at original resolution
score: Average predicted IoU score across scales
"""
import cv2
if scales is None:
scales = [1024, 1536, 2048]
orig_h, orig_w = image.shape[:2]
accumulated = np.zeros((orig_h, orig_w), dtype=np.float32)
total_score = 0.0
n_valid = 0
for target_size in scales:
# Scale image
scale_x = target_size / orig_w
scale_y = target_size / orig_h
resized = cv2.resize(image, (target_size, target_size))
# Scale box
scaled_box = box.copy().astype(np.float64)
scaled_box[0] *= scale_x
scaled_box[1] *= scale_y
scaled_box[2] *= scale_x
scaled_box[3] *= scale_y
scaled_box = scaled_box.astype(np.int32)
# Set image and predict with iterative refinement
self.set_image(resized)
mask, score = self.predict_iterative_box(scaled_box, iterations=iterations)
# Resize mask back to original resolution
mask_orig = cv2.resize(mask, (orig_w, orig_h), interpolation=cv2.INTER_LINEAR)
# Accumulate as soft votes (weighted by score)
accumulated += mask_orig.astype(np.float32) * score
total_score += score
n_valid += 1
# Majority vote: threshold at half the total weight
if n_valid > 0:
threshold = total_score / (2.0 * n_valid) * n_valid
final_mask = (accumulated >= threshold).astype(np.uint8)
avg_score = total_score / n_valid
else:
final_mask = np.zeros((orig_h, orig_w), dtype=np.uint8)
avg_score = 0.0
return final_mask, avg_score
def combine_masks(self, results: List[Dict], shape: Tuple[int, int]) -> np.ndarray:
"""
Combine per-object masks into a single binary mask.
Args:
results: List from predict_all_objects
shape: (H, W) output shape
Returns:
Combined binary mask (H, W)
"""
combined = np.zeros(shape, dtype=np.uint8)
for r in results:
mask = r['mask']
if mask.shape != shape:
import cv2
mask = cv2.resize(mask, (shape[1], shape[0]),
interpolation=cv2.INTER_NEAREST)
combined = np.maximum(combined, mask)
return combined
class SAM2FineTuner:
"""
Fine-tuning wrapper for SAM2.
Supports:
- Decoder-only fine-tuning (freeze encoder)
- LoRA adapter fine-tuning
- Full fine-tuning
"""
def __init__(self,
variant: str = 'sam2_hiera_large',
checkpoint_path: Optional[str] = None,
strategy: str = 'decoder_only',
device: str = 'cuda'):
"""
Args:
variant: SAM2 model variant
checkpoint_path: Path to checkpoint
strategy: 'decoder_only', 'lora', 'full'
device: Device
"""
if not SAM2_AVAILABLE:
raise RuntimeError("sam2 package not installed")
self.variant = variant
self.device = device
self.strategy = strategy
# Build model
if checkpoint_path is None:
wrapper = SAM2Wrapper(variant, device=device)
self.model = wrapper.model
else:
config = SAM2_CONFIGS[variant]
self.model = build_sam2(config, checkpoint_path, device=device)
# Apply freezing strategy
self._setup_training_strategy()
def _setup_training_strategy(self):
"""Configure which parameters to train."""
if self.strategy == 'decoder_only':
# Freeze everything except mask decoder
for name, param in self.model.named_parameters():
if 'mask_decoder' in name or 'output_upscaling' in name:
param.requires_grad = True
else:
param.requires_grad = False
elif self.strategy == 'full':
for param in self.model.parameters():
param.requires_grad = True
elif self.strategy == 'lora':
# Freeze all, then add LoRA adapters
for param in self.model.parameters():
param.requires_grad = False
# LoRA would be added here — requires peft library
print("WARNING: LoRA not yet implemented. Using decoder-only strategy.")
self._setup_decoder_only()
# Print parameter stats
total = sum(p.numel() for p in self.model.parameters())
trainable = sum(p.numel() for p in self.model.parameters() if p.requires_grad)
print(f"Fine-tuning strategy: {self.strategy}")
print(f" Total parameters: {total:,}")
print(f" Trainable parameters: {trainable:,} ({100*trainable/total:.1f}%)")
def get_trainable_parameters(self):
"""Return trainable parameters for optimizer."""
return [p for p in self.model.parameters() if p.requires_grad]
def get_predictor(self) -> 'SAM2ImagePredictor':
"""Get an image predictor from the current model state."""
return SAM2ImagePredictor(self.model)
if __name__ == '__main__':
if SAM2_AVAILABLE:
print("SAM2 is available!")
# Test would require checkpoint download
else:
print("SAM2 not installed. Run:")
print(" pip install git+https://github.com/facebookresearch/sam2.git")