""" SAM2 Wrapper for Bone Tumor Segmentation. Provides a unified interface for: - Zero-shot inference with box/point/mask prompts - Few-shot evaluation - Fine-tuning support Supports all SAM2 model variants: sam2_hiera_tiny, sam2_hiera_small, sam2_hiera_base_plus, sam2_hiera_large """ import os import numpy as np import torch from typing import Optional, List, Dict, Tuple, Union # SAM2 imports — these come from Meta's official sam2 package # Install: pip install git+https://github.com/facebookresearch/sam2.git try: from sam2.build_sam import build_sam2 from sam2.sam2_image_predictor import SAM2ImagePredictor SAM2_AVAILABLE = True except ImportError: SAM2_AVAILABLE = False print("WARNING: sam2 not installed. Install with: pip install git+https://github.com/facebookresearch/sam2.git") # Model config paths (relative to sam2 package) SAM2_CONFIGS = { 'sam2_hiera_tiny': 'configs/sam2/sam2_hiera_t.yaml', 'sam2_hiera_small': 'configs/sam2/sam2_hiera_s.yaml', 'sam2_hiera_base_plus': 'configs/sam2/sam2_hiera_b+.yaml', 'sam2_hiera_large': 'configs/sam2/sam2_hiera_l.yaml', } # Checkpoint URLs for auto-download SAM2_CHECKPOINTS = { 'sam2_hiera_tiny': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_tiny.pt', 'sam2_hiera_small': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_small.pt', 'sam2_hiera_base_plus': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_base_plus.pt', 'sam2_hiera_large': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt', } class SAM2Wrapper: """ Wrapper around SAM2 for bone tumor segmentation. Provides clean interface for box-prompted, point-prompted, and combined prompt inference. """ def __init__(self, variant: str = 'sam2_hiera_large', checkpoint_path: Optional[str] = None, device: str = 'cuda'): """ Args: variant: SAM2 model variant name checkpoint_path: Path to checkpoint (auto-downloads if None) device: 'cuda' or 'cpu' """ if not SAM2_AVAILABLE: raise RuntimeError("sam2 package not installed") self.variant = variant self.device = device # Resolve checkpoint if checkpoint_path is None: checkpoint_path = self._get_or_download_checkpoint(variant) # Build model config = SAM2_CONFIGS[variant] self.model = build_sam2(config, checkpoint_path, device=device) self.predictor = SAM2ImagePredictor(self.model) print(f"SAM2 ({variant}) loaded on {device}") def _get_or_download_checkpoint(self, variant: str) -> str: """Download checkpoint if not present.""" cache_dir = os.path.join(os.path.expanduser('~'), '.cache', 'sam2') os.makedirs(cache_dir, exist_ok=True) filename = f"{variant}.pt" filepath = os.path.join(cache_dir, filename) if not os.path.exists(filepath): url = SAM2_CHECKPOINTS[variant] print(f"Downloading {variant} checkpoint...") print(f" URL: {url}") print(f" Saving to: {filepath}") import urllib.request urllib.request.urlretrieve(url, filepath) print(" Download complete!") return filepath def set_image(self, image: np.ndarray): """ Set the image for prediction. Must be called before predict_*. Args: image: (H, W, 3) RGB numpy array """ self.predictor.set_image(image) self._current_image = image def predict_with_box(self, box: np.ndarray, multimask_output: bool = False) -> Tuple[np.ndarray, float]: """ Predict segmentation mask given a bounding box prompt. Args: box: (4,) array [x1, y1, x2, y2] multimask_output: If True, returns 3 masks with scores Returns: mask: (H, W) binary mask score: Predicted IoU score from SAM2 """ masks, scores, logits = self.predictor.predict( point_coords=None, point_labels=None, box=box[None, :], # (1, 4) multimask_output=multimask_output, ) # Take the best mask best_idx = scores.argmax() mask = masks[best_idx] # (H, W) score = float(scores[best_idx]) return mask.astype(np.uint8), score def predict_with_points(self, points: np.ndarray, labels: np.ndarray, multimask_output: bool = True) -> Tuple[np.ndarray, float]: """ Predict with point prompts. Args: points: (N, 2) array of (x, y) coordinates labels: (N,) array of 1 (foreground) or 0 (background) multimask_output: Return 3 mask candidates Returns: mask: (H, W) binary mask score: Predicted IoU score """ masks, scores, logits = self.predictor.predict( point_coords=points, point_labels=labels, box=None, multimask_output=multimask_output, ) best_idx = scores.argmax() return masks[best_idx].astype(np.uint8), float(scores[best_idx]) def predict_with_box_and_points(self, box: np.ndarray, points: np.ndarray, labels: np.ndarray) -> Tuple[np.ndarray, float]: """ Predict with combined box + point prompts. Args: box: (4,) array [x1, y1, x2, y2] points: (N, 2) point coordinates labels: (N,) point labels (1=fg, 0=bg) Returns: mask: (H, W) binary mask score: Predicted IoU """ masks, scores, logits = self.predictor.predict( point_coords=points, point_labels=labels, box=box[None, :], multimask_output=False, ) best_idx = scores.argmax() return masks[best_idx].astype(np.uint8), float(scores[best_idx]) def predict_all_objects(self, image: np.ndarray, prompts: List[Dict], prompt_type: str = 'box') -> List[Dict]: """ Predict masks for all objects in an image. Args: image: (H, W, 3) RGB image prompts: List of prompt dicts with 'box', 'center_point', 'class_id' prompt_type: 'box', 'center_point', 'random_3_points', 'box_and_center' Returns: List of result dicts with 'mask', 'score', 'class_id' """ self.set_image(image) results = [] for prompt in prompts: if prompt_type == 'box': mask, score = self.predict_with_box(prompt['box']) elif prompt_type == 'center_point': points = prompt['center_point'].reshape(1, 2) labels = np.array([1]) mask, score = self.predict_with_points(points, labels) elif prompt_type == 'random_3_points': # Generate 3 random points within the bounding box box = prompt['box'] rng = np.random.RandomState(42) pts_x = rng.uniform(box[0], box[2], 3) pts_y = rng.uniform(box[1], box[3], 3) points = np.stack([pts_x, pts_y], axis=1) labels = np.ones(3, dtype=int) mask, score = self.predict_with_points(points, labels) elif prompt_type == 'box_and_center': mask, score = self.predict_with_box_and_points( prompt['box'], prompt['center_point'].reshape(1, 2), np.array([1]) ) else: raise ValueError(f"Unknown prompt_type: {prompt_type}") results.append({ 'mask': mask, 'score': score, 'class_id': prompt['class_id'], 'prompt_type': prompt_type, }) return results def predict_iterative_box(self, box: np.ndarray, iterations: int = 3) -> Tuple[np.ndarray, float]: """ Iterative mask refinement using SAM2's logit feedback loop. Each iteration feeds the previous mask logits back into SAM2 as a mask prior, progressively refining boundaries. Args: box: (4,) array [x1, y1, x2, y2] iterations: Number of refinement iterations (default 3) Returns: mask: (H, W) binary mask (refined) score: Final predicted IoU score """ # First pass: box prompt only, get initial logits masks, scores, logits = self.predictor.predict( point_coords=None, point_labels=None, box=box[None, :], multimask_output=True, # Get 3 candidates on first pass ) best_idx = scores.argmax() best_mask = masks[best_idx] best_score = float(scores[best_idx]) best_logits = logits[best_idx:best_idx+1, :, :] # (1, H, W) # Iterative refinement passes for i in range(iterations - 1): masks, scores, logits = self.predictor.predict( point_coords=None, point_labels=None, box=box[None, :], mask_input=best_logits, multimask_output=False, # Single mask on refinement passes ) best_idx = scores.argmax() candidate_mask = masks[best_idx] candidate_score = float(scores[best_idx]) # Accept refinement only if score improves or stays close if candidate_score >= best_score - 0.02: best_mask = candidate_mask best_score = candidate_score best_logits = logits[best_idx:best_idx+1, :, :] return best_mask.astype(np.uint8), best_score def predict_multiscale_box(self, image: np.ndarray, box: np.ndarray, scales: List[int] = None, iterations: int = 3) -> Tuple[np.ndarray, float]: """ Multi-scale inference with majority-vote fusion. Runs SAM2 at multiple resolutions and fuses the predictions using soft averaging, capturing both fine detail and global context. Args: image: (H, W, 3) RGB image at original resolution box: (4,) array [x1, y1, x2, y2] at original resolution scales: List of target sizes (default [1024, 1536, 2048]) iterations: Iterative refinement passes per scale Returns: mask: (H, W) binary mask at original resolution score: Average predicted IoU score across scales """ import cv2 if scales is None: scales = [1024, 1536, 2048] orig_h, orig_w = image.shape[:2] accumulated = np.zeros((orig_h, orig_w), dtype=np.float32) total_score = 0.0 n_valid = 0 for target_size in scales: # Scale image scale_x = target_size / orig_w scale_y = target_size / orig_h resized = cv2.resize(image, (target_size, target_size)) # Scale box scaled_box = box.copy().astype(np.float64) scaled_box[0] *= scale_x scaled_box[1] *= scale_y scaled_box[2] *= scale_x scaled_box[3] *= scale_y scaled_box = scaled_box.astype(np.int32) # Set image and predict with iterative refinement self.set_image(resized) mask, score = self.predict_iterative_box(scaled_box, iterations=iterations) # Resize mask back to original resolution mask_orig = cv2.resize(mask, (orig_w, orig_h), interpolation=cv2.INTER_LINEAR) # Accumulate as soft votes (weighted by score) accumulated += mask_orig.astype(np.float32) * score total_score += score n_valid += 1 # Majority vote: threshold at half the total weight if n_valid > 0: threshold = total_score / (2.0 * n_valid) * n_valid final_mask = (accumulated >= threshold).astype(np.uint8) avg_score = total_score / n_valid else: final_mask = np.zeros((orig_h, orig_w), dtype=np.uint8) avg_score = 0.0 return final_mask, avg_score def combine_masks(self, results: List[Dict], shape: Tuple[int, int]) -> np.ndarray: """ Combine per-object masks into a single binary mask. Args: results: List from predict_all_objects shape: (H, W) output shape Returns: Combined binary mask (H, W) """ combined = np.zeros(shape, dtype=np.uint8) for r in results: mask = r['mask'] if mask.shape != shape: import cv2 mask = cv2.resize(mask, (shape[1], shape[0]), interpolation=cv2.INTER_NEAREST) combined = np.maximum(combined, mask) return combined class SAM2FineTuner: """ Fine-tuning wrapper for SAM2. Supports: - Decoder-only fine-tuning (freeze encoder) - LoRA adapter fine-tuning - Full fine-tuning """ def __init__(self, variant: str = 'sam2_hiera_large', checkpoint_path: Optional[str] = None, strategy: str = 'decoder_only', device: str = 'cuda'): """ Args: variant: SAM2 model variant checkpoint_path: Path to checkpoint strategy: 'decoder_only', 'lora', 'full' device: Device """ if not SAM2_AVAILABLE: raise RuntimeError("sam2 package not installed") self.variant = variant self.device = device self.strategy = strategy # Build model if checkpoint_path is None: wrapper = SAM2Wrapper(variant, device=device) self.model = wrapper.model else: config = SAM2_CONFIGS[variant] self.model = build_sam2(config, checkpoint_path, device=device) # Apply freezing strategy self._setup_training_strategy() def _setup_training_strategy(self): """Configure which parameters to train.""" if self.strategy == 'decoder_only': # Freeze everything except mask decoder for name, param in self.model.named_parameters(): if 'mask_decoder' in name or 'output_upscaling' in name: param.requires_grad = True else: param.requires_grad = False elif self.strategy == 'full': for param in self.model.parameters(): param.requires_grad = True elif self.strategy == 'lora': # Freeze all, then add LoRA adapters for param in self.model.parameters(): param.requires_grad = False # LoRA would be added here — requires peft library print("WARNING: LoRA not yet implemented. Using decoder-only strategy.") self._setup_decoder_only() # Print parameter stats total = sum(p.numel() for p in self.model.parameters()) trainable = sum(p.numel() for p in self.model.parameters() if p.requires_grad) print(f"Fine-tuning strategy: {self.strategy}") print(f" Total parameters: {total:,}") print(f" Trainable parameters: {trainable:,} ({100*trainable/total:.1f}%)") def get_trainable_parameters(self): """Return trainable parameters for optimizer.""" return [p for p in self.model.parameters() if p.requires_grad] def get_predictor(self) -> 'SAM2ImagePredictor': """Get an image predictor from the current model state.""" return SAM2ImagePredictor(self.model) if __name__ == '__main__': if SAM2_AVAILABLE: print("SAM2 is available!") # Test would require checkpoint download else: print("SAM2 not installed. Run:") print(" pip install git+https://github.com/facebookresearch/sam2.git")