Spaces:
Sleeping
Sleeping
Download src/models/sam2_wrapper.py from amitaddi/btxrd-workstation: direct link, hf CLI and curl.
- Browser
- Download file 17.3 kB
-
https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/models/sam2_wrapper.py
- Command line
-
hf download hf://spaces/amitaddi/btxrd-workstation/src/models/sam2_wrapper.py
-
curl -L -o sam2_wrapper.py https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/models/sam2_wrapper.py
17.3 kB
| """ | |
| SAM2 Wrapper for Bone Tumor Segmentation. | |
| Provides a unified interface for: | |
| - Zero-shot inference with box/point/mask prompts | |
| - Few-shot evaluation | |
| - Fine-tuning support | |
| Supports all SAM2 model variants: | |
| sam2_hiera_tiny, sam2_hiera_small, sam2_hiera_base_plus, sam2_hiera_large | |
| """ | |
| import os | |
| import numpy as np | |
| import torch | |
| from typing import Optional, List, Dict, Tuple, Union | |
| # SAM2 imports — these come from Meta's official sam2 package | |
| # Install: pip install git+https://github.com/facebookresearch/sam2.git | |
| try: | |
| from sam2.build_sam import build_sam2 | |
| from sam2.sam2_image_predictor import SAM2ImagePredictor | |
| SAM2_AVAILABLE = True | |
| except ImportError: | |
| SAM2_AVAILABLE = False | |
| print("WARNING: sam2 not installed. Install with: pip install git+https://github.com/facebookresearch/sam2.git") | |
| # Model config paths (relative to sam2 package) | |
| SAM2_CONFIGS = { | |
| 'sam2_hiera_tiny': 'configs/sam2/sam2_hiera_t.yaml', | |
| 'sam2_hiera_small': 'configs/sam2/sam2_hiera_s.yaml', | |
| 'sam2_hiera_base_plus': 'configs/sam2/sam2_hiera_b+.yaml', | |
| 'sam2_hiera_large': 'configs/sam2/sam2_hiera_l.yaml', | |
| } | |
| # Checkpoint URLs for auto-download | |
| SAM2_CHECKPOINTS = { | |
| 'sam2_hiera_tiny': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_tiny.pt', | |
| 'sam2_hiera_small': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_small.pt', | |
| 'sam2_hiera_base_plus': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_base_plus.pt', | |
| 'sam2_hiera_large': 'https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt', | |
| } | |
| class SAM2Wrapper: | |
| """ | |
| Wrapper around SAM2 for bone tumor segmentation. | |
| Provides clean interface for box-prompted, point-prompted, | |
| and combined prompt inference. | |
| """ | |
| def __init__(self, | |
| variant: str = 'sam2_hiera_large', | |
| checkpoint_path: Optional[str] = None, | |
| device: str = 'cuda'): | |
| """ | |
| Args: | |
| variant: SAM2 model variant name | |
| checkpoint_path: Path to checkpoint (auto-downloads if None) | |
| device: 'cuda' or 'cpu' | |
| """ | |
| if not SAM2_AVAILABLE: | |
| raise RuntimeError("sam2 package not installed") | |
| self.variant = variant | |
| self.device = device | |
| # Resolve checkpoint | |
| if checkpoint_path is None: | |
| checkpoint_path = self._get_or_download_checkpoint(variant) | |
| # Build model | |
| config = SAM2_CONFIGS[variant] | |
| self.model = build_sam2(config, checkpoint_path, device=device) | |
| self.predictor = SAM2ImagePredictor(self.model) | |
| print(f"SAM2 ({variant}) loaded on {device}") | |
| def _get_or_download_checkpoint(self, variant: str) -> str: | |
| """Download checkpoint if not present.""" | |
| cache_dir = os.path.join(os.path.expanduser('~'), '.cache', 'sam2') | |
| os.makedirs(cache_dir, exist_ok=True) | |
| filename = f"{variant}.pt" | |
| filepath = os.path.join(cache_dir, filename) | |
| if not os.path.exists(filepath): | |
| url = SAM2_CHECKPOINTS[variant] | |
| print(f"Downloading {variant} checkpoint...") | |
| print(f" URL: {url}") | |
| print(f" Saving to: {filepath}") | |
| import urllib.request | |
| urllib.request.urlretrieve(url, filepath) | |
| print(" Download complete!") | |
| return filepath | |
| def set_image(self, image: np.ndarray): | |
| """ | |
| Set the image for prediction. Must be called before predict_*. | |
| Args: | |
| image: (H, W, 3) RGB numpy array | |
| """ | |
| self.predictor.set_image(image) | |
| self._current_image = image | |
| def predict_with_box(self, | |
| box: np.ndarray, | |
| multimask_output: bool = False) -> Tuple[np.ndarray, float]: | |
| """ | |
| Predict segmentation mask given a bounding box prompt. | |
| Args: | |
| box: (4,) array [x1, y1, x2, y2] | |
| multimask_output: If True, returns 3 masks with scores | |
| Returns: | |
| mask: (H, W) binary mask | |
| score: Predicted IoU score from SAM2 | |
| """ | |
| masks, scores, logits = self.predictor.predict( | |
| point_coords=None, | |
| point_labels=None, | |
| box=box[None, :], # (1, 4) | |
| multimask_output=multimask_output, | |
| ) | |
| # Take the best mask | |
| best_idx = scores.argmax() | |
| mask = masks[best_idx] # (H, W) | |
| score = float(scores[best_idx]) | |
| return mask.astype(np.uint8), score | |
| def predict_with_points(self, | |
| points: np.ndarray, | |
| labels: np.ndarray, | |
| multimask_output: bool = True) -> Tuple[np.ndarray, float]: | |
| """ | |
| Predict with point prompts. | |
| Args: | |
| points: (N, 2) array of (x, y) coordinates | |
| labels: (N,) array of 1 (foreground) or 0 (background) | |
| multimask_output: Return 3 mask candidates | |
| Returns: | |
| mask: (H, W) binary mask | |
| score: Predicted IoU score | |
| """ | |
| masks, scores, logits = self.predictor.predict( | |
| point_coords=points, | |
| point_labels=labels, | |
| box=None, | |
| multimask_output=multimask_output, | |
| ) | |
| best_idx = scores.argmax() | |
| return masks[best_idx].astype(np.uint8), float(scores[best_idx]) | |
| def predict_with_box_and_points(self, | |
| box: np.ndarray, | |
| points: np.ndarray, | |
| labels: np.ndarray) -> Tuple[np.ndarray, float]: | |
| """ | |
| Predict with combined box + point prompts. | |
| Args: | |
| box: (4,) array [x1, y1, x2, y2] | |
| points: (N, 2) point coordinates | |
| labels: (N,) point labels (1=fg, 0=bg) | |
| Returns: | |
| mask: (H, W) binary mask | |
| score: Predicted IoU | |
| """ | |
| masks, scores, logits = self.predictor.predict( | |
| point_coords=points, | |
| point_labels=labels, | |
| box=box[None, :], | |
| multimask_output=False, | |
| ) | |
| best_idx = scores.argmax() | |
| return masks[best_idx].astype(np.uint8), float(scores[best_idx]) | |
| def predict_all_objects(self, | |
| image: np.ndarray, | |
| prompts: List[Dict], | |
| prompt_type: str = 'box') -> List[Dict]: | |
| """ | |
| Predict masks for all objects in an image. | |
| Args: | |
| image: (H, W, 3) RGB image | |
| prompts: List of prompt dicts with 'box', 'center_point', 'class_id' | |
| prompt_type: 'box', 'center_point', 'random_3_points', 'box_and_center' | |
| Returns: | |
| List of result dicts with 'mask', 'score', 'class_id' | |
| """ | |
| self.set_image(image) | |
| results = [] | |
| for prompt in prompts: | |
| if prompt_type == 'box': | |
| mask, score = self.predict_with_box(prompt['box']) | |
| elif prompt_type == 'center_point': | |
| points = prompt['center_point'].reshape(1, 2) | |
| labels = np.array([1]) | |
| mask, score = self.predict_with_points(points, labels) | |
| elif prompt_type == 'random_3_points': | |
| # Generate 3 random points within the bounding box | |
| box = prompt['box'] | |
| rng = np.random.RandomState(42) | |
| pts_x = rng.uniform(box[0], box[2], 3) | |
| pts_y = rng.uniform(box[1], box[3], 3) | |
| points = np.stack([pts_x, pts_y], axis=1) | |
| labels = np.ones(3, dtype=int) | |
| mask, score = self.predict_with_points(points, labels) | |
| elif prompt_type == 'box_and_center': | |
| mask, score = self.predict_with_box_and_points( | |
| prompt['box'], | |
| prompt['center_point'].reshape(1, 2), | |
| np.array([1]) | |
| ) | |
| else: | |
| raise ValueError(f"Unknown prompt_type: {prompt_type}") | |
| results.append({ | |
| 'mask': mask, | |
| 'score': score, | |
| 'class_id': prompt['class_id'], | |
| 'prompt_type': prompt_type, | |
| }) | |
| return results | |
| def predict_iterative_box(self, | |
| box: np.ndarray, | |
| iterations: int = 3) -> Tuple[np.ndarray, float]: | |
| """ | |
| Iterative mask refinement using SAM2's logit feedback loop. | |
| Each iteration feeds the previous mask logits back into SAM2 | |
| as a mask prior, progressively refining boundaries. | |
| Args: | |
| box: (4,) array [x1, y1, x2, y2] | |
| iterations: Number of refinement iterations (default 3) | |
| Returns: | |
| mask: (H, W) binary mask (refined) | |
| score: Final predicted IoU score | |
| """ | |
| # First pass: box prompt only, get initial logits | |
| masks, scores, logits = self.predictor.predict( | |
| point_coords=None, | |
| point_labels=None, | |
| box=box[None, :], | |
| multimask_output=True, # Get 3 candidates on first pass | |
| ) | |
| best_idx = scores.argmax() | |
| best_mask = masks[best_idx] | |
| best_score = float(scores[best_idx]) | |
| best_logits = logits[best_idx:best_idx+1, :, :] # (1, H, W) | |
| # Iterative refinement passes | |
| for i in range(iterations - 1): | |
| masks, scores, logits = self.predictor.predict( | |
| point_coords=None, | |
| point_labels=None, | |
| box=box[None, :], | |
| mask_input=best_logits, | |
| multimask_output=False, # Single mask on refinement passes | |
| ) | |
| best_idx = scores.argmax() | |
| candidate_mask = masks[best_idx] | |
| candidate_score = float(scores[best_idx]) | |
| # Accept refinement only if score improves or stays close | |
| if candidate_score >= best_score - 0.02: | |
| best_mask = candidate_mask | |
| best_score = candidate_score | |
| best_logits = logits[best_idx:best_idx+1, :, :] | |
| return best_mask.astype(np.uint8), best_score | |
| def predict_multiscale_box(self, | |
| image: np.ndarray, | |
| box: np.ndarray, | |
| scales: List[int] = None, | |
| iterations: int = 3) -> Tuple[np.ndarray, float]: | |
| """ | |
| Multi-scale inference with majority-vote fusion. | |
| Runs SAM2 at multiple resolutions and fuses the predictions | |
| using soft averaging, capturing both fine detail and global context. | |
| Args: | |
| image: (H, W, 3) RGB image at original resolution | |
| box: (4,) array [x1, y1, x2, y2] at original resolution | |
| scales: List of target sizes (default [1024, 1536, 2048]) | |
| iterations: Iterative refinement passes per scale | |
| Returns: | |
| mask: (H, W) binary mask at original resolution | |
| score: Average predicted IoU score across scales | |
| """ | |
| import cv2 | |
| if scales is None: | |
| scales = [1024, 1536, 2048] | |
| orig_h, orig_w = image.shape[:2] | |
| accumulated = np.zeros((orig_h, orig_w), dtype=np.float32) | |
| total_score = 0.0 | |
| n_valid = 0 | |
| for target_size in scales: | |
| # Scale image | |
| scale_x = target_size / orig_w | |
| scale_y = target_size / orig_h | |
| resized = cv2.resize(image, (target_size, target_size)) | |
| # Scale box | |
| scaled_box = box.copy().astype(np.float64) | |
| scaled_box[0] *= scale_x | |
| scaled_box[1] *= scale_y | |
| scaled_box[2] *= scale_x | |
| scaled_box[3] *= scale_y | |
| scaled_box = scaled_box.astype(np.int32) | |
| # Set image and predict with iterative refinement | |
| self.set_image(resized) | |
| mask, score = self.predict_iterative_box(scaled_box, iterations=iterations) | |
| # Resize mask back to original resolution | |
| mask_orig = cv2.resize(mask, (orig_w, orig_h), interpolation=cv2.INTER_LINEAR) | |
| # Accumulate as soft votes (weighted by score) | |
| accumulated += mask_orig.astype(np.float32) * score | |
| total_score += score | |
| n_valid += 1 | |
| # Majority vote: threshold at half the total weight | |
| if n_valid > 0: | |
| threshold = total_score / (2.0 * n_valid) * n_valid | |
| final_mask = (accumulated >= threshold).astype(np.uint8) | |
| avg_score = total_score / n_valid | |
| else: | |
| final_mask = np.zeros((orig_h, orig_w), dtype=np.uint8) | |
| avg_score = 0.0 | |
| return final_mask, avg_score | |
| def combine_masks(self, results: List[Dict], shape: Tuple[int, int]) -> np.ndarray: | |
| """ | |
| Combine per-object masks into a single binary mask. | |
| Args: | |
| results: List from predict_all_objects | |
| shape: (H, W) output shape | |
| Returns: | |
| Combined binary mask (H, W) | |
| """ | |
| combined = np.zeros(shape, dtype=np.uint8) | |
| for r in results: | |
| mask = r['mask'] | |
| if mask.shape != shape: | |
| import cv2 | |
| mask = cv2.resize(mask, (shape[1], shape[0]), | |
| interpolation=cv2.INTER_NEAREST) | |
| combined = np.maximum(combined, mask) | |
| return combined | |
| class SAM2FineTuner: | |
| """ | |
| Fine-tuning wrapper for SAM2. | |
| Supports: | |
| - Decoder-only fine-tuning (freeze encoder) | |
| - LoRA adapter fine-tuning | |
| - Full fine-tuning | |
| """ | |
| def __init__(self, | |
| variant: str = 'sam2_hiera_large', | |
| checkpoint_path: Optional[str] = None, | |
| strategy: str = 'decoder_only', | |
| device: str = 'cuda'): | |
| """ | |
| Args: | |
| variant: SAM2 model variant | |
| checkpoint_path: Path to checkpoint | |
| strategy: 'decoder_only', 'lora', 'full' | |
| device: Device | |
| """ | |
| if not SAM2_AVAILABLE: | |
| raise RuntimeError("sam2 package not installed") | |
| self.variant = variant | |
| self.device = device | |
| self.strategy = strategy | |
| # Build model | |
| if checkpoint_path is None: | |
| wrapper = SAM2Wrapper(variant, device=device) | |
| self.model = wrapper.model | |
| else: | |
| config = SAM2_CONFIGS[variant] | |
| self.model = build_sam2(config, checkpoint_path, device=device) | |
| # Apply freezing strategy | |
| self._setup_training_strategy() | |
| def _setup_training_strategy(self): | |
| """Configure which parameters to train.""" | |
| if self.strategy == 'decoder_only': | |
| # Freeze everything except mask decoder | |
| for name, param in self.model.named_parameters(): | |
| if 'mask_decoder' in name or 'output_upscaling' in name: | |
| param.requires_grad = True | |
| else: | |
| param.requires_grad = False | |
| elif self.strategy == 'full': | |
| for param in self.model.parameters(): | |
| param.requires_grad = True | |
| elif self.strategy == 'lora': | |
| # Freeze all, then add LoRA adapters | |
| for param in self.model.parameters(): | |
| param.requires_grad = False | |
| # LoRA would be added here — requires peft library | |
| print("WARNING: LoRA not yet implemented. Using decoder-only strategy.") | |
| self._setup_decoder_only() | |
| # Print parameter stats | |
| total = sum(p.numel() for p in self.model.parameters()) | |
| trainable = sum(p.numel() for p in self.model.parameters() if p.requires_grad) | |
| print(f"Fine-tuning strategy: {self.strategy}") | |
| print(f" Total parameters: {total:,}") | |
| print(f" Trainable parameters: {trainable:,} ({100*trainable/total:.1f}%)") | |
| def get_trainable_parameters(self): | |
| """Return trainable parameters for optimizer.""" | |
| return [p for p in self.model.parameters() if p.requires_grad] | |
| def get_predictor(self) -> 'SAM2ImagePredictor': | |
| """Get an image predictor from the current model state.""" | |
| return SAM2ImagePredictor(self.model) | |
| if __name__ == '__main__': | |
| if SAM2_AVAILABLE: | |
| print("SAM2 is available!") | |
| # Test would require checkpoint download | |
| else: | |
| print("SAM2 not installed. Run:") | |
| print(" pip install git+https://github.com/facebookresearch/sam2.git") | |