Spaces:
Sleeping
Sleeping
Download src/data/dataset.py from amitaddi/btxrd-workstation: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/data/dataset.py
- Command line
-
hf download hf://spaces/amitaddi/btxrd-workstation/src/data/dataset.py
-
curl -L -o dataset.py https://huggingface.co/spaces/amitaddi/btxrd-workstation/resolve/main/src/data/dataset.py
13.9 kB
| """ | |
| Dataset classes for Bone Tumor Segmentation. | |
| Provides: | |
| - BTXRDSegmentationDataset: For UNet training (returns image + mask pairs) | |
| - BTXRDSAMDataset: For SAM/MedSAM (returns image + bounding box/point prompts) | |
| """ | |
| import os | |
| import glob | |
| import numpy as np | |
| import cv2 | |
| from PIL import Image | |
| from pathlib import Path | |
| from typing import Optional, List, Dict, Tuple, Callable | |
| import torch | |
| from torch.utils.data import Dataset | |
| import albumentations as A | |
| from albumentations.pytorch import ToTensorV2 | |
| from src.data.yolo_to_masks import parse_yolo_label, yolo_to_xyxy, get_bboxes_for_sam | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Augmentation pipelines | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def get_unet_train_transforms(image_size: int = 512) -> A.Compose: | |
| """Training augmentations for UNet (Enhanced for Medical Segmentation).""" | |
| return A.Compose([ | |
| A.Resize(image_size, image_size), | |
| A.HorizontalFlip(p=0.5), | |
| A.VerticalFlip(p=0.5), | |
| A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=30, border_mode=cv2.BORDER_CONSTANT, p=0.5), | |
| A.ElasticTransform(alpha=1, sigma=50, alpha_affine=50, p=0.3), | |
| A.GridDistortion(num_steps=5, distort_limit=0.3, p=0.3), | |
| A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5), | |
| A.GaussNoise(var_limit=(10.0, 50.0), p=0.3), | |
| A.CLAHE(clip_limit=4.0, p=0.3), | |
| A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), | |
| ToTensorV2(), | |
| ]) | |
| def get_unet_val_transforms(image_size: int = 512) -> A.Compose: | |
| """Validation transforms for UNet (no augmentation).""" | |
| return A.Compose([ | |
| A.Resize(image_size, image_size), | |
| A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), | |
| ToTensorV2(), | |
| ]) | |
| def get_sam_transforms(image_size: int = 1024) -> A.Compose: | |
| """Transforms for SAM input (minimal β SAM has its own preprocessor).""" | |
| return A.Compose([ | |
| A.Resize(image_size, image_size), | |
| ]) | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # UNet Segmentation Dataset | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class BTXRDSegmentationDataset(Dataset): | |
| """ | |
| Dataset for UNet-based segmentation training. | |
| Loads X-ray images and corresponding bounding-box masks. | |
| Each sample is (image_tensor, mask_tensor, metadata_dict). | |
| """ | |
| def __init__(self, | |
| images_dir: str, | |
| labels_dir: str, | |
| masks_dir: Optional[str] = None, | |
| transform: Optional[A.Compose] = None, | |
| image_size: int = 512, | |
| binary: bool = True): | |
| """ | |
| Args: | |
| images_dir: Path to image directory | |
| labels_dir: Path to YOLO label directory | |
| masks_dir: Path to pre-generated masks (if None, creates on-the-fly) | |
| transform: Albumentations transform pipeline | |
| image_size: Target image size | |
| binary: Binary segmentation (True) or multi-class (False) | |
| """ | |
| self.images_dir = images_dir | |
| self.labels_dir = labels_dir | |
| self.masks_dir = masks_dir | |
| self.transform = transform | |
| self.image_size = image_size | |
| self.binary = binary | |
| # Collect valid image-label pairs | |
| self.samples = [] | |
| label_files = sorted(glob.glob(os.path.join(labels_dir, '*.txt'))) | |
| for label_path in label_files: | |
| stem = Path(label_path).stem | |
| # Find image | |
| img_path = None | |
| for ext in ['.jpeg', '.jpg', '.png']: | |
| candidate = os.path.join(images_dir, stem + ext) | |
| if os.path.exists(candidate): | |
| img_path = candidate | |
| break | |
| if img_path is not None: | |
| self.samples.append({ | |
| 'image_path': img_path, | |
| 'label_path': label_path, | |
| 'stem': stem, | |
| }) | |
| def __len__(self) -> int: | |
| return len(self.samples) | |
| def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor, Dict]: | |
| sample = self.samples[idx] | |
| # Load image | |
| image = cv2.imread(sample['image_path']) | |
| image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
| img_h, img_w = image.shape[:2] | |
| # Load or create mask | |
| if self.masks_dir and os.path.exists(os.path.join(self.masks_dir, sample['stem'] + '.png')): | |
| mask = cv2.imread(os.path.join(self.masks_dir, sample['stem'] + '.png'), | |
| cv2.IMREAD_GRAYSCALE) | |
| mask = (mask > 127).astype(np.uint8) | |
| else: | |
| # Create mask on-the-fly from YOLO labels | |
| from src.data.yolo_to_masks import create_bbox_mask | |
| annotations = parse_yolo_label(sample['label_path']) | |
| mask = create_bbox_mask(img_w, img_h, annotations, binary=self.binary) | |
| # Apply transforms | |
| if self.transform: | |
| transformed = self.transform(image=image, mask=mask) | |
| image = transformed['image'] | |
| mask = transformed['mask'] | |
| # Ensure mask is float tensor with channel dim | |
| if isinstance(mask, np.ndarray): | |
| mask = torch.from_numpy(mask) | |
| mask = mask.float().unsqueeze(0) # (1, H, W) | |
| metadata = { | |
| 'stem': sample['stem'], | |
| 'original_size': (img_h, img_w), | |
| } | |
| return image, mask, metadata | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # SAM / MedSAM Prompt Dataset | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class BTXRDSAMDataset(Dataset): | |
| """ | |
| Dataset for SAM2/MedSAM evaluation. | |
| Returns image as numpy array + list of prompt dicts (one per object). | |
| Each prompt dict has 'box', 'center_point', 'class_id'. | |
| Also returns bounding-box masks as ground truth for evaluation. | |
| """ | |
| def __init__(self, | |
| images_dir: str, | |
| labels_dir: str, | |
| image_size: int = 1024, | |
| expand_ratio: float = 0.0): | |
| """ | |
| Args: | |
| images_dir: Path to image directory | |
| labels_dir: Path to YOLO label directory | |
| image_size: SAM input size (typically 1024) | |
| expand_ratio: How much to expand bounding boxes | |
| """ | |
| self.images_dir = images_dir | |
| self.labels_dir = labels_dir | |
| self.image_size = image_size | |
| self.expand_ratio = expand_ratio | |
| # Collect valid samples | |
| self.samples = [] | |
| label_files = sorted(glob.glob(os.path.join(labels_dir, '*.txt'))) | |
| for label_path in label_files: | |
| stem = Path(label_path).stem | |
| img_path = None | |
| for ext in ['.jpeg', '.jpg', '.png']: | |
| candidate = os.path.join(images_dir, stem + ext) | |
| if os.path.exists(candidate): | |
| img_path = candidate | |
| break | |
| if img_path is not None: | |
| annotations = parse_yolo_label(label_path) | |
| if len(annotations) > 0: | |
| self.samples.append({ | |
| 'image_path': img_path, | |
| 'label_path': label_path, | |
| 'stem': stem, | |
| 'annotations': annotations, | |
| }) | |
| def __len__(self) -> int: | |
| return len(self.samples) | |
| def __getitem__(self, idx: int) -> Tuple[np.ndarray, List[Dict], np.ndarray, Dict]: | |
| """ | |
| Returns: | |
| image: (H, W, 3) RGB numpy array (resized to image_size) | |
| prompts: List of prompt dicts for SAM | |
| gt_mask: (H, W) binary mask from bounding boxes | |
| metadata: Dict with original size, stem, class info | |
| """ | |
| sample = self.samples[idx] | |
| # Load image | |
| image = cv2.imread(sample['image_path']) | |
| image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
| orig_h, orig_w = image.shape[:2] | |
| # Get prompts at original resolution | |
| prompts = get_bboxes_for_sam( | |
| orig_w, orig_h, sample['annotations'], self.expand_ratio | |
| ) | |
| # Create GT mask at original resolution | |
| from src.data.yolo_to_masks import create_bbox_mask | |
| gt_mask = create_bbox_mask(orig_w, orig_h, sample['annotations'], binary=True) | |
| # Resize image for SAM | |
| scale_x = self.image_size / orig_w | |
| scale_y = self.image_size / orig_h | |
| image_resized = cv2.resize(image, (self.image_size, self.image_size)) | |
| gt_mask_resized = cv2.resize(gt_mask, (self.image_size, self.image_size), | |
| interpolation=cv2.INTER_NEAREST) | |
| # Scale prompts to resized coordinates | |
| scaled_prompts = [] | |
| for p in prompts: | |
| box = p['box'].copy().astype(float) | |
| box[0] *= scale_x | |
| box[1] *= scale_y | |
| box[2] *= scale_x | |
| box[3] *= scale_y | |
| center = p['center_point'].copy().astype(float) | |
| center[0] *= scale_x | |
| center[1] *= scale_y | |
| scaled_prompts.append({ | |
| 'box': box.astype(int), | |
| 'class_id': p['class_id'], | |
| 'center_point': center.astype(int), | |
| 'box_original': p['box'], | |
| }) | |
| metadata = { | |
| 'stem': sample['stem'], | |
| 'original_size': (orig_h, orig_w), | |
| 'resized_size': (self.image_size, self.image_size), | |
| 'scale': (scale_x, scale_y), | |
| 'num_objects': len(prompts), | |
| 'class_ids': [p['class_id'] for p in prompts], | |
| } | |
| return image_resized, scaled_prompts, gt_mask_resized, metadata | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Utility: DataLoader creation | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def create_unet_dataloaders(project_root: str, | |
| batch_size: int = 8, | |
| image_size: int = 512, | |
| num_workers: int = 4): | |
| """Create train and val DataLoaders for UNet.""" | |
| from torch.utils.data import DataLoader | |
| train_ds = BTXRDSegmentationDataset( | |
| images_dir=os.path.join(project_root, 'BTXRD dataset', 'images_512', 'train'), | |
| labels_dir=os.path.join(project_root, 'BTXRD dataset', 'labels', 'train'), | |
| masks_dir=os.path.join(project_root, 'BTXRD dataset', 'masks_512', 'train'), | |
| transform=get_unet_train_transforms(image_size), | |
| image_size=image_size, | |
| ) | |
| val_ds = BTXRDSegmentationDataset( | |
| images_dir=os.path.join(project_root, 'BTXRD dataset', 'images_512', 'val'), | |
| labels_dir=os.path.join(project_root, 'BTXRD dataset', 'labels', 'val'), | |
| masks_dir=os.path.join(project_root, 'BTXRD dataset', 'masks_512', 'val'), | |
| transform=get_unet_val_transforms(image_size), | |
| image_size=image_size, | |
| ) | |
| train_loader = DataLoader( | |
| train_ds, batch_size=batch_size, shuffle=True, | |
| num_workers=num_workers, pin_memory=True, drop_last=True | |
| ) | |
| val_loader = DataLoader( | |
| val_ds, batch_size=batch_size, shuffle=False, | |
| num_workers=num_workers, pin_memory=True | |
| ) | |
| return train_loader, val_loader | |
| if __name__ == '__main__': | |
| # Quick test | |
| project_root = '.' | |
| print("Testing BTXRDSegmentationDataset...") | |
| ds = BTXRDSegmentationDataset( | |
| images_dir=os.path.join(project_root, 'BTXRD dataset', 'images', 'val'), | |
| labels_dir=os.path.join(project_root, 'BTXRD dataset', 'labels', 'val'), | |
| transform=get_unet_val_transforms(512), | |
| ) | |
| print(f" Val dataset size: {len(ds)}") | |
| img, mask, meta = ds[0] | |
| print(f" Image shape: {img.shape}, Mask shape: {mask.shape}") | |
| print(f" Mask unique values: {mask.unique().tolist()}") | |
| print("\nTesting BTXRDSAMDataset...") | |
| sam_ds = BTXRDSAMDataset( | |
| images_dir=os.path.join(project_root, 'BTXRD dataset', 'images', 'val'), | |
| labels_dir=os.path.join(project_root, 'BTXRD dataset', 'labels', 'val'), | |
| ) | |
| print(f" SAM dataset size: {len(sam_ds)}") | |
| img, prompts, gt_mask, meta = sam_ds[0] | |
| print(f" Image shape: {img.shape}") | |
| print(f" Num prompts: {len(prompts)}") | |
| print(f" GT mask shape: {gt_mask.shape}") | |
| print(f" First prompt box: {prompts[0]['box']}") | |