import os import pathlib import logging import cv2 import numpy as np import torch from label_studio_sdk.converter import brush from typing import List, Dict, Optional from uuid import uuid4 from label_studio_ml.model import LabelStudioMLBase, ModelResponse from label_studio_sdk._extensions.label_studio_tools.core.utils.params import get_bool_env from label_studio_sdk.label_interface.objects import PredictionValue from segment_anything.utils.transforms import ResizeLongestSide from groundingdino.util.inference import load_model, load_image, predict, annotate from groundingdino.util import box_ops # ----Extra Libraries from typing import Tuple, List from groundingdino.util.utils import get_phrases_from_posmap from groundingdino.util.inference import preprocess_caption logger = logging.getLogger(__name__) def predict_batch( model, images: torch.Tensor, caption: str, box_threshold: float, text_threshold: float, device: str = "cuda" ) -> Tuple[List[torch.Tensor], List[torch.Tensor], List[List[str]]]: # copy from https://github.com/yuwenmichael/Grounding-DINO-Batch-Inference/blob/main/batch_utlities.py ''' return: bboxes_batch: list of tensors of shape (n, 4) predicts_batch: list of tensors of shape (n,) phrases_batch: list of list of strings of shape (n,) n is the number of boxes in one image ''' caption = preprocess_caption(caption=caption) model = model.to(device) image = images.to(device) with torch.no_grad(): outputs = model(image, captions=[caption for _ in range( len(images))]) # <------- I use the same caption for all the images for my use-case prediction_logits = outputs["pred_logits"].cpu().sigmoid() # prediction_logits.shape = (num_batch, nq, 256) prediction_boxes = outputs["pred_boxes"].cpu() # prediction_boxes.shape = (num_batch, nq, 4) mask = prediction_logits.max(dim=2)[0] > box_threshold # mask: torch.Size([num_batch, 256]) bboxes_batch = [] predicts_batch = [] phrases_batch = [] # list of lists tokenizer = model.tokenizer tokenized = tokenizer(caption) for i in range(prediction_logits.shape[0]): logits = prediction_logits[i][mask[i]] # logits.shape = (n, 256) phrases = [ get_phrases_from_posmap(logit > text_threshold, tokenized, tokenizer).replace('.', '') for logit # logit is a tensor of shape (256,) torch.Size([256]) in logits # torch.Size([7, 256]) ] boxes = prediction_boxes[i][mask[i]] # boxes.shape = (n, 4) phrases_batch.append(phrases) bboxes_batch.append(boxes) predicts_batch.append(logits.max(dim=1)[0]) return bboxes_batch, predicts_batch, phrases_batch # LOADING THE MODEL groundingdino_model = load_model( pathlib.Path(os.environ.get('GROUNDINGDINO_REPO_PATH', "./GroundingDINO")) / "groundingdino" / "config" / "GroundingDINO_SwinT_OGC.py", pathlib.Path(os.environ.get('GROUNDINGDINO_REPO_PATH', "./GroundingDINO")) / "weights" / "groundingdino_swint_ogc.pth" ) BOX_THRESHOLD = os.environ.get("BOX_THRESHOLD", 0.3) TEXT_THRESHOLD = os.environ.get("TEXT_THRESHOLD", 0.25) LABEL_STUDIO_ACCESS_TOKEN = ( os.environ.get("LABEL_STUDIO_ACCESS_TOKEN") or os.environ.get("LABEL_STUDIO_API_KEY") ) LABEL_STUDIO_HOST = ( os.environ.get("LABEL_STUDIO_HOST") or os.environ.get("LABEL_STUDIO_URL") ) USE_SAM = get_bool_env("USE_SAM", default=False) USE_MOBILE_SAM = get_bool_env("USE_MOBILE_SAM", default=False) MOBILESAM_CHECKPOINT = os.environ.get("MOBILESAM_CHECKPOINT", "mobile_sam.pt") SAM_CHECKPOINT = os.environ.get("SAM_CHECKPOINT", "sam_vit_h_4b8939.pth") device = "cuda" if torch.cuda.is_available() else "cpu" logger.info(f"Using device {device}") if USE_MOBILE_SAM: logger.info(f"Using Mobile-SAM with checkpoint {MOBILESAM_CHECKPOINT}") from mobile_sam import SamPredictor, sam_model_registry model_checkpoint = MOBILESAM_CHECKPOINT reg_key = 'vit_t' elif USE_SAM: logger.info(f"Using SAM with checkpoint {SAM_CHECKPOINT}") from segment_anything import SamPredictor, sam_model_registry model_checkpoint = SAM_CHECKPOINT reg_key = 'vit_h' else: reg_key = None model_checkpoint = None logger.info("Using GroundingDINO without SAM") if USE_MOBILE_SAM or USE_SAM: logger.info(f"Loading SAM model with checkpoint {model_checkpoint}") sam = sam_model_registry[reg_key](checkpoint=model_checkpoint) sam.to(device=device) predictor = SamPredictor(sam) logger.info("SAM model successfully loaded!") class DINOBackend(LabelStudioMLBase): def setup(self): self.set("model_version", f'{self.__class__.__name__}-v0.0.1') def predict(self, tasks: List[Dict], context: Optional[Dict] = None, **kwargs) -> List[Dict]: if not context or not context.get('result'): # if there is no context, no interaction has happened yet return [] from_name_r, to_name_r, value = self.get_first_tag_occurence('RectangleLabels', 'Image') from_name_b, to_name_b, _ = self.get_first_tag_occurence('BrushLabels', 'Image') text_prompt = context['result'][0]['value']['text'][0] logger.debug(f"Prompt: {text_prompt}") logger.info(f"the prompt is {text_prompt} and {from_name_r} and {from_name_b}") final_predictions = [] if len(tasks) > 1: logger.info(f"Running multiple tasks with {len(tasks)} images") final_predictions = self.multiple_tasks( tasks, text_prompt, from_name_r, to_name_r, from_name_b, to_name_b, value) elif len(tasks) == 1: logger.info(f"Running single task {tasks[0]}") final_predictions = self.one_task( tasks[0], text_prompt, from_name_r, to_name_r, from_name_b, to_name_b, value) return final_predictions def one_task(self, task, prompt, from_name_r, to_name_r, from_name_b, to_name_b, value): all_points = [] all_scores = [] all_lengths = [] predictions = [] raw_img_path = task['data'][value] try: img_path = self.get_local_path( raw_img_path, ls_access_token=LABEL_STUDIO_ACCESS_TOKEN, ls_host=LABEL_STUDIO_HOST, task_id=task.get('id') ) except Exception as e: logger.error(f"Error getting image path: {e}") img_path = raw_img_path src, img = load_image(img_path) boxes, logits, _ = predict( model=groundingdino_model, image=img, caption=prompt, box_threshold=float(BOX_THRESHOLD), text_threshold=float(TEXT_THRESHOLD), device=device ) H, W, _ = src.shape boxes_xyxy = box_ops.box_cxcywh_to_xyxy(boxes) * torch.Tensor([W, H, W, H]) points = boxes_xyxy.cpu().numpy() for point, logit in zip(points, logits): all_points.append(point) all_scores.append(logit) all_lengths.append((H, W)) if USE_MOBILE_SAM or USE_SAM: # get results predictions.append(self.get_sam_results(img_path, all_points, all_lengths, from_name_b, to_name_b)) else: # get results predictions.append(self.get_results(all_points, all_scores, all_lengths, from_name_r, to_name_r)) return predictions def multiple_tasks(self, tasks, prompt, from_name_r, to_name_r, from_name_b, to_name_b, value): # first getting all the image paths image_paths = [] for task in tasks: raw_img_path = task['data'][value] try: img_path = self.get_local_path( raw_img_path, ls_access_token=LABEL_STUDIO_ACCESS_TOKEN, ls_host=LABEL_STUDIO_HOST, task_id=task.get('id') ) except Exception as e: logger.error(f"Error getting local path: {e}") img_path = raw_img_path image_paths.append(img_path) boxes, logits, lengths = self.batch_dino(image_paths, prompt) box_by_task = [] for (box_task, (H, W)) in zip(boxes, lengths): boxes_xyxy = box_ops.box_cxcywh_to_xyxy(box_task) * torch.Tensor([W, H, W, H]) box_by_task.append(boxes_xyxy) if USE_MOBILE_SAM or USE_SAM: batched_output = self.batch_sam(input_boxes_list=box_by_task, image_paths=image_paths) predictions = self.get_batched_sam_results(batched_output, from_name_b, to_name_b) else: predictions = [] for boxes_xyxy, (H, W), logits in zip(box_by_task, lengths, logits): points = boxes_xyxy.cpu().numpy() all_points = [] all_scores = [] all_lengths = [] for point, logit in zip(points, logits): all_points.append(point) all_scores.append(logit) all_lengths.append((H, W)) # figure out how to get this predictions.append(self.get_results(all_points, all_scores, all_lengths, from_name_r, to_name_r)) return predictions # make sure you use new github repo when predicting in batch def batch_dino(self, image_paths, prompt): # text prompt is same as self.label loaded_images = [] lengths = [] for img in image_paths: src, img = load_image(img) loaded_images.append(img) H, W, _ = src.shape lengths.append((H, W)) images = torch.stack(loaded_images) if len(image_paths) <= 3: boxes, logits, _ = predict_batch( model=groundingdino_model, images=images, caption=prompt, # text prompt is same as self.label box_threshold=float(BOX_THRESHOLD), text_threshold=float(TEXT_THRESHOLD), device=device ) else: all_boxes = [] all_logits = [] for img in loaded_images: boxes, logits, _ = predict( model=groundingdino_model, image=img, caption=prompt, box_threshold=float(BOX_THRESHOLD), text_threshold=float(TEXT_THRESHOLD), device=device ) all_boxes.append(boxes) all_logits.append(logits) boxes = all_boxes logits = all_logits return boxes, logits, lengths def batch_sam(self, input_boxes_list, image_paths): resize_transform = ResizeLongestSide(sam.image_encoder.img_size) # from SAM code base def prepare_image(image, transform, device): image = transform.apply_image(image) image = torch.as_tensor(image, device=device.device) return image.permute(2, 0, 1).contiguous() batched_input = [] for input_box, path in zip(input_boxes_list, image_paths): image = cv2.imread(path) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) batched_input.append({ 'image': prepare_image(image, resize_transform, sam), 'boxes': resize_transform.apply_boxes_torch(input_box, image.shape[:2]), 'original_size': image.shape[:2] }) batched_output = sam(batched_input, multimask_output=False) return batched_output def get_batched_sam_results(self, batched_output, from_name_b, to_name_b): predictions = [] for output in batched_output: masks = output['masks'] masks = masks[:, 0, :, :].cpu().numpy().astype(np.uint8) probs = output['iou_predictions'].cpu().numpy() num_masks = masks.shape[0] height = masks.shape[-2] width = masks.shape[-1] lengths = [(height, width)] * num_masks predictions.append(self.sam_predictions(masks, probs, lengths, from_name_b, to_name_b)) return predictions def get_results(self, all_points, all_scores, all_lengths, from_name_r, to_name_r): results = [] total_score = 0 for points, scores, lengths in zip(all_points, all_scores, all_lengths): # random ID label_id = str(uuid4())[:9] height, width = lengths score = scores.item() total_score += score results.append({ 'id': label_id, 'from_name': from_name_r, 'to_name': to_name_r, 'original_width': width, 'original_height': height, 'image_rotation': 0, 'value': { 'rotation': 0, 'width': (points[2] - points[0]) / width * 100, 'height': (points[3] - points[1]) / height * 100, 'x': points[0] / width * 100, 'y': points[1] / height * 100 }, 'score': score, 'type': 'rectanglelabels', 'readonly': False }) total_score /= max(len(results), 1) return { 'result': results, 'score': total_score, 'model_version': self.get('model_version') } def get_sam_results( self, img_path, input_boxes, lengths, from_name_b, to_name_b ): image = cv2.imread(img_path) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) predictor.set_image(image) input_boxes = torch.from_numpy(np.array(input_boxes)) transformed_boxes = predictor.transform.apply_boxes_torch(input_boxes, image.shape[:2]).to(device) masks, probs, _ = predictor.predict_torch( point_coords=None, point_labels=None, boxes=transformed_boxes, multimask_output=False, ) masks = masks[:, 0, :, :].cpu().numpy().astype(np.uint8) probs = probs.cpu().numpy() return self.sam_predictions(masks, probs, lengths, from_name_b, to_name_b) # takes straight masks and returns predictions def sam_predictions(self, masks, probs, lengths, from_name_b, to_name_b): results = [] total_score = 0 for mask, prob, length in zip(masks, probs, lengths): height, width = length # creates a random ID for your label everytime so no chance for errors label_id = str(uuid4())[:9] # converting the mask from the model to RLE format which is usable in Label Studio mask = mask * 255 rle = brush.mask2rle(mask) score = float(prob[0]) results.append({ 'id': label_id, 'from_name': from_name_b, 'to_name': to_name_b, 'original_width': width, 'original_height': height, 'image_rotation': 0, 'value': { 'format': 'rle', 'rle': rle }, 'score': score, 'type': 'brushlabels', 'readonly': False }) total_score += score return { 'result': results, 'score': total_score / max(len(results), 1), 'model_version': self.get('model_version') }