from io import BytesIO import numpy as np import torch from segment_anything import sam_model_registry from skimage import transform import torch.nn.functional as F import argparse from PIL import Image import os import random def perform_segmentation(image_data, seg_path="assets/", box=[95, 255, 190, 350], device="cpu", checkpoint="medsam_vit_b.pth"): print("phai do dc") join = os.path.join @torch.no_grad() def medsam_inference(medsam_model, img_embed, box_1024, H, W): box_torch = torch.as_tensor(box_1024, dtype=torch.float, device=img_embed.device) if len(box_torch.shape) == 2: box_torch = box_torch[:, None, :] # (B, 1, 4) # Extract features for prompt and image regions sparse_embeddings, dense_embeddings = medsam_model.prompt_encoder( points=None, boxes=box_torch, masks=None, ) low_res_logits, _ = medsam_model.mask_decoder( image_embeddings=img_embed, # (B, 256, 64, 64) image_pe=medsam_model.prompt_encoder.get_dense_pe(), # (1, 256, 64, 64) sparse_prompt_embeddings=sparse_embeddings, # (B, 2, 256) dense_prompt_embeddings=dense_embeddings, # (B, 256, 64, 64) multimask_output=False, ) # Generate low-resolution segmentation prediction low_res_pred = torch.sigmoid(low_res_logits) # (1, 1, 256, 256) low_res_pred = F.interpolate( low_res_pred, size=(H, W), mode="bilinear", align_corners=False, ) # (1, 1, gt.shape) low_res_pred = low_res_pred.squeeze().cpu().numpy() # (256, 256) medsam_seg = (low_res_pred > 0.5).astype(np.uint8) # Adjust threshold as needed return medsam_seg args = argparse.Namespace(image_data=image_data, seg_path=seg_path, box=box, device=device, checkpoint=checkpoint) try: # Load the model checkpoint on the specified device medsam_model = sam_model_registry["vit_b"](checkpoint=args.checkpoint, map_location=torch.device(device)) medsam_model = medsam_model.to(device) medsam_model.eval() except FileNotFoundError: raise FileNotFoundError("Checkpoint file not found. Please verify the file path.") # Load image from bytes try: image = Image.open(BytesIO(args.image_data)) except (IOError, OSError): # Handle potential errors during image loading print("Error: Could not open image. Please check the image data or path.") return None # Indicate failure img_np = np.array(image) # # Handle images with alpha channel (if necessary) # if len(img_np.shape) == 3 and img_np.shape[-1] == 4: # img_3c = img_np[:, :, :3] # Extract RGB channels (assuming alpha is last) # else: # img_3c = img_np if len(img_np.shape) == 2: img_3c = np.repeat(img_np[:, :, None], 3, axis=-1) elif img_np.shape[-1] == 4: # Convert RGBA image to RGB by discarding the alpha channel img_3c = img_np[..., :3] else: img_3c = img_np H, W, _ = img_3c.shape # Image preprocessing img_1024 = transform.resize( img_3c, (1024, 1024), order=3, preserve_range=True, anti_aliasing=True ).astype(np.uint8) img_1024 = img_1024 / 255.0 # Normalize to [0, 1] # Convert the shape to (3, H, W) img_1024_tensor = ( torch.tensor(img_1024).float().permute(2, 0, 1).unsqueeze(0).to(device) ) box_np = np.array([args.box]) # Transfer box_np to 1024x1024 scale box_1024 = box_np / np.array([W, H, W, H]) * 1024 with torch.no_grad(): image_embedding = medsam_model.image_encoder(img_1024_tensor) # (1, 256, 64, 64) medsam_seg = medsam_inference(medsam_model, image_embedding, box_1024, H, W) # Save segmentation image try: random_number = random.randint(1, 100000) # Generate a random number for the filename filename = f"seg_{random_number}.jpg" Image.fromarray(medsam_seg).save(join(seg_path, filename)) except (IOError, OSError): print("Error: Could not save segmentation image. Please check the output path.") medsam_seg_list = medsam_seg.tolist() # Convert NumPy array to list of lists print("medsam_seg", medsam_seg) print("medsam_seg_list", medsam_seg_list) return medsam_seg_list