import spaces import gradio as gr import numpy as np from PIL import Image import os import json from huggingface_hub import hf_hub_download from segment_anything import sam_model_registry, SamPredictor hf_token = os.environ.get("HF_TOKEN") # Download only the 375MB SAM file so logs don't get flooded and cut off sam_checkpoint = hf_hub_download( repo_id="LiuZichen/MagicQuillV2-models", filename="sam/sam_vit_b_01ec64.pth", repo_type="model", local_dir="models_v2", token=hf_token, ) print("Initializing SAM...") sam = sam_model_registry["vit_b"](checkpoint=sam_checkpoint) sam_predictor = SamPredictor(sam) print("SAM initialized.") def get_mask_bbox(mask_np): if mask_np.ndim == 3: mask_np = mask_np[0] rows = np.any(mask_np, axis=1) cols = np.any(mask_np, axis=0) if not np.any(rows) or not np.any(cols): return None y_min, y_max = np.where(rows)[0][[0, -1]] x_min, x_max = np.where(cols)[0][[0, -1]] return int(x_min), int(y_min), int(x_max), int(y_max) @spaces.GPU def segment(image, coordinates_positive, coordinates_negative, bboxes): # Move SAM to GPU inside the @spaces.GPU context if next(sam.parameters()).device.type != "cuda": sam.to(device="cuda") print("image.shape:", image.shape) print("coordinates_positive:", coordinates_positive) print("coordinates_negative:", coordinates_negative) print("bboxes:", bboxes) sam_predictor.set_image(image) input_point = [] input_label = [] if coordinates_positive: coords = ( json.loads(coordinates_positive) if isinstance(coordinates_positive, str) else coordinates_positive ) for p in coords: input_point.append([p["x"], p["y"]]) input_label.append(1) if coordinates_negative: coords = ( json.loads(coordinates_negative) if isinstance(coordinates_negative, str) else coordinates_negative ) for p in coords: input_point.append([p["x"], p["y"]]) input_label.append(0) input_box = None if bboxes: if isinstance(bboxes, str): try: bboxes = json.loads(bboxes) except Exception: pass box_list = [] if isinstance(bboxes, list): for box in bboxes: box_list.append(list(box)) if len(box_list) > 0: input_box = np.array(box_list) if len(input_point) > 0: input_point = np.array(input_point) input_label = np.array(input_label) else: input_point = None input_label = None masks, scores, logits = sam_predictor.predict( point_coords=input_point, point_labels=input_label, box=input_box, multimask_output=False, ) mask_np = masks[0] if mask_np.dtype == bool: mask_np = mask_np.astype(np.uint8) * 255 else: mask_np = (mask_np > 0).astype(np.uint8) * 255 res_pil = Image.fromarray(mask_np) mask_bbox = get_mask_bbox(mask_np) if mask_bbox: x_min, y_min, x_max, y_max = mask_bbox seg_bbox = { "startX": x_min, "startY": y_min, "endX": x_max, "endY": y_max, } else: seg_bbox = { "startX": 0, "startY": 0, "endX": 0, "endY": 0, } return res_pil, json.dumps(seg_bbox) with gr.Blocks() as app: gr.Markdown("# MagicQuill SAM Helper") input_image = gr.Image(type="numpy", label="Image") pos_coords = gr.Textbox(label="Positive Coordinates JSON") neg_coords = gr.Textbox(label="Negative Coordinates JSON") boxes = gr.Textbox(label="Bounding Boxes JSON") output_mask = gr.Image(label="Mask", format="png") output_bbox = gr.Textbox(label="BBox") run_btn = gr.Button("Segment") run_btn.click( fn=segment, inputs=[ input_image, pos_coords, neg_coords, boxes, ], outputs=[ output_mask, output_bbox, ], api_name="segment", concurrency_limit=5, ) if __name__ == "__main__": app.queue(max_size=40).launch(max_threads=5)