import os import gradio as gr import torch from model import MattingRefine, MattingBase import numpy as np import cv2 import gdown import zipfile import random device = torch.device('cpu') os.makedirs("checkpoints", exist_ok=True) os.makedirs("examples", exist_ok=True) os.system("wget https://github.com/PeterL1n/BackgroundMattingV2/releases/download/v1.0.0/pytorch_mobilenetv2.pth -O ./checkpoints/pytorch_mobilenetv2.pth") os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-p3m10k.pth -O ./checkpoints/mattingrefine-mobilenetv3-p3m10k.pth") os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-videomatte240k.pth -O ./checkpoints/mattingrefine-mobilenetv3-videomatte240k.pth") os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth -O ./checkpoints/mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth") os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingbase-mobilenetv3-adobe.pth -O ./checkpoints/mattingbase-mobilenetv3-adobe.pth") os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-adobe.pth -O ./checkpoints/mattingrefine-mobilenetv3-adobe.pth") output_filename = "./examples/Images.zip" Images = gdown.download("https://drive.google.com/uc?id=1yOpao-sbFKmEuKYcPN9e95586RQHjiE6", output=output_filename, quiet=False) with zipfile.ZipFile(output_filename, 'r') as zip_ref: zip_ref.extractall("./examples") MODELS = { "Pretrained MobileNetV2": "pytorch_mobilenetv2.pth", "Custom Base MobileNetV3 (Adobe)": "mattingbase-mobilenetv3-adobe.pth", "Custom Refine MobileNetV3 (P3M10K)": "mattingrefine-mobilenetv3-p3m10k.pth", "Custom Refine MobileNetV3 (VideoMatte240K)": "mattingrefine-mobilenetv3-videomatte240k.pth", "Custom Refine MobileNetV3 (VideoMatte240K Epoch 1)": "mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth", "Custom Refine MobileNetV3 (Adobe)": "mattingrefine-mobilenetv3-adobe.pth" } loaded_model = { 'pytorch_mobilenetv2.pth': MattingRefine( backbone='mobilenetv2', backbone_scale=.25, refine_mode='sampling', refine_sample_pixels=80_000 ), 'mattingrefine-mobilenetv3-p3m10k.pth': MattingRefine( backbone='mobilenetv3', backbone_scale=.25, refine_mode='sampling', refine_sample_pixels=80_000, refine_threshold=.7, refine_kernel_size=3, ), 'mattingrefine-mobilenetv3-videomatte240k.pth': MattingRefine( backbone='mobilenetv3', backbone_scale=.25, refine_mode='sampling', refine_sample_pixels=80_000, ), 'mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth': MattingRefine( backbone='mobilenetv3', backbone_scale=.25, refine_mode='sampling', refine_sample_pixels=80_000, ), 'mattingbase-mobilenetv3-adobe.pth': MattingBase( backbone='mobilenetv3', ), 'mattingrefine-mobilenetv3-adobe.pth': MattingRefine( backbone='mobilenetv3', backbone_scale=.25, refine_mode='sampling', refine_sample_pixels=80_000, ) } for model_path in MODELS.values(): loaded_model[model_path].load_state_dict( torch.load(f'checkpoints/{model_path}', map_location=torch.device('cpu'), weights_only=True) ) def alignImages(src, bgr2, pha, fgr): """ Composite the foreground onto a new background image and adjust foreground exposure to match the new background Parameters: src: Original image bgr2: Target background image pha: Alpha mask fgr: Extracted foreground Returns: tuple: (alpha, composite) - Alpha mask and composited image """ # Ensure input images have consistent dimensions height, width = src.shape[:2] bgr2 = cv2.resize(bgr2, (width, height)) # Ensure alpha and foreground have correct dimensions if len(pha.shape) == 3: pha = pha.squeeze() # Remove extra dimensions pha = cv2.resize(pha, (width, height)) fgr = cv2.resize(fgr, (width, height)) # Composite final image alpha_3channel = np.stack([pha, pha, pha], axis=2).astype(float) / 255 composite = fgr * alpha_3channel + bgr2 * (1 - alpha_3channel) return pha, composite.astype(np.uint8) def process_image(src_image, bgr_image, new_bgr_image, model_name): if src_image is None or bgr_image is None or new_bgr_image is None: raise gr.Error("Please provide all required images") src_image = cv2.imread(src_image) bgr_image = cv2.imread(bgr_image) new_bgr_image = cv2.imread(new_bgr_image) src_image = cv2.cvtColor(src_image, cv2.COLOR_BGR2RGB) bgr_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) new_bgr_image = cv2.cvtColor(new_bgr_image, cv2.COLOR_BGR2RGB) model = loaded_model[MODELS[model_name]] model.eval() src_image = np.array(src_image) bgr_image = np.array(bgr_image) new_bgr_image = np.array(new_bgr_image) src = torch.from_numpy(src_image).float().permute(2, 0, 1).unsqueeze(0) / 255 bgr = torch.from_numpy(bgr_image).float().permute(2, 0, 1).unsqueeze(0) / 255 with torch.no_grad(): pha, fgr = model(src, bgr)[:2] alpha = (pha[0].cpu().numpy() * 255).astype(np.uint8) foreground = (fgr[0].permute(1, 2, 0).cpu().numpy() * 255).astype(np.uint8) alpha_out, composite = alignImages(src_image, new_bgr_image, alpha, foreground) return alpha_out, composite def generate_examples(max_examples=5): img_dir = "examples/Images/img" bgr_dir = "examples/Images/bgr" img_files = [f for f in os.listdir(img_dir) if f.endswith(('.png', '.jpg', '.jpeg'))] bgr_files = [f for f in os.listdir(bgr_dir) if f.endswith(('.png', '.jpg', '.jpeg'))] examples = [] max_examples = min(max_examples, len(img_files)) for img_file in img_files: bgr_file = img_file if bgr_file in bgr_files: tar_file = random.choice([f for f in bgr_files if f != bgr_file]) example = [ os.path.join("examples/Images/img", img_file), os.path.join("examples/Images/bgr", bgr_file), os.path.join("examples/Images/bgr", tar_file), "Pretrained MobileNetV2" ] examples.append(example) if len(examples) == max_examples - 1: break return examples interface = gr.Interface( fn=process_image, inputs=[ gr.Image(label="Source Image", type='filepath'), gr.Image(label="Background Image", type='filepath'), gr.Image(label="New Background Image", type='filepath'), gr.Dropdown( choices=list(MODELS.keys()), value="Pretrained MobileNetV2", label="Choose a Model" ) ], outputs=[ gr.Image(label="Alpha Mask"), gr.Image(label="Foreground") ], title="Image Matting", description="Upload three images: 1. Source image 2. Background image 3. New background image. The system will extract the foreground and composite it onto the new background.", examples=generate_examples(), cache_examples=True ) if __name__ == "__main__": interface.launch()