qbzju
:sparkles: Add adobe mattingrefine model
7d031b9
Raw History Blame Contribute Delete
8.32 kB
import os
import gradio as gr
import torch
from model import MattingRefine, MattingBase
import numpy as np
import cv2
import gdown
import zipfile
import random
device = torch.device('cpu')
os.makedirs("checkpoints", exist_ok=True)
os.makedirs("examples", exist_ok=True)
os.system("wget https://github.com/PeterL1n/BackgroundMattingV2/releases/download/v1.0.0/pytorch_mobilenetv2.pth -O ./checkpoints/pytorch_mobilenetv2.pth")
os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-p3m10k.pth -O ./checkpoints/mattingrefine-mobilenetv3-p3m10k.pth")
os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-videomatte240k.pth -O ./checkpoints/mattingrefine-mobilenetv3-videomatte240k.pth")
os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth -O ./checkpoints/mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth")
os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingbase-mobilenetv3-adobe.pth -O ./checkpoints/mattingbase-mobilenetv3-adobe.pth")
os.system("wget https://huggingface.co/Shangyunle/BackgroundMatting/resolve/main/mattingrefine-mobilenetv3-adobe.pth -O ./checkpoints/mattingrefine-mobilenetv3-adobe.pth")
output_filename = "./examples/Images.zip"
Images = gdown.download("https://drive.google.com/uc?id=1yOpao-sbFKmEuKYcPN9e95586RQHjiE6", output=output_filename, quiet=False)
with zipfile.ZipFile(output_filename, 'r') as zip_ref:
zip_ref.extractall("./examples")
MODELS = {
"Pretrained MobileNetV2": "pytorch_mobilenetv2.pth",
"Custom Base MobileNetV3 (Adobe)": "mattingbase-mobilenetv3-adobe.pth",
"Custom Refine MobileNetV3 (P3M10K)": "mattingrefine-mobilenetv3-p3m10k.pth",
"Custom Refine MobileNetV3 (VideoMatte240K)": "mattingrefine-mobilenetv3-videomatte240k.pth",
"Custom Refine MobileNetV3 (VideoMatte240K Epoch 1)": "mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth",
"Custom Refine MobileNetV3 (Adobe)": "mattingrefine-mobilenetv3-adobe.pth"
}
loaded_model = {
'pytorch_mobilenetv2.pth': MattingRefine(
backbone='mobilenetv2',
backbone_scale=.25,
refine_mode='sampling',
refine_sample_pixels=80_000
),
'mattingrefine-mobilenetv3-p3m10k.pth': MattingRefine(
backbone='mobilenetv3',
backbone_scale=.25,
refine_mode='sampling',
refine_sample_pixels=80_000,
refine_threshold=.7,
refine_kernel_size=3,
),
'mattingrefine-mobilenetv3-videomatte240k.pth': MattingRefine(
backbone='mobilenetv3',
backbone_scale=.25,
refine_mode='sampling',
refine_sample_pixels=80_000,
),
'mattingrefine-mobilenetv3-videomatte240k-epoch-1.pth': MattingRefine(
backbone='mobilenetv3',
backbone_scale=.25,
refine_mode='sampling',
refine_sample_pixels=80_000,
),
'mattingbase-mobilenetv3-adobe.pth': MattingBase(
backbone='mobilenetv3',
),
'mattingrefine-mobilenetv3-adobe.pth': MattingRefine(
backbone='mobilenetv3',
backbone_scale=.25,
refine_mode='sampling',
refine_sample_pixels=80_000,
)
}
for model_path in MODELS.values():
loaded_model[model_path].load_state_dict(
torch.load(f'checkpoints/{model_path}',
map_location=torch.device('cpu'),
weights_only=True)
)
def alignImages(src, bgr2, pha, fgr):
"""
Composite the foreground onto a new background image and adjust foreground exposure to match the new background
Parameters:
src: Original image
bgr2: Target background image
pha: Alpha mask
fgr: Extracted foreground
Returns:
tuple: (alpha, composite) - Alpha mask and composited image
"""
# Ensure input images have consistent dimensions
height, width = src.shape[:2]
bgr2 = cv2.resize(bgr2, (width, height))
# Ensure alpha and foreground have correct dimensions
if len(pha.shape) == 3:
pha = pha.squeeze() # Remove extra dimensions
pha = cv2.resize(pha, (width, height))
fgr = cv2.resize(fgr, (width, height))
# Composite final image
alpha_3channel = np.stack([pha, pha, pha], axis=2).astype(float) / 255
composite = fgr * alpha_3channel + bgr2 * (1 - alpha_3channel)
return pha, composite.astype(np.uint8)
def process_image(src_image, bgr_image, new_bgr_image, model_name):
if src_image is None or bgr_image is None or new_bgr_image is None:
raise gr.Error("Please provide all required images")
src_image = cv2.imread(src_image)
bgr_image = cv2.imread(bgr_image)
new_bgr_image = cv2.imread(new_bgr_image)
src_image = cv2.cvtColor(src_image, cv2.COLOR_BGR2RGB)
bgr_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
new_bgr_image = cv2.cvtColor(new_bgr_image, cv2.COLOR_BGR2RGB)
model = loaded_model[MODELS[model_name]]
model.eval()
src_image = np.array(src_image)
bgr_image = np.array(bgr_image)
new_bgr_image = np.array(new_bgr_image)
src = torch.from_numpy(src_image).float().permute(2, 0, 1).unsqueeze(0) / 255
bgr = torch.from_numpy(bgr_image).float().permute(2, 0, 1).unsqueeze(0) / 255
with torch.no_grad():
pha, fgr = model(src, bgr)[:2]
alpha = (pha[0].cpu().numpy() * 255).astype(np.uint8)
foreground = (fgr[0].permute(1, 2, 0).cpu().numpy() * 255).astype(np.uint8)
alpha_out, composite = alignImages(src_image, new_bgr_image, alpha, foreground)
return alpha_out, composite
def generate_examples(max_examples=5):
img_dir = "examples/Images/img"
bgr_dir = "examples/Images/bgr"
img_files = [f for f in os.listdir(img_dir) if f.endswith(('.png', '.jpg', '.jpeg'))]
bgr_files = [f for f in os.listdir(bgr_dir) if f.endswith(('.png', '.jpg', '.jpeg'))]
examples = []
max_examples = min(max_examples, len(img_files))
for img_file in img_files:
bgr_file = img_file
if bgr_file in bgr_files:
tar_file = random.choice([f for f in bgr_files if f != bgr_file])
example = [
os.path.join("examples/Images/img", img_file),
os.path.join("examples/Images/bgr", bgr_file),
os.path.join("examples/Images/bgr", tar_file),
"Pretrained MobileNetV2"
]
examples.append(example)
if len(examples) == max_examples - 1:
break
return examples
interface = gr.Interface(
fn=process_image,
inputs=[
gr.Image(label="Source Image", type='filepath'),
gr.Image(label="Background Image", type='filepath'),
gr.Image(label="New Background Image", type='filepath'),
gr.Dropdown(
choices=list(MODELS.keys()),
value="Pretrained MobileNetV2",
label="Choose a Model"
)
],
outputs=[
gr.Image(label="Alpha Mask"),
gr.Image(label="Foreground")
],
title="Image Matting",
description="Upload three images: 1. Source image 2. Background image 3. New background image. The system will extract the foreground and composite it onto the new background.",
examples=generate_examples(),
cache_examples=True
)
if __name__ == "__main__":
interface.launch()