boom-vvt / src /process /sam3_detect.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
6ba0ed9 verified
Raw History Blame Contribute Delete
6.07 kB
import argparse
import os
import cv2
import numpy as np
import torch
from PIL import Image
from diffusers.utils import export_to_video, load_video
from transformers import Sam3VideoModel, Sam3VideoProcessor
def to_numpy(array):
if isinstance(array, torch.Tensor):
return array.detach().cpu().numpy()
return np.asarray(array)
def to_mask_stack(masks, height, width):
if masks is None:
return np.zeros((0, height, width), dtype=bool)
masks_np = to_numpy(masks)
if masks_np.ndim == 2:
masks_np = masks_np[None, ...]
elif masks_np.ndim == 4 and masks_np.shape[1] == 1:
masks_np = masks_np[:, 0]
elif masks_np.ndim != 3:
return np.zeros((0, height, width), dtype=bool)
if masks_np.dtype != np.bool_:
masks_np = masks_np > 0
if masks_np.shape[-2:] != (height, width):
masks_np = np.stack(
[
cv2.resize(mask.astype(np.uint8), (width, height), interpolation=cv2.INTER_NEAREST) > 0
for mask in masks_np
],
axis=0,
)
return masks_np
def union_selected(masks_np, object_ids, selected_ids, height, width):
if masks_np.shape[0] == 0 or not selected_ids:
return np.zeros((height, width), dtype=bool)
selected = {int(i) for i in selected_ids}
rows = [masks_np[i] for i, obj_id in enumerate(object_ids) if int(obj_id) in selected]
if not rows:
return np.zeros((height, width), dtype=bool)
return np.any(np.stack(rows, axis=0), axis=0)
def combine_masks(processed, prompts, height, width):
masks_np = to_mask_stack(processed.get("masks"), height, width)
object_ids = processed.get("object_ids")
if object_ids is None:
object_ids = list(range(masks_np.shape[0]))
else:
object_ids = [int(i) for i in to_numpy(object_ids).reshape(-1)]
if isinstance(prompts, str):
prompts = [prompts]
keep_prompt = prompts[0]
drop_prompts = prompts[1:]
prompt_to_obj_ids = processed.get("prompt_to_obj_ids") or {}
if prompt_to_obj_ids:
keep_ids = list(prompt_to_obj_ids.get(keep_prompt, []))
drop_ids = []
for prompt in drop_prompts:
drop_ids.extend(prompt_to_obj_ids.get(prompt, []))
else:
keep_ids = object_ids[:1]
drop_ids = object_ids[1:]
keep = union_selected(masks_np, object_ids, keep_ids, height, width)
drop = union_selected(masks_np, object_ids, drop_ids, height, width)
return ((keep & ~drop).astype(np.uint8) * 255)
def init_model(model_path, device="cuda", dtype=torch.bfloat16):
print(f"[sam3] init model: {model_path}, device={device}, dtype={dtype}")
model = Sam3VideoModel.from_pretrained(model_path).to(device, dtype=dtype).eval()
processor = Sam3VideoProcessor.from_pretrained(model_path)
return model, processor
def data_process(video_path, num_frames=65):
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise RuntimeError(f"无法打开视频: {video_path}")
fps = cap.get(cv2.CAP_PROP_FPS)
cap.release()
frames = load_video(video_path)
if not frames:
raise ValueError(f"视频为空: {video_path}")
if num_frames is not None and int(num_frames) >= 0:
frames = frames[: int(num_frames)]
print(f"[data] frames={len(frames)}, size={frames[0].size}, fps={fps}")
return frames, fps
def infer_sam3(model, processor, frames, prompt, device="cuda"):
session = processor.init_video_session(
video=frames,
inference_device=device,
processing_device="cpu",
video_storage_device="cpu",
dtype=model.dtype,
)
session = processor.add_text_prompt(inference_session=session, text=prompt)
outputs = {}
for result in model.propagate_in_video_iterator(
inference_session=session,
max_frame_num_to_track=len(frames),
):
processed = processor.postprocess_outputs(session, result)
idx = int(result.frame_idx)
width, height = frames[idx].size
mask = combine_masks(processed, prompt, height, width)
outputs[idx] = Image.fromarray(mask).convert("RGB")
if (idx + 1) % 20 == 0:
print(f"[sam3] frame={idx + 1}/{len(frames)}")
mask_video = []
for i in range(len(frames)):
if i not in outputs:
width, height = frames[i].size
mask_video.append(Image.new("RGB", (width, height), 0))
else:
mask_video.append(outputs[i])
return mask_video
SAM3_PROMPTS = {
"upper_body": ["the upper body of the person","the head of the person"],
"lower_body": ["the lower body of the person","the feet of the person"],
"full_body": ["person","the head of the person","the feet of the person"],
}
def parse_args():
parser = argparse.ArgumentParser(description="SAM3 region detection")
parser.add_argument("--video", required=True, help="person video path")
parser.add_argument("--output", required=True, help="output directory")
parser.add_argument("--garment_type", default="full_body", choices=["upper_body", "lower_body", "full_body"])
parser.add_argument("--model_path", default="facebook/sam3")
parser.add_argument("--device", default="cuda")
parser.add_argument("--dtype", default="bfloat16")
parser.add_argument("--num_frames", type=int, default=65, help="max frames to process, -1 for all")
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
os.makedirs(args.output, exist_ok=True)
save_video = os.path.join(args.output, "sam3_mask.mp4")
model, processor = init_model(args.model_path, device=args.device, dtype=getattr(torch, args.dtype))
frames, fps = data_process(args.video, num_frames=args.num_frames)
mask_video = infer_sam3(
model, processor, frames, SAM3_PROMPTS[args.garment_type], device=args.device
)
if os.path.exists(save_video):
os.remove(save_video)
export_to_video(mask_video, save_video, fps=fps)
print(f"[sam3] video={save_video}")