import argparse import os import cv2 import numpy as np import torch from PIL import Image from diffusers.utils import export_to_video, load_video from transformers import Sam3VideoModel, Sam3VideoProcessor def to_numpy(array): if isinstance(array, torch.Tensor): return array.detach().cpu().numpy() return np.asarray(array) def to_mask_stack(masks, height, width): if masks is None: return np.zeros((0, height, width), dtype=bool) masks_np = to_numpy(masks) if masks_np.ndim == 2: masks_np = masks_np[None, ...] elif masks_np.ndim == 4 and masks_np.shape[1] == 1: masks_np = masks_np[:, 0] elif masks_np.ndim != 3: return np.zeros((0, height, width), dtype=bool) if masks_np.dtype != np.bool_: masks_np = masks_np > 0 if masks_np.shape[-2:] != (height, width): masks_np = np.stack( [ cv2.resize(mask.astype(np.uint8), (width, height), interpolation=cv2.INTER_NEAREST) > 0 for mask in masks_np ], axis=0, ) return masks_np def union_selected(masks_np, object_ids, selected_ids, height, width): if masks_np.shape[0] == 0 or not selected_ids: return np.zeros((height, width), dtype=bool) selected = {int(i) for i in selected_ids} rows = [masks_np[i] for i, obj_id in enumerate(object_ids) if int(obj_id) in selected] if not rows: return np.zeros((height, width), dtype=bool) return np.any(np.stack(rows, axis=0), axis=0) def combine_masks(processed, prompts, height, width): masks_np = to_mask_stack(processed.get("masks"), height, width) object_ids = processed.get("object_ids") if object_ids is None: object_ids = list(range(masks_np.shape[0])) else: object_ids = [int(i) for i in to_numpy(object_ids).reshape(-1)] if isinstance(prompts, str): prompts = [prompts] keep_prompt = prompts[0] drop_prompts = prompts[1:] prompt_to_obj_ids = processed.get("prompt_to_obj_ids") or {} if prompt_to_obj_ids: keep_ids = list(prompt_to_obj_ids.get(keep_prompt, [])) drop_ids = [] for prompt in drop_prompts: drop_ids.extend(prompt_to_obj_ids.get(prompt, [])) else: keep_ids = object_ids[:1] drop_ids = object_ids[1:] keep = union_selected(masks_np, object_ids, keep_ids, height, width) drop = union_selected(masks_np, object_ids, drop_ids, height, width) return ((keep & ~drop).astype(np.uint8) * 255) def init_model(model_path, device="cuda", dtype=torch.bfloat16): print(f"[sam3] init model: {model_path}, device={device}, dtype={dtype}") model = Sam3VideoModel.from_pretrained(model_path).to(device, dtype=dtype).eval() processor = Sam3VideoProcessor.from_pretrained(model_path) return model, processor def data_process(video_path, num_frames=65): cap = cv2.VideoCapture(video_path) if not cap.isOpened(): raise RuntimeError(f"无法打开视频: {video_path}") fps = cap.get(cv2.CAP_PROP_FPS) cap.release() frames = load_video(video_path) if not frames: raise ValueError(f"视频为空: {video_path}") if num_frames is not None and int(num_frames) >= 0: frames = frames[: int(num_frames)] print(f"[data] frames={len(frames)}, size={frames[0].size}, fps={fps}") return frames, fps def infer_sam3(model, processor, frames, prompt, device="cuda"): session = processor.init_video_session( video=frames, inference_device=device, processing_device="cpu", video_storage_device="cpu", dtype=model.dtype, ) session = processor.add_text_prompt(inference_session=session, text=prompt) outputs = {} for result in model.propagate_in_video_iterator( inference_session=session, max_frame_num_to_track=len(frames), ): processed = processor.postprocess_outputs(session, result) idx = int(result.frame_idx) width, height = frames[idx].size mask = combine_masks(processed, prompt, height, width) outputs[idx] = Image.fromarray(mask).convert("RGB") if (idx + 1) % 20 == 0: print(f"[sam3] frame={idx + 1}/{len(frames)}") mask_video = [] for i in range(len(frames)): if i not in outputs: width, height = frames[i].size mask_video.append(Image.new("RGB", (width, height), 0)) else: mask_video.append(outputs[i]) return mask_video SAM3_PROMPTS = { "upper_body": ["the upper body of the person","the head of the person"], "lower_body": ["the lower body of the person","the feet of the person"], "full_body": ["person","the head of the person","the feet of the person"], } def parse_args(): parser = argparse.ArgumentParser(description="SAM3 region detection") parser.add_argument("--video", required=True, help="person video path") parser.add_argument("--output", required=True, help="output directory") parser.add_argument("--garment_type", default="full_body", choices=["upper_body", "lower_body", "full_body"]) parser.add_argument("--model_path", default="facebook/sam3") parser.add_argument("--device", default="cuda") parser.add_argument("--dtype", default="bfloat16") parser.add_argument("--num_frames", type=int, default=65, help="max frames to process, -1 for all") return parser.parse_args() if __name__ == "__main__": args = parse_args() os.makedirs(args.output, exist_ok=True) save_video = os.path.join(args.output, "sam3_mask.mp4") model, processor = init_model(args.model_path, device=args.device, dtype=getattr(torch, args.dtype)) frames, fps = data_process(args.video, num_frames=args.num_frames) mask_video = infer_sam3( model, processor, frames, SAM3_PROMPTS[args.garment_type], device=args.device ) if os.path.exists(save_video): os.remove(save_video) export_to_video(mask_video, save_video, fps=fps) print(f"[sam3] video={save_video}")