Spaces:
Running on Zero
Running on Zero
Download src/process/sam3_detect.py from hugging-apps/boom-vvt: direct link, hf CLI and curl.
- Browser
- Download file 6.07 kB
-
https://huggingface.co/spaces/hugging-apps/boom-vvt/resolve/main/src/process/sam3_detect.py
- Command line
-
hf download hf://spaces/hugging-apps/boom-vvt/src/process/sam3_detect.py
-
curl -L -o sam3_detect.py https://huggingface.co/spaces/hugging-apps/boom-vvt/resolve/main/src/process/sam3_detect.py
6.07 kB
| import argparse | |
| import os | |
| import cv2 | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from diffusers.utils import export_to_video, load_video | |
| from transformers import Sam3VideoModel, Sam3VideoProcessor | |
| def to_numpy(array): | |
| if isinstance(array, torch.Tensor): | |
| return array.detach().cpu().numpy() | |
| return np.asarray(array) | |
| def to_mask_stack(masks, height, width): | |
| if masks is None: | |
| return np.zeros((0, height, width), dtype=bool) | |
| masks_np = to_numpy(masks) | |
| if masks_np.ndim == 2: | |
| masks_np = masks_np[None, ...] | |
| elif masks_np.ndim == 4 and masks_np.shape[1] == 1: | |
| masks_np = masks_np[:, 0] | |
| elif masks_np.ndim != 3: | |
| return np.zeros((0, height, width), dtype=bool) | |
| if masks_np.dtype != np.bool_: | |
| masks_np = masks_np > 0 | |
| if masks_np.shape[-2:] != (height, width): | |
| masks_np = np.stack( | |
| [ | |
| cv2.resize(mask.astype(np.uint8), (width, height), interpolation=cv2.INTER_NEAREST) > 0 | |
| for mask in masks_np | |
| ], | |
| axis=0, | |
| ) | |
| return masks_np | |
| def union_selected(masks_np, object_ids, selected_ids, height, width): | |
| if masks_np.shape[0] == 0 or not selected_ids: | |
| return np.zeros((height, width), dtype=bool) | |
| selected = {int(i) for i in selected_ids} | |
| rows = [masks_np[i] for i, obj_id in enumerate(object_ids) if int(obj_id) in selected] | |
| if not rows: | |
| return np.zeros((height, width), dtype=bool) | |
| return np.any(np.stack(rows, axis=0), axis=0) | |
| def combine_masks(processed, prompts, height, width): | |
| masks_np = to_mask_stack(processed.get("masks"), height, width) | |
| object_ids = processed.get("object_ids") | |
| if object_ids is None: | |
| object_ids = list(range(masks_np.shape[0])) | |
| else: | |
| object_ids = [int(i) for i in to_numpy(object_ids).reshape(-1)] | |
| if isinstance(prompts, str): | |
| prompts = [prompts] | |
| keep_prompt = prompts[0] | |
| drop_prompts = prompts[1:] | |
| prompt_to_obj_ids = processed.get("prompt_to_obj_ids") or {} | |
| if prompt_to_obj_ids: | |
| keep_ids = list(prompt_to_obj_ids.get(keep_prompt, [])) | |
| drop_ids = [] | |
| for prompt in drop_prompts: | |
| drop_ids.extend(prompt_to_obj_ids.get(prompt, [])) | |
| else: | |
| keep_ids = object_ids[:1] | |
| drop_ids = object_ids[1:] | |
| keep = union_selected(masks_np, object_ids, keep_ids, height, width) | |
| drop = union_selected(masks_np, object_ids, drop_ids, height, width) | |
| return ((keep & ~drop).astype(np.uint8) * 255) | |
| def init_model(model_path, device="cuda", dtype=torch.bfloat16): | |
| print(f"[sam3] init model: {model_path}, device={device}, dtype={dtype}") | |
| model = Sam3VideoModel.from_pretrained(model_path).to(device, dtype=dtype).eval() | |
| processor = Sam3VideoProcessor.from_pretrained(model_path) | |
| return model, processor | |
| def data_process(video_path, num_frames=65): | |
| cap = cv2.VideoCapture(video_path) | |
| if not cap.isOpened(): | |
| raise RuntimeError(f"无法打开视频: {video_path}") | |
| fps = cap.get(cv2.CAP_PROP_FPS) | |
| cap.release() | |
| frames = load_video(video_path) | |
| if not frames: | |
| raise ValueError(f"视频为空: {video_path}") | |
| if num_frames is not None and int(num_frames) >= 0: | |
| frames = frames[: int(num_frames)] | |
| print(f"[data] frames={len(frames)}, size={frames[0].size}, fps={fps}") | |
| return frames, fps | |
| def infer_sam3(model, processor, frames, prompt, device="cuda"): | |
| session = processor.init_video_session( | |
| video=frames, | |
| inference_device=device, | |
| processing_device="cpu", | |
| video_storage_device="cpu", | |
| dtype=model.dtype, | |
| ) | |
| session = processor.add_text_prompt(inference_session=session, text=prompt) | |
| outputs = {} | |
| for result in model.propagate_in_video_iterator( | |
| inference_session=session, | |
| max_frame_num_to_track=len(frames), | |
| ): | |
| processed = processor.postprocess_outputs(session, result) | |
| idx = int(result.frame_idx) | |
| width, height = frames[idx].size | |
| mask = combine_masks(processed, prompt, height, width) | |
| outputs[idx] = Image.fromarray(mask).convert("RGB") | |
| if (idx + 1) % 20 == 0: | |
| print(f"[sam3] frame={idx + 1}/{len(frames)}") | |
| mask_video = [] | |
| for i in range(len(frames)): | |
| if i not in outputs: | |
| width, height = frames[i].size | |
| mask_video.append(Image.new("RGB", (width, height), 0)) | |
| else: | |
| mask_video.append(outputs[i]) | |
| return mask_video | |
| SAM3_PROMPTS = { | |
| "upper_body": ["the upper body of the person","the head of the person"], | |
| "lower_body": ["the lower body of the person","the feet of the person"], | |
| "full_body": ["person","the head of the person","the feet of the person"], | |
| } | |
| def parse_args(): | |
| parser = argparse.ArgumentParser(description="SAM3 region detection") | |
| parser.add_argument("--video", required=True, help="person video path") | |
| parser.add_argument("--output", required=True, help="output directory") | |
| parser.add_argument("--garment_type", default="full_body", choices=["upper_body", "lower_body", "full_body"]) | |
| parser.add_argument("--model_path", default="facebook/sam3") | |
| parser.add_argument("--device", default="cuda") | |
| parser.add_argument("--dtype", default="bfloat16") | |
| parser.add_argument("--num_frames", type=int, default=65, help="max frames to process, -1 for all") | |
| return parser.parse_args() | |
| if __name__ == "__main__": | |
| args = parse_args() | |
| os.makedirs(args.output, exist_ok=True) | |
| save_video = os.path.join(args.output, "sam3_mask.mp4") | |
| model, processor = init_model(args.model_path, device=args.device, dtype=getattr(torch, args.dtype)) | |
| frames, fps = data_process(args.video, num_frames=args.num_frames) | |
| mask_video = infer_sam3( | |
| model, processor, frames, SAM3_PROMPTS[args.garment_type], device=args.device | |
| ) | |
| if os.path.exists(save_video): | |
| os.remove(save_video) | |
| export_to_video(mask_video, save_video, fps=fps) | |
| print(f"[sam3] video={save_video}") | |