""" Sanity test: SAM3 (text prompt) -> object mask propagation -> CoTracker3 on a Stack-Can demo. Saves overlays so you can eyeball results. Usage: HF_HOME=/data0/hongyi/models/hf_cache \ CUDA_VISIBLE_DEVICES=0 python sam3_cotracker_sanity.py \ --video /home/hongyi/PWM/external/DiffSynth-Studio/data/robot/Stack-Can/table2_failure_ep000.mp4 \ --prompt "the white can" \ --output_dir /home/hongyi/PWM/external/sam3_test_out """ import argparse, os, json from pathlib import Path import cv2 import numpy as np import torch def load_video_frames(video_path, max_frames=None): cap = cv2.VideoCapture(video_path) frames = [] while True: ret, f = cap.read() if not ret: break frames.append(cv2.cvtColor(f, cv2.COLOR_BGR2RGB)) if max_frames and len(frames) >= max_frames: break cap.release() return frames def run_sam3(video_path, text_prompt): import sam3.model_builder as _mb from sam3.model_builder import build_sam3_video_predictor bpe_path = str(Path(_mb.__file__).parent / "assets" / "bpe_simple_vocab_16e6.txt.gz") print(f"[SAM3] Building predictor ...") predictor = build_sam3_video_predictor( bpe_path=bpe_path, gpus_to_use=[0], apply_temporal_disambiguation=False, ) print(f"[SAM3] Starting session for: {video_path}") resp = predictor.handle_request(request=dict(type="start_session", resource_path=video_path)) sid = resp["session_id"] print(f"[SAM3] Adding text prompt: '{text_prompt}' on frame 0 ...") add_resp = predictor.handle_request(request=dict( type="add_prompt", session_id=sid, frame_index=0, text=text_prompt, )) f0 = add_resp.get("outputs", {}) obj_ids = f0.get("out_obj_ids", np.array([])) probs = f0.get("out_probs", np.array([])) print(f"[SAM3] frame-0 detections: {len(obj_ids)} objects, scores={probs.tolist() if len(obj_ids) else []}") print(f"[SAM3] Propagating ...") raw = {} for r in predictor.handle_stream_request( request=dict(type="propagate_in_video", session_id=sid)): raw[r["frame_index"]] = r["outputs"] predictor.handle_request(request=dict(type="close_session", session_id=sid)) masks_per_frame = {} for fi, out in raw.items(): bm = out["out_binary_masks"] if bm.shape[0] == 0: masks_per_frame[fi] = np.zeros(bm.shape[1:], dtype=bool) else: masks_per_frame[fi] = bm.any(axis=0) n_nonempty = sum(1 for m in masks_per_frame.values() if m.any()) print(f"[SAM3] Propagated to {len(masks_per_frame)} frames ({n_nonempty} with non-empty mask)") return masks_per_frame def run_cotracker(frames, mask, point_stride=8, ckpt=None): from cotracker.predictor import CoTrackerPredictor if ckpt is None: ckpt = "/home/hongyi/PWM/external/co-tracker/checkpoints/scaled_online.pth" H, W = mask.shape ys, xs = np.where(mask) sub = (ys % point_stride == 0) & (xs % point_stride == 0) ys, xs = ys[sub], xs[sub] if len(xs) == 0: raise ValueError("No mask pixels.") print(f"[CoTracker] {len(xs)} query points (mask had {mask.sum()} px)") queries = np.stack([np.zeros_like(xs), xs, ys], axis=1).astype(np.float32) # (N, 3) = (frame_id, x, y) queries_t = torch.from_numpy(queries[None]).cuda() video = np.stack(frames, axis=0) # (T, H, W, 3) video_t = torch.from_numpy(video).permute(0, 3, 1, 2).float()[None].cuda() # (1, T, 3, H, W) predictor = CoTrackerPredictor(checkpoint=ckpt) predictor = predictor.cuda() print(f"[CoTracker] Tracking {video_t.shape[1]} frames ...") with torch.no_grad(): tracks, vis = predictor(video_t, queries=queries_t) return tracks[0].cpu().numpy(), vis[0].cpu().numpy() def overlay_masks(frames, masks_per_frame, out_path): from imageio import mimwrite out = [] for i, f in enumerate(frames): ov = f.copy() if i in masks_per_frame: m = masks_per_frame[i] ov[m] = (0.5 * ov[m] + 0.5 * np.array([255, 0, 0])).astype(np.uint8) out.append(ov) mimwrite(out_path, out, fps=15, macro_block_size=1) print(f" Saved {out_path}") def overlay_tracks(frames, tracks, vis, out_path, stride=4): from imageio import mimwrite out = [] for ti in range(len(frames)): f = frames[ti].copy() if ti < tracks.shape[0]: for pt_idx in range(0, tracks.shape[1], stride): if vis[ti, pt_idx]: x, y = tracks[ti, pt_idx] cv2.circle(f, (int(x), int(y)), 2, (0, 255, 0), -1) out.append(f) mimwrite(out_path, out, fps=15, macro_block_size=1) print(f" Saved {out_path}") def main(): ap = argparse.ArgumentParser() ap.add_argument("--video", required=True) ap.add_argument("--prompt", default="the white can") ap.add_argument("--output_dir", required=True) ap.add_argument("--max_frames", type=int, default=60) args = ap.parse_args() os.makedirs(args.output_dir, exist_ok=True) print(f"Loading {args.video} (up to {args.max_frames} frames) ...") frames = load_video_frames(args.video, max_frames=args.max_frames) print(f" {len(frames)} frames, shape {frames[0].shape}") # Truncate video to max_frames so SAM3 sees the same clip short_path = os.path.join(args.output_dir, "input_short.mp4") from imageio import mimwrite mimwrite(short_path, frames, fps=15, macro_block_size=1) masks = run_sam3(short_path, args.prompt) if not masks: print("SAM3 returned no masks — try a different prompt.") return overlay_masks(frames, masks, os.path.join(args.output_dir, "sam3_overlay.mp4")) # Use frame-0 mask as CoTracker query basis init_mask = masks.get(0) if init_mask is None: init_mask = next(iter(masks.values())) tracks, vis = run_cotracker(frames, init_mask) np.savez(os.path.join(args.output_dir, "tracks.npz"), tracks=tracks, vis=vis, init_mask=init_mask) overlay_tracks(frames, tracks, vis, os.path.join(args.output_dir, "tracks_overlay.mp4")) print(f"\nDone. Outputs in {args.output_dir}") if __name__ == "__main__": main()