PWM / PWM_code /reward_model /scripts /archive /sam3_cotracker_sanity.py
hchen657Naza's picture
Add files using upload-large-folder tool
23e1f91 verified
Raw History Blame Contribute Delete
6.26 kB
"""
Sanity test: SAM3 (text prompt) -> object mask propagation -> CoTracker3
on a Stack-Can demo. Saves overlays so you can eyeball results.
Usage:
HF_HOME=/data0/hongyi/models/hf_cache \
CUDA_VISIBLE_DEVICES=0 python sam3_cotracker_sanity.py \
--video /home/hongyi/PWM/external/DiffSynth-Studio/data/robot/Stack-Can/table2_failure_ep000.mp4 \
--prompt "the white can" \
--output_dir /home/hongyi/PWM/external/sam3_test_out
"""
import argparse, os, json
from pathlib import Path
import cv2
import numpy as np
import torch
def load_video_frames(video_path, max_frames=None):
cap = cv2.VideoCapture(video_path)
frames = []
while True:
ret, f = cap.read()
if not ret: break
frames.append(cv2.cvtColor(f, cv2.COLOR_BGR2RGB))
if max_frames and len(frames) >= max_frames: break
cap.release()
return frames
def run_sam3(video_path, text_prompt):
import sam3.model_builder as _mb
from sam3.model_builder import build_sam3_video_predictor
bpe_path = str(Path(_mb.__file__).parent / "assets" / "bpe_simple_vocab_16e6.txt.gz")
print(f"[SAM3] Building predictor ...")
predictor = build_sam3_video_predictor(
bpe_path=bpe_path,
gpus_to_use=[0],
apply_temporal_disambiguation=False,
)
print(f"[SAM3] Starting session for: {video_path}")
resp = predictor.handle_request(request=dict(type="start_session", resource_path=video_path))
sid = resp["session_id"]
print(f"[SAM3] Adding text prompt: '{text_prompt}' on frame 0 ...")
add_resp = predictor.handle_request(request=dict(
type="add_prompt", session_id=sid, frame_index=0, text=text_prompt,
))
f0 = add_resp.get("outputs", {})
obj_ids = f0.get("out_obj_ids", np.array([]))
probs = f0.get("out_probs", np.array([]))
print(f"[SAM3] frame-0 detections: {len(obj_ids)} objects, scores={probs.tolist() if len(obj_ids) else []}")
print(f"[SAM3] Propagating ...")
raw = {}
for r in predictor.handle_stream_request(
request=dict(type="propagate_in_video", session_id=sid)):
raw[r["frame_index"]] = r["outputs"]
predictor.handle_request(request=dict(type="close_session", session_id=sid))
masks_per_frame = {}
for fi, out in raw.items():
bm = out["out_binary_masks"]
if bm.shape[0] == 0:
masks_per_frame[fi] = np.zeros(bm.shape[1:], dtype=bool)
else:
masks_per_frame[fi] = bm.any(axis=0)
n_nonempty = sum(1 for m in masks_per_frame.values() if m.any())
print(f"[SAM3] Propagated to {len(masks_per_frame)} frames ({n_nonempty} with non-empty mask)")
return masks_per_frame
def run_cotracker(frames, mask, point_stride=8, ckpt=None):
from cotracker.predictor import CoTrackerPredictor
if ckpt is None:
ckpt = "/home/hongyi/PWM/external/co-tracker/checkpoints/scaled_online.pth"
H, W = mask.shape
ys, xs = np.where(mask)
sub = (ys % point_stride == 0) & (xs % point_stride == 0)
ys, xs = ys[sub], xs[sub]
if len(xs) == 0:
raise ValueError("No mask pixels.")
print(f"[CoTracker] {len(xs)} query points (mask had {mask.sum()} px)")
queries = np.stack([np.zeros_like(xs), xs, ys], axis=1).astype(np.float32) # (N, 3) = (frame_id, x, y)
queries_t = torch.from_numpy(queries[None]).cuda()
video = np.stack(frames, axis=0) # (T, H, W, 3)
video_t = torch.from_numpy(video).permute(0, 3, 1, 2).float()[None].cuda() # (1, T, 3, H, W)
predictor = CoTrackerPredictor(checkpoint=ckpt)
predictor = predictor.cuda()
print(f"[CoTracker] Tracking {video_t.shape[1]} frames ...")
with torch.no_grad():
tracks, vis = predictor(video_t, queries=queries_t)
return tracks[0].cpu().numpy(), vis[0].cpu().numpy()
def overlay_masks(frames, masks_per_frame, out_path):
from imageio import mimwrite
out = []
for i, f in enumerate(frames):
ov = f.copy()
if i in masks_per_frame:
m = masks_per_frame[i]
ov[m] = (0.5 * ov[m] + 0.5 * np.array([255, 0, 0])).astype(np.uint8)
out.append(ov)
mimwrite(out_path, out, fps=15, macro_block_size=1)
print(f" Saved {out_path}")
def overlay_tracks(frames, tracks, vis, out_path, stride=4):
from imageio import mimwrite
out = []
for ti in range(len(frames)):
f = frames[ti].copy()
if ti < tracks.shape[0]:
for pt_idx in range(0, tracks.shape[1], stride):
if vis[ti, pt_idx]:
x, y = tracks[ti, pt_idx]
cv2.circle(f, (int(x), int(y)), 2, (0, 255, 0), -1)
out.append(f)
mimwrite(out_path, out, fps=15, macro_block_size=1)
print(f" Saved {out_path}")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--video", required=True)
ap.add_argument("--prompt", default="the white can")
ap.add_argument("--output_dir", required=True)
ap.add_argument("--max_frames", type=int, default=60)
args = ap.parse_args()
os.makedirs(args.output_dir, exist_ok=True)
print(f"Loading {args.video} (up to {args.max_frames} frames) ...")
frames = load_video_frames(args.video, max_frames=args.max_frames)
print(f" {len(frames)} frames, shape {frames[0].shape}")
# Truncate video to max_frames so SAM3 sees the same clip
short_path = os.path.join(args.output_dir, "input_short.mp4")
from imageio import mimwrite
mimwrite(short_path, frames, fps=15, macro_block_size=1)
masks = run_sam3(short_path, args.prompt)
if not masks:
print("SAM3 returned no masks — try a different prompt.")
return
overlay_masks(frames, masks, os.path.join(args.output_dir, "sam3_overlay.mp4"))
# Use frame-0 mask as CoTracker query basis
init_mask = masks.get(0)
if init_mask is None:
init_mask = next(iter(masks.values()))
tracks, vis = run_cotracker(frames, init_mask)
np.savez(os.path.join(args.output_dir, "tracks.npz"), tracks=tracks, vis=vis,
init_mask=init_mask)
overlay_tracks(frames, tracks, vis, os.path.join(args.output_dir, "tracks_overlay.mp4"))
print(f"\nDone. Outputs in {args.output_dir}")
if __name__ == "__main__":
main()