Instructions to use hchen657Naza/PWM with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use hchen657Naza/PWM with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("hchen657Naza/PWM", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
Download PWM_code/reward_model/scripts/archive/sam3_cotracker_sanity.py from hchen657Naza/PWM: direct link, hf CLI and curl.
- Browser
- Download file 6.26 kB
-
https://huggingface.co/hchen657Naza/PWM/resolve/main/PWM_code/reward_model/scripts/archive/sam3_cotracker_sanity.py
- Command line
-
hf download hf://hchen657Naza/PWM/PWM_code/reward_model/scripts/archive/sam3_cotracker_sanity.py
-
curl -L -o sam3_cotracker_sanity.py https://huggingface.co/hchen657Naza/PWM/resolve/main/PWM_code/reward_model/scripts/archive/sam3_cotracker_sanity.py
6.26 kB
| """ | |
| Sanity test: SAM3 (text prompt) -> object mask propagation -> CoTracker3 | |
| on a Stack-Can demo. Saves overlays so you can eyeball results. | |
| Usage: | |
| HF_HOME=/data0/hongyi/models/hf_cache \ | |
| CUDA_VISIBLE_DEVICES=0 python sam3_cotracker_sanity.py \ | |
| --video /home/hongyi/PWM/external/DiffSynth-Studio/data/robot/Stack-Can/table2_failure_ep000.mp4 \ | |
| --prompt "the white can" \ | |
| --output_dir /home/hongyi/PWM/external/sam3_test_out | |
| """ | |
| import argparse, os, json | |
| from pathlib import Path | |
| import cv2 | |
| import numpy as np | |
| import torch | |
| def load_video_frames(video_path, max_frames=None): | |
| cap = cv2.VideoCapture(video_path) | |
| frames = [] | |
| while True: | |
| ret, f = cap.read() | |
| if not ret: break | |
| frames.append(cv2.cvtColor(f, cv2.COLOR_BGR2RGB)) | |
| if max_frames and len(frames) >= max_frames: break | |
| cap.release() | |
| return frames | |
| def run_sam3(video_path, text_prompt): | |
| import sam3.model_builder as _mb | |
| from sam3.model_builder import build_sam3_video_predictor | |
| bpe_path = str(Path(_mb.__file__).parent / "assets" / "bpe_simple_vocab_16e6.txt.gz") | |
| print(f"[SAM3] Building predictor ...") | |
| predictor = build_sam3_video_predictor( | |
| bpe_path=bpe_path, | |
| gpus_to_use=[0], | |
| apply_temporal_disambiguation=False, | |
| ) | |
| print(f"[SAM3] Starting session for: {video_path}") | |
| resp = predictor.handle_request(request=dict(type="start_session", resource_path=video_path)) | |
| sid = resp["session_id"] | |
| print(f"[SAM3] Adding text prompt: '{text_prompt}' on frame 0 ...") | |
| add_resp = predictor.handle_request(request=dict( | |
| type="add_prompt", session_id=sid, frame_index=0, text=text_prompt, | |
| )) | |
| f0 = add_resp.get("outputs", {}) | |
| obj_ids = f0.get("out_obj_ids", np.array([])) | |
| probs = f0.get("out_probs", np.array([])) | |
| print(f"[SAM3] frame-0 detections: {len(obj_ids)} objects, scores={probs.tolist() if len(obj_ids) else []}") | |
| print(f"[SAM3] Propagating ...") | |
| raw = {} | |
| for r in predictor.handle_stream_request( | |
| request=dict(type="propagate_in_video", session_id=sid)): | |
| raw[r["frame_index"]] = r["outputs"] | |
| predictor.handle_request(request=dict(type="close_session", session_id=sid)) | |
| masks_per_frame = {} | |
| for fi, out in raw.items(): | |
| bm = out["out_binary_masks"] | |
| if bm.shape[0] == 0: | |
| masks_per_frame[fi] = np.zeros(bm.shape[1:], dtype=bool) | |
| else: | |
| masks_per_frame[fi] = bm.any(axis=0) | |
| n_nonempty = sum(1 for m in masks_per_frame.values() if m.any()) | |
| print(f"[SAM3] Propagated to {len(masks_per_frame)} frames ({n_nonempty} with non-empty mask)") | |
| return masks_per_frame | |
| def run_cotracker(frames, mask, point_stride=8, ckpt=None): | |
| from cotracker.predictor import CoTrackerPredictor | |
| if ckpt is None: | |
| ckpt = "/home/hongyi/PWM/external/co-tracker/checkpoints/scaled_online.pth" | |
| H, W = mask.shape | |
| ys, xs = np.where(mask) | |
| sub = (ys % point_stride == 0) & (xs % point_stride == 0) | |
| ys, xs = ys[sub], xs[sub] | |
| if len(xs) == 0: | |
| raise ValueError("No mask pixels.") | |
| print(f"[CoTracker] {len(xs)} query points (mask had {mask.sum()} px)") | |
| queries = np.stack([np.zeros_like(xs), xs, ys], axis=1).astype(np.float32) # (N, 3) = (frame_id, x, y) | |
| queries_t = torch.from_numpy(queries[None]).cuda() | |
| video = np.stack(frames, axis=0) # (T, H, W, 3) | |
| video_t = torch.from_numpy(video).permute(0, 3, 1, 2).float()[None].cuda() # (1, T, 3, H, W) | |
| predictor = CoTrackerPredictor(checkpoint=ckpt) | |
| predictor = predictor.cuda() | |
| print(f"[CoTracker] Tracking {video_t.shape[1]} frames ...") | |
| with torch.no_grad(): | |
| tracks, vis = predictor(video_t, queries=queries_t) | |
| return tracks[0].cpu().numpy(), vis[0].cpu().numpy() | |
| def overlay_masks(frames, masks_per_frame, out_path): | |
| from imageio import mimwrite | |
| out = [] | |
| for i, f in enumerate(frames): | |
| ov = f.copy() | |
| if i in masks_per_frame: | |
| m = masks_per_frame[i] | |
| ov[m] = (0.5 * ov[m] + 0.5 * np.array([255, 0, 0])).astype(np.uint8) | |
| out.append(ov) | |
| mimwrite(out_path, out, fps=15, macro_block_size=1) | |
| print(f" Saved {out_path}") | |
| def overlay_tracks(frames, tracks, vis, out_path, stride=4): | |
| from imageio import mimwrite | |
| out = [] | |
| for ti in range(len(frames)): | |
| f = frames[ti].copy() | |
| if ti < tracks.shape[0]: | |
| for pt_idx in range(0, tracks.shape[1], stride): | |
| if vis[ti, pt_idx]: | |
| x, y = tracks[ti, pt_idx] | |
| cv2.circle(f, (int(x), int(y)), 2, (0, 255, 0), -1) | |
| out.append(f) | |
| mimwrite(out_path, out, fps=15, macro_block_size=1) | |
| print(f" Saved {out_path}") | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--video", required=True) | |
| ap.add_argument("--prompt", default="the white can") | |
| ap.add_argument("--output_dir", required=True) | |
| ap.add_argument("--max_frames", type=int, default=60) | |
| args = ap.parse_args() | |
| os.makedirs(args.output_dir, exist_ok=True) | |
| print(f"Loading {args.video} (up to {args.max_frames} frames) ...") | |
| frames = load_video_frames(args.video, max_frames=args.max_frames) | |
| print(f" {len(frames)} frames, shape {frames[0].shape}") | |
| # Truncate video to max_frames so SAM3 sees the same clip | |
| short_path = os.path.join(args.output_dir, "input_short.mp4") | |
| from imageio import mimwrite | |
| mimwrite(short_path, frames, fps=15, macro_block_size=1) | |
| masks = run_sam3(short_path, args.prompt) | |
| if not masks: | |
| print("SAM3 returned no masks — try a different prompt.") | |
| return | |
| overlay_masks(frames, masks, os.path.join(args.output_dir, "sam3_overlay.mp4")) | |
| # Use frame-0 mask as CoTracker query basis | |
| init_mask = masks.get(0) | |
| if init_mask is None: | |
| init_mask = next(iter(masks.values())) | |
| tracks, vis = run_cotracker(frames, init_mask) | |
| np.savez(os.path.join(args.output_dir, "tracks.npz"), tracks=tracks, vis=vis, | |
| init_mask=init_mask) | |
| overlay_tracks(frames, tracks, vis, os.path.join(args.output_dir, "tracks_overlay.mp4")) | |
| print(f"\nDone. Outputs in {args.output_dir}") | |
| if __name__ == "__main__": | |
| main() | |