Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/doppel-world-model: direct link, hf CLI and curl.
- Browser
- Download file 13.7 kB
-
https://huggingface.co/spaces/hugging-apps/doppel-world-model/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/doppel-world-model/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/doppel-world-model/resolve/main/app.py
13.7 kB
| import spaces # must come before torch / diffusers | |
| import json | |
| import os | |
| import tempfile | |
| import time | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| from accelerate import init_empty_weights | |
| from huggingface_hub import hf_hub_download, snapshot_download | |
| from PIL import Image | |
| from safetensors.torch import load_file | |
| from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler | |
| from transformers import CLIPImageProcessor, CLIPVisionModel | |
| from doppel import data_multi as dm | |
| from doppel.checkpoint import remap_state_dict | |
| from doppel.data import frames_to_uint8, read_all_frames, resize_frames, subsample_frames, write_video | |
| from doppel.model import RenderConditionedWanI2VPipeline | |
| from doppel.model_redar import RenderActionWanTransformer | |
| # --------------------------------------------------------------------------- config | |
| WAN_ID = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" | |
| WAN_REV = "b184e23a8a16b20f108f727c902e769e873ffc73" # revision Doppel was trained from | |
| DOPPEL_ID = "sourlreapwr/doppel-emb8" | |
| DTYPE = torch.bfloat16 | |
| H, W, T = 240, 432, 17 | |
| FP32_KEYS = ("time_embedder", "time_proj", "render_action_adapter", "scale_shift_table", "norm_out") | |
| ROBOT_TYPES = { | |
| "panda": "Franka Panda (DROID) β 8-D: 7 joints, gripper", | |
| "widowx": "WidowX 250 (Bridge) β 8-D: 6 joints, 0, gripper", | |
| "ur5e_2f85": "UR5e + 2F-85 (RoboMIND) β 8-D: 6 joints, 0, gripper", | |
| "yam_bimanual": "Bimanual YAM (ABC) β 14-D", | |
| "vega1_sharpa": "Vega-1 humanoid + Sharpa hands (T-Rex) β 58-D", | |
| "mano_hands": "Two MANO hands (GigaHands) β 108-D", | |
| "smplx_human": "SMPL-X human (OakInk2) β 159-D", | |
| "smplh_human": "SMPL-H human (Ego-Exo4D) β 159-D", | |
| } | |
| # --------------------------------------------------------------------------- weights | |
| doppel_dir = snapshot_download(DOPPEL_ID) | |
| wan_dir = snapshot_download( | |
| WAN_ID, | |
| revision=WAN_REV, | |
| allow_patterns=[ | |
| "model_index.json", | |
| "transformer/config.json", | |
| "transformer/diffusion_pytorch_model-00001-of-00014.safetensors", # non-block tensors | |
| "vae/*", | |
| "image_encoder/*", | |
| "image_processor/*", | |
| "scheduler/*", | |
| ], | |
| ) | |
| with open(os.path.join(doppel_dir, "config.json")) as f: | |
| DOPPEL_CFG = json.load(f) | |
| def build_transformer() -> RenderActionWanTransformer: | |
| """Wan 2.1 I2V DiT + render-action adapter, filled from the Doppel overlay (99.5% of the | |
| weights) and the few untouched base tensors (patch/time/text/image embedders).""" | |
| with open(os.path.join(wan_dir, "transformer", "config.json")) as f: | |
| tcfg = json.load(f) | |
| with init_empty_weights(include_buffers=False): # RoPE buffers stay real | |
| model = RenderActionWanTransformer.from_config( | |
| tcfg, redar_adapter_kwargs=DOPPEL_CFG["architecture"]["v4_adapter_kwargs"] | |
| ) | |
| params = dict(model.named_parameters()) | |
| def target_dtype(name): | |
| return torch.float32 if any(k in name for k in FP32_KEYS) else DTYPE | |
| loaded = {} | |
| # Base tensors not in the overlay (shard 1 holds all non-block tensors). | |
| base = load_file(os.path.join(wan_dir, "transformer", "diffusion_pytorch_model-00001-of-00014.safetensors")) | |
| for k, v in base.items(): | |
| if k in params and not k.startswith("blocks."): | |
| loaded[k] = v.to(target_dtype(k)) | |
| del base | |
| model.load_state_dict(loaded, strict=False, assign=True) | |
| n = len(loaded) | |
| # Doppel overlay (fine-tuned blocks, time_proj, proj_out, adapter), stored in inference dtypes. | |
| with open(os.path.join(doppel_dir, "transformer_overlay", "model.safetensors.index.json")) as f: | |
| shards = sorted(set(json.load(f)["weight_map"].values())) | |
| for shard in shards: | |
| sd, _ = remap_state_dict(load_file(os.path.join(doppel_dir, "transformer_overlay", shard))) | |
| sd = {k: v.to(target_dtype(k)) for k, v in sd.items()} | |
| unexpected = model.load_state_dict(sd, strict=False, assign=True).unexpected_keys | |
| if unexpected: | |
| raise RuntimeError(f"unexpected keys in overlay: {unexpected[:5]}") | |
| n += len(sd) | |
| del sd | |
| still_meta = [k for k, p in model.named_parameters() if p.is_meta] | |
| if still_meta: | |
| raise RuntimeError(f"{len(still_meta)} parameters were not loaded, e.g. {still_meta[:5]}") | |
| print(f"[doppel] transformer loaded: {n} tensors", flush=True) | |
| return model.eval() | |
| transformer = build_transformer() | |
| vae = AutoencoderKLWan.from_pretrained(wan_dir, subfolder="vae", torch_dtype=DTYPE) | |
| image_encoder = CLIPVisionModel.from_pretrained(wan_dir, subfolder="image_encoder", torch_dtype=torch.float32) | |
| image_processor = CLIPImageProcessor.from_pretrained(wan_dir, subfolder="image_processor") | |
| scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(wan_dir, subfolder="scheduler") | |
| # Doppel ignores text (trained with an empty prompt): the UMT5 embedding of "" is precomputed | |
| # offline with the same Wan text encoder, so the 11 GB text encoder is not loaded at all. | |
| EMPTY_TOKEN = torch.load(os.path.join(os.path.dirname(__file__), "empty_prompt_embeds.pt")) # (1, 1, 4096) | |
| PROMPT_EMBEDS = torch.cat([EMPTY_TOKEN, EMPTY_TOKEN.new_zeros(1, 511, EMPTY_TOKEN.shape[-1])], dim=1) | |
| pipe = RenderConditionedWanI2VPipeline( | |
| tokenizer=None, | |
| text_encoder=None, | |
| vae=vae, | |
| scheduler=scheduler, | |
| image_processor=image_processor, | |
| image_encoder=image_encoder, | |
| transformer=transformer, | |
| ) | |
| pipe.vae.enable_tiling() | |
| pipe.vae.enable_slicing() | |
| pipe.to("cuda") | |
| STATS = dm.load_stats_map( | |
| {rt: os.path.join(dm.ASSETS_DIR, "action_stats", f"{rt}.json") for rt in ROBOT_TYPES} | |
| ) | |
| ACTION_PAD = int(DOPPEL_CFG["inputs"]["action_pad_dim"]) | |
| # --------------------------------------------------------------------------- helpers | |
| def load_actions(path: str): | |
| """(17, D) raw states from .json (list of lists or {"state": ...}), .npz (key 'state') or .npy.""" | |
| if path.endswith(".npz"): | |
| z = np.load(path) | |
| return np.asarray(z["state"], np.float32), None | |
| if path.endswith(".npy"): | |
| return np.asarray(np.load(path), np.float32), None | |
| with open(path) as f: | |
| blob = json.load(f) | |
| if isinstance(blob, dict): | |
| return np.asarray(blob["state"], np.float32), blob.get("robot_type") | |
| return np.asarray(blob, np.float32), None | |
| def prepare_actions(actions: np.ndarray, robot_type: str) -> torch.Tensor: | |
| if actions.ndim != 2: | |
| raise gr.Error(f"Actions must be a (T, D) array, got shape {actions.shape}.") | |
| if actions.shape[0] != T: # resample onto the 17-frame grid | |
| idx = np.linspace(0, actions.shape[0] - 1, T).round().astype(int) | |
| actions = actions[idx] | |
| D = dm.action_dim(robot_type) | |
| if actions.shape[1] != D: | |
| raise gr.Error(f"{robot_type} expects {D}-D actions per frame, got {actions.shape[1]}.") | |
| a, m = dm.normalize_actions(actions, robot_type, STATS) | |
| a, _ = dm.pad_actions(a, m, ACTION_PAD) | |
| return torch.from_numpy(np.ascontiguousarray(a)).unsqueeze(0) | |
| def _duration(first_frame, render_video, actions_file, robot_type="panda", seed=0, steps=50, *a, **k): | |
| # measured on ZeroGPU: 38.6 s for 50 steps (incl. VAE) -> ~15% margin | |
| return int(10 + 0.75 * int(steps)) | |
| # --------------------------------------------------------------------------- inference | |
| def predict( | |
| first_frame: Image.Image, | |
| render_video: str, | |
| actions_file: str, | |
| robot_type: str = "panda", | |
| seed: int = 0, | |
| steps: int = 50, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """Predict the next 17 frames of a scene from its first frame, a camera-aligned rendering of | |
| the embodiment along a trajectory, and the trajectory's raw per-frame states. | |
| Args: | |
| first_frame: RGB first camera frame of the clip. | |
| render_video: Video (>= 17 frames) of the embodiment rendered from the same camera, black background. | |
| actions_file: .json / .npz / .npy file with the raw (17, D) per-frame states ("state" key for npz/json dict). | |
| robot_type: Embodiment of the actions (panda, widowx, ur5e_2f85, yam_bimanual, vega1_sharpa, mano_hands, smplx_human, smplh_human). | |
| seed: Random seed. | |
| steps: Number of flow-matching Euler steps (paper: 50). | |
| Returns: | |
| The predicted 17-frame video (frame 0 = input frame) and a side-by-side [render | prediction] video. | |
| """ | |
| if first_frame is None or render_video is None or actions_file is None: | |
| raise gr.Error("Please provide a first frame, a render video and an actions file.") | |
| actions_path = actions_file if isinstance(actions_file, str) else actions_file.name | |
| raw, _ = load_actions(actions_path) | |
| acts = prepare_actions(raw, robot_type).to("cuda", DTYPE) | |
| image = resize_frames([first_frame.convert("RGB")], H, W)[0] | |
| render = resize_frames(subsample_frames(read_all_frames(render_video), T, render_video), H, W) | |
| t0 = time.time() | |
| gen = torch.Generator(device="cuda").manual_seed(int(seed)) | |
| with torch.inference_mode(), torch.autocast(device_type="cuda", dtype=DTYPE): | |
| out = pipe( | |
| image=image, | |
| prompt=None, | |
| negative_prompt=None, | |
| prompt_embeds=PROMPT_EMBEDS.to("cuda", DTYPE), | |
| render_video=render, | |
| actions=acts, | |
| num_frames=T, | |
| height=H, | |
| width=W, | |
| num_inference_steps=int(steps), | |
| guidance_scale=1.0, | |
| max_sequence_length=512, | |
| generator=gen, | |
| output_type="np", | |
| return_dict=False, | |
| ) | |
| pred = frames_to_uint8(out[0])[:T] | |
| print(f"[doppel] {robot_type} steps={steps} seed={seed}: {time.time() - t0:.1f}s", flush=True) | |
| tmp = tempfile.mkdtemp() | |
| pred_path = os.path.join(tmp, "doppel_prediction.mp4") | |
| side_path = os.path.join(tmp, "render_vs_prediction.mp4") | |
| write_video(pred_path, pred, fps=8) | |
| render_np = np.stack([np.asarray(f) for f in render]) | |
| write_video(side_path, np.concatenate([render_np, pred], axis=2), fps=8) | |
| return pred_path, side_path | |
| # --------------------------------------------------------------------------- UI | |
| CSS = """ | |
| #col-container { max-width: 1200px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks() as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.Markdown( | |
| """ | |
| # Doppel: Render the Robot, Generate the World | |
| **One action-conditioned video world model across eight embodiments** (Wan 2.1 I2V 14B + render-action adapter). | |
| Give it the first camera frame, a rendering of the robot/body along a trajectory (seen from the same camera, black | |
| background) and the raw per-frame states β it predicts how the embodiment **and the scene** evolve (17 frames, 240Γ432). | |
| [Paper](https://huggingface.co/papers/2611.24925) Β· [Project page](https://doppelwm.github.io) Β· [Code](https://github.com/DoppelWM/Doppel) Β· [Model](https://huggingface.co/sourlreapwr/doppel-emb8) | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| first_frame = gr.Image(label="First camera frame", type="pil", height=260) | |
| render_video = gr.Video(label="Embodiment rendering (β₯17 frames, same camera)", height=260) | |
| with gr.Row(): | |
| actions_file = gr.File( | |
| label="Actions: raw (17, D) states (.json / .npz / .npy)", | |
| file_types=[".json", ".npz", ".npy"], | |
| ) | |
| robot_type = gr.Dropdown( | |
| choices=[(v, k) for k, v in ROBOT_TYPES.items()], | |
| value="panda", | |
| label="Embodiment", | |
| ) | |
| run = gr.Button("Predict the future", variant="primary") | |
| with gr.Column(): | |
| pred_video = gr.Video(label="Doppel prediction", autoplay=True, loop=True, height=260) | |
| side_video = gr.Video(label="Rendering (input) | Prediction", autoplay=True, loop=True, height=260) | |
| with gr.Accordion("Advanced settings", open=False): | |
| seed = gr.Number(label="Seed", value=0, precision=0) | |
| steps = gr.Slider(label="Euler steps", minimum=10, maximum=50, step=1, value=50) | |
| gr.Markdown( | |
| "Actions are raw per-frame states (not deltas); grippers are closures in [0, 1]. " | |
| "Layouts per embodiment are listed in the [Doppel README](https://github.com/DoppelWM/Doppel#data-format). " | |
| "The rendering must be aligned with the camera of the first frame β misaligned renders degrade predictions. " | |
| "Human embodiments require SMPL-H / SMPL-X / MANO renders you produce yourself." | |
| ) | |
| inputs = [first_frame, render_video, actions_file, robot_type] | |
| gr.Examples( | |
| examples=[ | |
| ["examples/droid_frame0.png", "examples/droid_render.mp4", "examples/droid_actions.json", "panda"], | |
| ["examples/bridge_frame0.png", "examples/bridge_render.mp4", "examples/bridge_actions.json", "widowx"], | |
| ["examples/robomind_frame0.png", "examples/robomind_render.mp4", "examples/robomind_actions.json", "ur5e_2f85"], | |
| ], | |
| inputs=inputs, | |
| outputs=[pred_video, side_video], | |
| fn=predict, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Held-out clips from the Doppel project page (DROID / Bridge / RoboMIND)", | |
| ) | |
| run.click(predict, inputs=inputs + [seed, steps], outputs=[pred_video, side_video]) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |