import spaces # must come before torch / diffusers import json import os import tempfile import time import gradio as gr import numpy as np import torch from accelerate import init_empty_weights from huggingface_hub import hf_hub_download, snapshot_download from PIL import Image from safetensors.torch import load_file from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler from transformers import CLIPImageProcessor, CLIPVisionModel from doppel import data_multi as dm from doppel.checkpoint import remap_state_dict from doppel.data import frames_to_uint8, read_all_frames, resize_frames, subsample_frames, write_video from doppel.model import RenderConditionedWanI2VPipeline from doppel.model_redar import RenderActionWanTransformer # --------------------------------------------------------------------------- config WAN_ID = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" WAN_REV = "b184e23a8a16b20f108f727c902e769e873ffc73" # revision Doppel was trained from DOPPEL_ID = "sourlreapwr/doppel-emb8" DTYPE = torch.bfloat16 H, W, T = 240, 432, 17 FP32_KEYS = ("time_embedder", "time_proj", "render_action_adapter", "scale_shift_table", "norm_out") ROBOT_TYPES = { "panda": "Franka Panda (DROID) — 8-D: 7 joints, gripper", "widowx": "WidowX 250 (Bridge) — 8-D: 6 joints, 0, gripper", "ur5e_2f85": "UR5e + 2F-85 (RoboMIND) — 8-D: 6 joints, 0, gripper", "yam_bimanual": "Bimanual YAM (ABC) — 14-D", "vega1_sharpa": "Vega-1 humanoid + Sharpa hands (T-Rex) — 58-D", "mano_hands": "Two MANO hands (GigaHands) — 108-D", "smplx_human": "SMPL-X human (OakInk2) — 159-D", "smplh_human": "SMPL-H human (Ego-Exo4D) — 159-D", } # --------------------------------------------------------------------------- weights doppel_dir = snapshot_download(DOPPEL_ID) wan_dir = snapshot_download( WAN_ID, revision=WAN_REV, allow_patterns=[ "model_index.json", "transformer/config.json", "transformer/diffusion_pytorch_model-00001-of-00014.safetensors", # non-block tensors "vae/*", "image_encoder/*", "image_processor/*", "scheduler/*", ], ) with open(os.path.join(doppel_dir, "config.json")) as f: DOPPEL_CFG = json.load(f) def build_transformer() -> RenderActionWanTransformer: """Wan 2.1 I2V DiT + render-action adapter, filled from the Doppel overlay (99.5% of the weights) and the few untouched base tensors (patch/time/text/image embedders).""" with open(os.path.join(wan_dir, "transformer", "config.json")) as f: tcfg = json.load(f) with init_empty_weights(include_buffers=False): # RoPE buffers stay real model = RenderActionWanTransformer.from_config( tcfg, redar_adapter_kwargs=DOPPEL_CFG["architecture"]["v4_adapter_kwargs"] ) params = dict(model.named_parameters()) def target_dtype(name): return torch.float32 if any(k in name for k in FP32_KEYS) else DTYPE loaded = {} # Base tensors not in the overlay (shard 1 holds all non-block tensors). base = load_file(os.path.join(wan_dir, "transformer", "diffusion_pytorch_model-00001-of-00014.safetensors")) for k, v in base.items(): if k in params and not k.startswith("blocks."): loaded[k] = v.to(target_dtype(k)) del base model.load_state_dict(loaded, strict=False, assign=True) n = len(loaded) # Doppel overlay (fine-tuned blocks, time_proj, proj_out, adapter), stored in inference dtypes. with open(os.path.join(doppel_dir, "transformer_overlay", "model.safetensors.index.json")) as f: shards = sorted(set(json.load(f)["weight_map"].values())) for shard in shards: sd, _ = remap_state_dict(load_file(os.path.join(doppel_dir, "transformer_overlay", shard))) sd = {k: v.to(target_dtype(k)) for k, v in sd.items()} unexpected = model.load_state_dict(sd, strict=False, assign=True).unexpected_keys if unexpected: raise RuntimeError(f"unexpected keys in overlay: {unexpected[:5]}") n += len(sd) del sd still_meta = [k for k, p in model.named_parameters() if p.is_meta] if still_meta: raise RuntimeError(f"{len(still_meta)} parameters were not loaded, e.g. {still_meta[:5]}") print(f"[doppel] transformer loaded: {n} tensors", flush=True) return model.eval() transformer = build_transformer() vae = AutoencoderKLWan.from_pretrained(wan_dir, subfolder="vae", torch_dtype=DTYPE) image_encoder = CLIPVisionModel.from_pretrained(wan_dir, subfolder="image_encoder", torch_dtype=torch.float32) image_processor = CLIPImageProcessor.from_pretrained(wan_dir, subfolder="image_processor") scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(wan_dir, subfolder="scheduler") # Doppel ignores text (trained with an empty prompt): the UMT5 embedding of "" is precomputed # offline with the same Wan text encoder, so the 11 GB text encoder is not loaded at all. EMPTY_TOKEN = torch.load(os.path.join(os.path.dirname(__file__), "empty_prompt_embeds.pt")) # (1, 1, 4096) PROMPT_EMBEDS = torch.cat([EMPTY_TOKEN, EMPTY_TOKEN.new_zeros(1, 511, EMPTY_TOKEN.shape[-1])], dim=1) pipe = RenderConditionedWanI2VPipeline( tokenizer=None, text_encoder=None, vae=vae, scheduler=scheduler, image_processor=image_processor, image_encoder=image_encoder, transformer=transformer, ) pipe.vae.enable_tiling() pipe.vae.enable_slicing() pipe.to("cuda") STATS = dm.load_stats_map( {rt: os.path.join(dm.ASSETS_DIR, "action_stats", f"{rt}.json") for rt in ROBOT_TYPES} ) ACTION_PAD = int(DOPPEL_CFG["inputs"]["action_pad_dim"]) # --------------------------------------------------------------------------- helpers def load_actions(path: str): """(17, D) raw states from .json (list of lists or {"state": ...}), .npz (key 'state') or .npy.""" if path.endswith(".npz"): z = np.load(path) return np.asarray(z["state"], np.float32), None if path.endswith(".npy"): return np.asarray(np.load(path), np.float32), None with open(path) as f: blob = json.load(f) if isinstance(blob, dict): return np.asarray(blob["state"], np.float32), blob.get("robot_type") return np.asarray(blob, np.float32), None def prepare_actions(actions: np.ndarray, robot_type: str) -> torch.Tensor: if actions.ndim != 2: raise gr.Error(f"Actions must be a (T, D) array, got shape {actions.shape}.") if actions.shape[0] != T: # resample onto the 17-frame grid idx = np.linspace(0, actions.shape[0] - 1, T).round().astype(int) actions = actions[idx] D = dm.action_dim(robot_type) if actions.shape[1] != D: raise gr.Error(f"{robot_type} expects {D}-D actions per frame, got {actions.shape[1]}.") a, m = dm.normalize_actions(actions, robot_type, STATS) a, _ = dm.pad_actions(a, m, ACTION_PAD) return torch.from_numpy(np.ascontiguousarray(a)).unsqueeze(0) def _duration(first_frame, render_video, actions_file, robot_type="panda", seed=0, steps=50, *a, **k): # measured on ZeroGPU: 38.6 s for 50 steps (incl. VAE) -> ~15% margin return int(10 + 0.75 * int(steps)) # --------------------------------------------------------------------------- inference @spaces.GPU(duration=_duration) def predict( first_frame: Image.Image, render_video: str, actions_file: str, robot_type: str = "panda", seed: int = 0, steps: int = 50, progress=gr.Progress(track_tqdm=True), ): """Predict the next 17 frames of a scene from its first frame, a camera-aligned rendering of the embodiment along a trajectory, and the trajectory's raw per-frame states. Args: first_frame: RGB first camera frame of the clip. render_video: Video (>= 17 frames) of the embodiment rendered from the same camera, black background. actions_file: .json / .npz / .npy file with the raw (17, D) per-frame states ("state" key for npz/json dict). robot_type: Embodiment of the actions (panda, widowx, ur5e_2f85, yam_bimanual, vega1_sharpa, mano_hands, smplx_human, smplh_human). seed: Random seed. steps: Number of flow-matching Euler steps (paper: 50). Returns: The predicted 17-frame video (frame 0 = input frame) and a side-by-side [render | prediction] video. """ if first_frame is None or render_video is None or actions_file is None: raise gr.Error("Please provide a first frame, a render video and an actions file.") actions_path = actions_file if isinstance(actions_file, str) else actions_file.name raw, _ = load_actions(actions_path) acts = prepare_actions(raw, robot_type).to("cuda", DTYPE) image = resize_frames([first_frame.convert("RGB")], H, W)[0] render = resize_frames(subsample_frames(read_all_frames(render_video), T, render_video), H, W) t0 = time.time() gen = torch.Generator(device="cuda").manual_seed(int(seed)) with torch.inference_mode(), torch.autocast(device_type="cuda", dtype=DTYPE): out = pipe( image=image, prompt=None, negative_prompt=None, prompt_embeds=PROMPT_EMBEDS.to("cuda", DTYPE), render_video=render, actions=acts, num_frames=T, height=H, width=W, num_inference_steps=int(steps), guidance_scale=1.0, max_sequence_length=512, generator=gen, output_type="np", return_dict=False, ) pred = frames_to_uint8(out[0])[:T] print(f"[doppel] {robot_type} steps={steps} seed={seed}: {time.time() - t0:.1f}s", flush=True) tmp = tempfile.mkdtemp() pred_path = os.path.join(tmp, "doppel_prediction.mp4") side_path = os.path.join(tmp, "render_vs_prediction.mp4") write_video(pred_path, pred, fps=8) render_np = np.stack([np.asarray(f) for f in render]) write_video(side_path, np.concatenate([render_np, pred], axis=2), fps=8) return pred_path, side_path # --------------------------------------------------------------------------- UI CSS = """ #col-container { max-width: 1200px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks() as demo: with gr.Column(elem_id="col-container"): gr.Markdown( """ # Doppel: Render the Robot, Generate the World **One action-conditioned video world model across eight embodiments** (Wan 2.1 I2V 14B + render-action adapter). Give it the first camera frame, a rendering of the robot/body along a trajectory (seen from the same camera, black background) and the raw per-frame states — it predicts how the embodiment **and the scene** evolve (17 frames, 240×432). [Paper](https://huggingface.co/papers/2611.24925) · [Project page](https://doppelwm.github.io) · [Code](https://github.com/DoppelWM/Doppel) · [Model](https://huggingface.co/sourlreapwr/doppel-emb8) """ ) with gr.Row(): with gr.Column(): first_frame = gr.Image(label="First camera frame", type="pil", height=260) render_video = gr.Video(label="Embodiment rendering (≥17 frames, same camera)", height=260) with gr.Row(): actions_file = gr.File( label="Actions: raw (17, D) states (.json / .npz / .npy)", file_types=[".json", ".npz", ".npy"], ) robot_type = gr.Dropdown( choices=[(v, k) for k, v in ROBOT_TYPES.items()], value="panda", label="Embodiment", ) run = gr.Button("Predict the future", variant="primary") with gr.Column(): pred_video = gr.Video(label="Doppel prediction", autoplay=True, loop=True, height=260) side_video = gr.Video(label="Rendering (input) | Prediction", autoplay=True, loop=True, height=260) with gr.Accordion("Advanced settings", open=False): seed = gr.Number(label="Seed", value=0, precision=0) steps = gr.Slider(label="Euler steps", minimum=10, maximum=50, step=1, value=50) gr.Markdown( "Actions are raw per-frame states (not deltas); grippers are closures in [0, 1]. " "Layouts per embodiment are listed in the [Doppel README](https://github.com/DoppelWM/Doppel#data-format). " "The rendering must be aligned with the camera of the first frame — misaligned renders degrade predictions. " "Human embodiments require SMPL-H / SMPL-X / MANO renders you produce yourself." ) inputs = [first_frame, render_video, actions_file, robot_type] gr.Examples( examples=[ ["examples/droid_frame0.png", "examples/droid_render.mp4", "examples/droid_actions.json", "panda"], ["examples/bridge_frame0.png", "examples/bridge_render.mp4", "examples/bridge_actions.json", "widowx"], ["examples/robomind_frame0.png", "examples/robomind_render.mp4", "examples/robomind_actions.json", "ur5e_2f85"], ], inputs=inputs, outputs=[pred_video, side_video], fn=predict, cache_examples=True, cache_mode="lazy", label="Held-out clips from the Doppel project page (DROID / Bridge / RoboMIND)", ) run.click(predict, inputs=inputs + [seed, steps], outputs=[pred_video, side_video]) if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)