multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
c51026c verified
Raw History Blame Contribute Delete
13.7 kB
import spaces # must come before torch / diffusers
import json
import os
import tempfile
import time
import gradio as gr
import numpy as np
import torch
from accelerate import init_empty_weights
from huggingface_hub import hf_hub_download, snapshot_download
from PIL import Image
from safetensors.torch import load_file
from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
from transformers import CLIPImageProcessor, CLIPVisionModel
from doppel import data_multi as dm
from doppel.checkpoint import remap_state_dict
from doppel.data import frames_to_uint8, read_all_frames, resize_frames, subsample_frames, write_video
from doppel.model import RenderConditionedWanI2VPipeline
from doppel.model_redar import RenderActionWanTransformer
# --------------------------------------------------------------------------- config
WAN_ID = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"
WAN_REV = "b184e23a8a16b20f108f727c902e769e873ffc73" # revision Doppel was trained from
DOPPEL_ID = "sourlreapwr/doppel-emb8"
DTYPE = torch.bfloat16
H, W, T = 240, 432, 17
FP32_KEYS = ("time_embedder", "time_proj", "render_action_adapter", "scale_shift_table", "norm_out")
ROBOT_TYPES = {
"panda": "Franka Panda (DROID) β€” 8-D: 7 joints, gripper",
"widowx": "WidowX 250 (Bridge) β€” 8-D: 6 joints, 0, gripper",
"ur5e_2f85": "UR5e + 2F-85 (RoboMIND) β€” 8-D: 6 joints, 0, gripper",
"yam_bimanual": "Bimanual YAM (ABC) β€” 14-D",
"vega1_sharpa": "Vega-1 humanoid + Sharpa hands (T-Rex) β€” 58-D",
"mano_hands": "Two MANO hands (GigaHands) β€” 108-D",
"smplx_human": "SMPL-X human (OakInk2) β€” 159-D",
"smplh_human": "SMPL-H human (Ego-Exo4D) β€” 159-D",
}
# --------------------------------------------------------------------------- weights
doppel_dir = snapshot_download(DOPPEL_ID)
wan_dir = snapshot_download(
WAN_ID,
revision=WAN_REV,
allow_patterns=[
"model_index.json",
"transformer/config.json",
"transformer/diffusion_pytorch_model-00001-of-00014.safetensors", # non-block tensors
"vae/*",
"image_encoder/*",
"image_processor/*",
"scheduler/*",
],
)
with open(os.path.join(doppel_dir, "config.json")) as f:
DOPPEL_CFG = json.load(f)
def build_transformer() -> RenderActionWanTransformer:
"""Wan 2.1 I2V DiT + render-action adapter, filled from the Doppel overlay (99.5% of the
weights) and the few untouched base tensors (patch/time/text/image embedders)."""
with open(os.path.join(wan_dir, "transformer", "config.json")) as f:
tcfg = json.load(f)
with init_empty_weights(include_buffers=False): # RoPE buffers stay real
model = RenderActionWanTransformer.from_config(
tcfg, redar_adapter_kwargs=DOPPEL_CFG["architecture"]["v4_adapter_kwargs"]
)
params = dict(model.named_parameters())
def target_dtype(name):
return torch.float32 if any(k in name for k in FP32_KEYS) else DTYPE
loaded = {}
# Base tensors not in the overlay (shard 1 holds all non-block tensors).
base = load_file(os.path.join(wan_dir, "transformer", "diffusion_pytorch_model-00001-of-00014.safetensors"))
for k, v in base.items():
if k in params and not k.startswith("blocks."):
loaded[k] = v.to(target_dtype(k))
del base
model.load_state_dict(loaded, strict=False, assign=True)
n = len(loaded)
# Doppel overlay (fine-tuned blocks, time_proj, proj_out, adapter), stored in inference dtypes.
with open(os.path.join(doppel_dir, "transformer_overlay", "model.safetensors.index.json")) as f:
shards = sorted(set(json.load(f)["weight_map"].values()))
for shard in shards:
sd, _ = remap_state_dict(load_file(os.path.join(doppel_dir, "transformer_overlay", shard)))
sd = {k: v.to(target_dtype(k)) for k, v in sd.items()}
unexpected = model.load_state_dict(sd, strict=False, assign=True).unexpected_keys
if unexpected:
raise RuntimeError(f"unexpected keys in overlay: {unexpected[:5]}")
n += len(sd)
del sd
still_meta = [k for k, p in model.named_parameters() if p.is_meta]
if still_meta:
raise RuntimeError(f"{len(still_meta)} parameters were not loaded, e.g. {still_meta[:5]}")
print(f"[doppel] transformer loaded: {n} tensors", flush=True)
return model.eval()
transformer = build_transformer()
vae = AutoencoderKLWan.from_pretrained(wan_dir, subfolder="vae", torch_dtype=DTYPE)
image_encoder = CLIPVisionModel.from_pretrained(wan_dir, subfolder="image_encoder", torch_dtype=torch.float32)
image_processor = CLIPImageProcessor.from_pretrained(wan_dir, subfolder="image_processor")
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(wan_dir, subfolder="scheduler")
# Doppel ignores text (trained with an empty prompt): the UMT5 embedding of "" is precomputed
# offline with the same Wan text encoder, so the 11 GB text encoder is not loaded at all.
EMPTY_TOKEN = torch.load(os.path.join(os.path.dirname(__file__), "empty_prompt_embeds.pt")) # (1, 1, 4096)
PROMPT_EMBEDS = torch.cat([EMPTY_TOKEN, EMPTY_TOKEN.new_zeros(1, 511, EMPTY_TOKEN.shape[-1])], dim=1)
pipe = RenderConditionedWanI2VPipeline(
tokenizer=None,
text_encoder=None,
vae=vae,
scheduler=scheduler,
image_processor=image_processor,
image_encoder=image_encoder,
transformer=transformer,
)
pipe.vae.enable_tiling()
pipe.vae.enable_slicing()
pipe.to("cuda")
STATS = dm.load_stats_map(
{rt: os.path.join(dm.ASSETS_DIR, "action_stats", f"{rt}.json") for rt in ROBOT_TYPES}
)
ACTION_PAD = int(DOPPEL_CFG["inputs"]["action_pad_dim"])
# --------------------------------------------------------------------------- helpers
def load_actions(path: str):
"""(17, D) raw states from .json (list of lists or {"state": ...}), .npz (key 'state') or .npy."""
if path.endswith(".npz"):
z = np.load(path)
return np.asarray(z["state"], np.float32), None
if path.endswith(".npy"):
return np.asarray(np.load(path), np.float32), None
with open(path) as f:
blob = json.load(f)
if isinstance(blob, dict):
return np.asarray(blob["state"], np.float32), blob.get("robot_type")
return np.asarray(blob, np.float32), None
def prepare_actions(actions: np.ndarray, robot_type: str) -> torch.Tensor:
if actions.ndim != 2:
raise gr.Error(f"Actions must be a (T, D) array, got shape {actions.shape}.")
if actions.shape[0] != T: # resample onto the 17-frame grid
idx = np.linspace(0, actions.shape[0] - 1, T).round().astype(int)
actions = actions[idx]
D = dm.action_dim(robot_type)
if actions.shape[1] != D:
raise gr.Error(f"{robot_type} expects {D}-D actions per frame, got {actions.shape[1]}.")
a, m = dm.normalize_actions(actions, robot_type, STATS)
a, _ = dm.pad_actions(a, m, ACTION_PAD)
return torch.from_numpy(np.ascontiguousarray(a)).unsqueeze(0)
def _duration(first_frame, render_video, actions_file, robot_type="panda", seed=0, steps=50, *a, **k):
# measured on ZeroGPU: 38.6 s for 50 steps (incl. VAE) -> ~15% margin
return int(10 + 0.75 * int(steps))
# --------------------------------------------------------------------------- inference
@spaces.GPU(duration=_duration)
def predict(
first_frame: Image.Image,
render_video: str,
actions_file: str,
robot_type: str = "panda",
seed: int = 0,
steps: int = 50,
progress=gr.Progress(track_tqdm=True),
):
"""Predict the next 17 frames of a scene from its first frame, a camera-aligned rendering of
the embodiment along a trajectory, and the trajectory's raw per-frame states.
Args:
first_frame: RGB first camera frame of the clip.
render_video: Video (>= 17 frames) of the embodiment rendered from the same camera, black background.
actions_file: .json / .npz / .npy file with the raw (17, D) per-frame states ("state" key for npz/json dict).
robot_type: Embodiment of the actions (panda, widowx, ur5e_2f85, yam_bimanual, vega1_sharpa, mano_hands, smplx_human, smplh_human).
seed: Random seed.
steps: Number of flow-matching Euler steps (paper: 50).
Returns:
The predicted 17-frame video (frame 0 = input frame) and a side-by-side [render | prediction] video.
"""
if first_frame is None or render_video is None or actions_file is None:
raise gr.Error("Please provide a first frame, a render video and an actions file.")
actions_path = actions_file if isinstance(actions_file, str) else actions_file.name
raw, _ = load_actions(actions_path)
acts = prepare_actions(raw, robot_type).to("cuda", DTYPE)
image = resize_frames([first_frame.convert("RGB")], H, W)[0]
render = resize_frames(subsample_frames(read_all_frames(render_video), T, render_video), H, W)
t0 = time.time()
gen = torch.Generator(device="cuda").manual_seed(int(seed))
with torch.inference_mode(), torch.autocast(device_type="cuda", dtype=DTYPE):
out = pipe(
image=image,
prompt=None,
negative_prompt=None,
prompt_embeds=PROMPT_EMBEDS.to("cuda", DTYPE),
render_video=render,
actions=acts,
num_frames=T,
height=H,
width=W,
num_inference_steps=int(steps),
guidance_scale=1.0,
max_sequence_length=512,
generator=gen,
output_type="np",
return_dict=False,
)
pred = frames_to_uint8(out[0])[:T]
print(f"[doppel] {robot_type} steps={steps} seed={seed}: {time.time() - t0:.1f}s", flush=True)
tmp = tempfile.mkdtemp()
pred_path = os.path.join(tmp, "doppel_prediction.mp4")
side_path = os.path.join(tmp, "render_vs_prediction.mp4")
write_video(pred_path, pred, fps=8)
render_np = np.stack([np.asarray(f) for f in render])
write_video(side_path, np.concatenate([render_np, pred], axis=2), fps=8)
return pred_path, side_path
# --------------------------------------------------------------------------- UI
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks() as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"""
# Doppel: Render the Robot, Generate the World
**One action-conditioned video world model across eight embodiments** (Wan 2.1 I2V 14B + render-action adapter).
Give it the first camera frame, a rendering of the robot/body along a trajectory (seen from the same camera, black
background) and the raw per-frame states β€” it predicts how the embodiment **and the scene** evolve (17 frames, 240Γ—432).
[Paper](https://huggingface.co/papers/2611.24925) Β· [Project page](https://doppelwm.github.io) Β· [Code](https://github.com/DoppelWM/Doppel) Β· [Model](https://huggingface.co/sourlreapwr/doppel-emb8)
"""
)
with gr.Row():
with gr.Column():
first_frame = gr.Image(label="First camera frame", type="pil", height=260)
render_video = gr.Video(label="Embodiment rendering (β‰₯17 frames, same camera)", height=260)
with gr.Row():
actions_file = gr.File(
label="Actions: raw (17, D) states (.json / .npz / .npy)",
file_types=[".json", ".npz", ".npy"],
)
robot_type = gr.Dropdown(
choices=[(v, k) for k, v in ROBOT_TYPES.items()],
value="panda",
label="Embodiment",
)
run = gr.Button("Predict the future", variant="primary")
with gr.Column():
pred_video = gr.Video(label="Doppel prediction", autoplay=True, loop=True, height=260)
side_video = gr.Video(label="Rendering (input) | Prediction", autoplay=True, loop=True, height=260)
with gr.Accordion("Advanced settings", open=False):
seed = gr.Number(label="Seed", value=0, precision=0)
steps = gr.Slider(label="Euler steps", minimum=10, maximum=50, step=1, value=50)
gr.Markdown(
"Actions are raw per-frame states (not deltas); grippers are closures in [0, 1]. "
"Layouts per embodiment are listed in the [Doppel README](https://github.com/DoppelWM/Doppel#data-format). "
"The rendering must be aligned with the camera of the first frame β€” misaligned renders degrade predictions. "
"Human embodiments require SMPL-H / SMPL-X / MANO renders you produce yourself."
)
inputs = [first_frame, render_video, actions_file, robot_type]
gr.Examples(
examples=[
["examples/droid_frame0.png", "examples/droid_render.mp4", "examples/droid_actions.json", "panda"],
["examples/bridge_frame0.png", "examples/bridge_render.mp4", "examples/bridge_actions.json", "widowx"],
["examples/robomind_frame0.png", "examples/robomind_render.mp4", "examples/robomind_actions.json", "ur5e_2f85"],
],
inputs=inputs,
outputs=[pred_video, side_video],
fn=predict,
cache_examples=True,
cache_mode="lazy",
label="Held-out clips from the Doppel project page (DROID / Bridge / RoboMIND)",
)
run.click(predict, inputs=inputs + [seed, steps], outputs=[pred_video, side_video])
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)