Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/invoker-render: direct link, hf CLI and curl.
- Browser
- Download file 13 kB
-
https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/invoker-render/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/app.py
13 kB
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # noqa: E402 (must precede torch) | |
| import gc # noqa: E402 | |
| import tempfile # noqa: E402 | |
| import time # noqa: E402 | |
| import gradio as gr # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import torch # noqa: E402 | |
| from huggingface_hub import hf_hub_download # noqa: E402 | |
| from PIL import Image, ImageOps # noqa: E402 | |
| import media # noqa: E402 | |
| from h3vae import H3VideoCodec, build_vae # noqa: E402 | |
| from invoker_dit import InvokerDiT, InvokerSession, load_invoker_dit # noqa: E402 | |
| from qwen_encoder import encode_condition, load_qwen # noqa: E402 | |
| MODEL_REPO = "inclusionAI/invoker-render-1.0" | |
| H3_REPO = "MiniMaxAI/MiniMax-H3" | |
| TOKEN = os.environ.get("HF_TOKEN") | |
| FRAME_CHOICES = media.valid_frame_counts(124) # 39, 56, 73, 90, 107, 124 | |
| DEFAULT_FRAMES = 73 | |
| t0 = time.time() | |
| # Qwen3-VL condition encoder: FP8 decoder linears, kept in host RAM and visiting the GPU only for | |
| # the single encode at the start of a request (it cannot share the card with the DiT's KV cache). | |
| QWEN, PROCESSOR = load_qwen(H3_REPO, "Ref2VA", token=TOKEN) | |
| QWEN_MASTERS = {} | |
| for _mname, _mod in QWEN.named_modules(): | |
| for _pname, _p in list(_mod._parameters.items()): | |
| if _p is not None: | |
| QWEN_MASTERS[(_mname, "p", _pname)] = _p.data | |
| for _bname, _b in list(_mod._buffers.items()): | |
| if _b is not None: | |
| QWEN_MASTERS[(_mname, "b", _bname)] = _b | |
| print(f"[load] qwen ready after {time.time() - t0:.0f}s", flush=True) | |
| # Invoker-Render causal DiT (FP32 checkpoint -> FP8 block linears + precomputed AdaLN tables). | |
| DIT_MODEL, ADALN_TABLES = load_invoker_dit(MODEL_REPO, token=TOKEN) | |
| DIT_MODEL.to("cuda") | |
| ADALN_TABLES = ADALN_TABLES.to("cuda") | |
| DIT = InvokerDiT(DIT_MODEL, ADALN_TABLES) | |
| print(f"[load] dit ready after {time.time() - t0:.0f}s", flush=True) | |
| # Native MiniMax-H3 video VAE (BF16, posterior mean), from the Ref2VA partition. | |
| VAE = build_vae(hf_hub_download(H3_REPO, "Ref2VA/video_vae/source/model.safetensors", token=TOKEN)).to("cuda") | |
| CODEC = H3VideoCodec(VAE) | |
| # Optional helper: real footage -> relative-depth proxy (Depth Anything V2, as in the paper's proxy pipeline). | |
| from transformers import AutoImageProcessor, AutoModelForDepthEstimation # noqa: E402 | |
| DEPTH_PROCESSOR = AutoImageProcessor.from_pretrained(media.DEPTH_MODEL) | |
| DEPTH_MODEL = AutoModelForDepthEstimation.from_pretrained(media.DEPTH_MODEL, torch_dtype=torch.float16).to("cuda").eval() | |
| print(f"[load] all models ready after {time.time() - t0:.0f}s", flush=True) | |
| def _qwen_to(device: str) -> None: | |
| for (mname, kind, name), master in QWEN_MASTERS.items(): | |
| mod = QWEN.get_submodule(mname) if mname else QWEN | |
| value = master if device == "cpu" else master.to(device, non_blocking=True) | |
| if kind == "p": | |
| mod._parameters[name].data = value | |
| else: | |
| mod._buffers[name] = value | |
| def _latents_for(num_frames: int) -> int: | |
| return 5 * ((num_frames - 5) // 17 + 1) - 3 | |
| def _duration(source_frames, *a, **k): | |
| latents = _latents_for(len(source_frames)) | |
| # Measured on the live Space: ~56 s (12 latents), ~78 s (22), ~111 s (37) warm, plus up to ~25 s when the | |
| # Qwen weights make their first host->GPU trip in a cold worker. | |
| return int(55 + 2.3 * latents) | |
| PROXY_KIND = "Structural proxy (use as-is)" | |
| REAL_KIND = "Real footage → depth proxy" | |
| def _render(source_frames, first_image, prompt, seed, input_kind): | |
| stats = {} | |
| started = time.time() | |
| if input_kind == REAL_KIND: | |
| proxy_small = media.depth_proxy(source_frames, DEPTH_MODEL, DEPTH_PROCESSOR, device="cuda") | |
| else: | |
| proxy_small = media.frames_to_canvas(source_frames, media.PROXY_CANVAS, Image.NEAREST) | |
| stats["proxy"] = time.time() - started | |
| # 1. Qwen3-VL static condition: `<Picture 1>: <first image> <prompt>`, layer-50 hidden state. | |
| t = time.time() | |
| _qwen_to("cuda") | |
| try: | |
| prompt_embeds, token_tags = encode_condition(QWEN, PROCESSOR, prompt, first_image, device="cuda") | |
| finally: | |
| _qwen_to("cpu") | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| stats["qwen"] = time.time() - t | |
| # 2. VAE: first image (single-frame path) and proxy (17-frame blocks, nearest resize to 1344x768). | |
| t = time.time() | |
| proxy_full = np.stack([np.asarray(Image.fromarray(f).resize(media.CANVAS, Image.NEAREST)) for f in proxy_small]) | |
| first_latent = CODEC.image(np.asarray(first_image)) | |
| proxy_latents = CODEC.video(proxy_full) | |
| stats["encode"] = time.time() - t | |
| # 3. Causal four-step streaming generation. | |
| t = time.time() | |
| generator = torch.Generator("cpu").manual_seed(int(seed)) | |
| session = InvokerSession( | |
| DIT, | |
| prompt_embeds=prompt_embeds, | |
| token_tags=token_tags, | |
| first_frame=first_latent.to("cuda"), | |
| first_frame_noise=torch.randn(first_latent.shape, generator=generator).to("cuda"), | |
| ) | |
| latents = session.generate(proxy_latents, generator=generator) | |
| del session | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| stats["dit"] = time.time() - t | |
| # 4. Native H3 decode of the whole latent stream. | |
| t = time.time() | |
| video = CODEC.decode_video(latents, num_frames=len(source_frames)) | |
| stats["decode"] = time.time() - t | |
| stats["peak_gb"] = torch.cuda.max_memory_allocated() / 2**30 | |
| return video, proxy_small, stats | |
| def render( | |
| proxy_video: str, | |
| first_image: str | None, | |
| prompt: str, | |
| seed: int = 42, | |
| num_frames: int = DEFAULT_FRAMES, | |
| input_kind: str = PROXY_KIND, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """Render a photorealistic video that follows a structural proxy video. | |
| Args: | |
| proxy_video: Structural proxy video (semantic map, depth, gray-box, wireframe, skeleton, ...), | |
| or real footage when `input_kind` asks for an automatic depth proxy. Resampled to 24 FPS. | |
| first_image: Appearance anchor - the opening RGB frame of the rendered video (cropped to 16:9, | |
| 1344x768). Optional only for real footage, where its first frame is used. | |
| prompt: Description of the scene (environment, subjects, lighting, events, camera motion). | |
| seed: Noise seed. | |
| num_frames: Output length in frames at 24 FPS; must be 17*N+5 (39, 56, 73, ...). | |
| input_kind: "Structural proxy (use as-is)" or "Real footage → depth proxy". | |
| Returns: | |
| The rendered MP4, the 448x256 proxy that conditioned it, and a timing summary. | |
| """ | |
| if not proxy_video: | |
| raise gr.Error("Upload a proxy video (or real footage with the depth-proxy option).") | |
| if not prompt or not prompt.strip(): | |
| raise gr.Error("Invoker-Render always takes a text description.") | |
| num_frames = int(num_frames) | |
| if num_frames not in FRAME_CHOICES: | |
| raise gr.Error(f"Number of frames must be one of {FRAME_CHOICES}.") | |
| frames = media.read_video_24fps(proxy_video, num_frames) | |
| if len(frames) < 39: | |
| raise gr.Error(f"The video is too short: {len(frames)} frames at 24 FPS, the model needs at least 39 (~1.6 s).") | |
| usable = max(n for n in FRAME_CHOICES if n <= len(frames)) | |
| frames = frames[:usable] | |
| if first_image: | |
| image = ImageOps.exif_transpose(Image.open(first_image)).convert("RGB") | |
| elif input_kind == REAL_KIND: | |
| image = Image.fromarray(frames[0]) | |
| else: | |
| raise gr.Error("Provide a first image (the appearance anchor) for a structural proxy.") | |
| image = media.cover_crop(image, media.CANVAS) | |
| video, proxy_small, stats = _render(frames, image, prompt.strip(), int(seed), input_kind) | |
| out_dir = tempfile.mkdtemp() | |
| out = media.write_mp4(video, os.path.join(out_dir, "invoker_render.mp4")) | |
| proxy_out = media.write_mp4(proxy_small, os.path.join(out_dir, "proxy.mp4")) | |
| info = ( | |
| f"{len(video)} frames ({len(video) / media.FPS:.2f} s) at {video.shape[2]}x{video.shape[1]} · " | |
| f"{_latents_for(usable)} latents · seed {int(seed)} \n" | |
| f"proxy {stats['proxy']:.1f}s · Qwen {stats['qwen']:.1f}s · VAE encode {stats['encode']:.1f}s · " | |
| f"DiT {stats['dit']:.1f}s · decode {stats['decode']:.1f}s · peak {stats['peak_gb']:.1f} GiB" | |
| ) | |
| print(f"[render] {info}", flush=True) | |
| return out, proxy_out, info | |
| EX = "examples" | |
| EXAMPLES = [ | |
| [ | |
| f"{EX}/city_traffic_night_depth_proxy.mp4", | |
| f"{EX}/city_traffic_night_first_frame.jpg", | |
| "A wide multi-lane city road at night. Cars and a white bus drive from left to right with glowing " | |
| "headlights and red taillights on dark wet asphalt; a long green neon storefront, street lamps and " | |
| "colorful signs glow on the far side. Static wide shot, cool low-light night exposure.", | |
| ], | |
| [ | |
| f"{EX}/cycling_forest_depth_proxy.mp4", | |
| f"{EX}/cycling_forest_first_frame.jpg", | |
| "A cyclist in a black jersey, white helmet and sunglasses rides a black road bike along a paved road " | |
| "through a forest of tall pine trees. Muted overcast daylight; the camera tracks alongside the rider " | |
| "as he pedals steadily.", | |
| ], | |
| [ | |
| f"{EX}/running_jog_depth_proxy.mp4", | |
| f"{EX}/running_jog_first_frame.jpg", | |
| "A bearded man in a grey tank top jogs toward the camera on a red running track beside green trees " | |
| "and an elevated highway overpass. Warm hazy daylight, shallow depth of field, the camera tracks " | |
| "backward to keep him centered.", | |
| ], | |
| [ | |
| f"{EX}/train_arriving_depth_proxy.mp4", | |
| f"{EX}/train_arriving_first_frame.jpg", | |
| "A white-and-red commuter train glides from left to right past a station sign across a busy rail " | |
| "yard with overhead wires, cranes and city buildings behind. Bright sunny sky with scattered " | |
| "clouds, stationary camera.", | |
| ], | |
| ] | |
| INTRO = """ | |
| # Invoker-Render 1.0 | |
| <div style="display:flex;gap:12px;flex-wrap:wrap"> | |
| <a href="https://huggingface.co/inclusionAI/invoker-render-1.0">Model</a> · | |
| <a href="https://github.com/inclusionAI/CodeWorldModel">Code</a> · | |
| <a href="https://www.inclusion-ai.org/CodeWorldModel/">Project page</a> · | |
| <a href="https://arxiv.org/abs/2611.22925">Paper</a> | |
| </div> | |
| **Code World Model 2.0's renderer.** Give it a *structural proxy* video (depth, semantic map, gray-box, wireframe, | |
| skeleton…) that fixes the camera path, layout and motion, an *appearance anchor* (the opening frame) and a | |
| description; the 33B causal DiT renders the video in five-latent chunks with four steps each. | |
| No proxy at hand? Upload real footage and pick **Real footage → depth proxy** under *Advanced*. | |
| """ | |
| CSS = """ | |
| #col-container { max-width: 1200px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="Invoker-Render 1.0") as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.Markdown(INTRO) | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Row(): | |
| proxy_in = gr.Video(label="Proxy video (24 FPS)", height=260) | |
| image_in = gr.Image(label="First image (appearance anchor)", type="filepath", height=260) | |
| prompt_in = gr.Textbox(label="Description", lines=3, | |
| placeholder="Environment, subjects, lighting, events and camera motion…") | |
| run = gr.Button("Render", variant="primary") | |
| with gr.Accordion("Advanced", open=False): | |
| kind_in = gr.Radio([PROXY_KIND, REAL_KIND], value=PROXY_KIND, label="Input video is") | |
| frames_in = gr.Dropdown(FRAME_CHOICES, value=DEFAULT_FRAMES, label="Frames (24 FPS, 17·N+5)") | |
| seed_in = gr.Number(label="Seed", value=42, precision=0) | |
| with gr.Column(): | |
| video_out = gr.Video(label="Rendered video", autoplay=True, height=360) | |
| proxy_out = gr.Video(label="Proxy used (448x256)", height=200) | |
| info_out = gr.Markdown() | |
| gr.Examples( | |
| examples=EXAMPLES, | |
| inputs=[proxy_in, image_in, prompt_in], | |
| outputs=[video_out, proxy_out, info_out], | |
| fn=render, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| gr.Markdown( | |
| "Example clips: CC0 footage from " | |
| "[linoyts/repo-to-space-example-videos](https://huggingface.co/datasets/linoyts/repo-to-space-example-videos), " | |
| "converted to depth proxies with Depth Anything V2. Weights: Apache-2.0 / CC BY-NC 4.0 (non-commercial); " | |
| "VAE and Qwen3-VL encoder from [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) " | |
| "(MiniMax H3 Community License)." | |
| ) | |
| run.click(render, [proxy_in, image_in, prompt_in, seed_in, frames_in, kind_in], | |
| [video_out, proxy_out, info_out], api_name="render") | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |