"""MPEG-Diffusion — mixed-resolution text-to-video generation on MiniMax-H3. This Space implements the MPEG-Diffusion inference recipe end to end: a low-resolution track denoises the whole frame, a linear region head self-selects where detail matters, and a high-resolution track refines only that region through dual-resolution attention with ridge-fitted up-maps, before the two tracks are merged in pixel space. The reference implementation is the paper's port for MiniMax-H3 (github.com/scrya-com/mpeg-diffusion, Apache-2.0), vendored here as ``lot/`` plus a subset of its musubi-tuner (``third_party/musubi_tuner/``). The weights are the published research artifacts (johndpope/MiniMax-H3-MPEG-Diffusion): the fitted KV up-maps and the v3 region head, applied to the *untrained* ComfyUI int8 ConvRot base DiT — exactly the "quality preset" the port's README documents (`--preset quality`: steps 20, switch 16, hr_steps 6, switch_upscale="crop", final="pixel", feather 4.0, alpha 0.5, block 25). The 62 GiB Qwen3-VL text encoder does not fit beside the transformer on a single ZeroGPU worker, so conditioning is delegated to ``multimodalart/qwen3vl-conditioner`` over the gradio API — the same split every MiniMax-H3 Space in this org uses. Its ``/encode`` returns the layer-50 hidden state ``prompt_embeds (1, L, 5120) bfloat16``, which is precisely the ``text`` input ``two_track_sample`` consumes. """ from __future__ import annotations import os # expandable_segments avoids the fragmentation the 21 GB int8 streaming load would otherwise # hit; must be set before torch is imported. Same for the muted triton autotune cache dir. os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # The int8 Triton kernels autotune; keep the cache on the ephemeral disk, not a read-only layer. os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton-cache") import spaces # noqa: E402 — FIRST, before any CUDA-touching import import sys import tempfile import time import traceback from functools import cache from pathlib import Path import gradio as gr import torch SPACE_ROOT = Path(__file__).resolve().parent # The weights the Space downloads at runtime. The paths are exported *before* any ``lot`` # import because ``lot_paths`` reads them at import time (``procrustes_h3`` reads # ``LOT_H3_CHECKPOINT`` the moment it is imported). BASE_REPO = "Comfy-Org/MiniMax-H3" DIT_FILE = "diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors" VAE_FILE = "vae/minimax_h3_video_vae_fp16.safetensors" TT_REPO = "johndpope/MiniMax-H3-MPEG-Diffusion" MAPS_FILE = "kv_maps/up_maps_tt.safetensors" REGION_FILE = "region_head_v3/region_head_tt.safetensors" CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner") # The quality preset from the port's README, pinned as defaults. 1344x768 is the canvas the # maps and the head were fitted/validated on. 22 pixel frames (latent_t 7, 0.92 s at 24 fps) # is the render length the port itself measures and ships: peak 26.8 GB with every block # resident. Longer clips blow up the HR dual-resolution attention — the author's own note # says 73 pixel frames OOM'd a 32 GB 5090 without streaming, and 56 frames OOM'd this # Space's 48 GB worker on the first live run. CANVAS_W, CANVAS_H = 1344, 768 NUM_FRAMES = 22 # The conditioner enforces a 2 s floor on num_frames, but its prompt_embeds are text-only # states that do NOT depend on clip length (its _encode never sees num_frames — the value # only gates validation and the optional prompt rewrite). So the /encode request passes 56 # (2.33 s, the shortest grid value over the floor) purely to clear that check, and the # sampler renders the 22-frame reference clip from those same states. CONDITIONER_FRAMES = 56 FPS = 24 DEFAULT_STEPS, DEFAULT_SWITCH, DEFAULT_HR_STEPS = 20, 16, 6 DEFAULT_FEATHER, DEFAULT_ALPHA = 4.0, 0.5 OUTPUT_DIR = tempfile.mkdtemp(prefix="mpeg-diffusion-out-") GPU_SIZE = "large" DIT: torch.nn.Module | None = None UPSCALE = None # lot.two_track_h3.VaeUpscaler MAPS = None # lot.mixed_res.UpMaps HEAD = None # lot.mixed_res.RegionHead HEAD_BLOCK = 25 LOAD_ERROR: str | None = None LOADED_IN: float | None = None # ── Model loading (module scope, eager) ───────────────────────────────────── def load_models() -> str | None: """Load DiT + VAE + up-maps + region head at startup, `.to("cuda")` so the ZeroGPU startup pack streams them disk->VRAM on a cold worker (26 GB total).""" global DIT, UPSCALE, MAPS, HEAD, HEAD_BLOCK, LOAD_ERROR, LOADED_IN if DIT is not None or LOAD_ERROR is not None: return LOAD_ERROR started = time.time() try: from huggingface_hub import hf_hub_download os.environ["LOT_H3_CHECKPOINT"] = hf_hub_download(BASE_REPO, DIT_FILE) os.environ["LOT_H3_VAE"] = hf_hub_download(BASE_REPO, VAE_FILE) maps_path = hf_hub_download(TT_REPO, MAPS_FILE) region_path = hf_hub_download(TT_REPO, REGION_FILE) sys.path.insert(0, str(SPACE_ROOT / "lot")) sys.path.insert(0, str(SPACE_ROOT / "third_party")) from two_track_h3 import load_maps, load_region, load_upscaler # lot/ is on sys.path from h3_musubi import load_dit print(f"[load] int8 ConvRot DiT from {BASE_REPO}/{DIT_FILE} ...", flush=True) DIT = load_dit(os.environ["LOT_H3_CHECKPOINT"], device="cuda", swap=0) print(f"[load] DiT resident ({time.time() - started:.0f}s), loading VAE ...", flush=True) UPSCALE = load_upscaler() print(f"[load] VAE resident, loading fitted up-maps + region head ...", flush=True) MAPS = load_maps(Path(maps_path)) saved = load_region(Path(region_path)) from mixed_res import RegionHead HEAD = RegionHead(saved["hidden"]).cuda().eval() HEAD.load_state_dict(saved["state"]) HEAD_BLOCK = int(saved["block"]) LOADED_IN = time.time() - started print(f"[load] ready in {LOADED_IN:.0f}s (region head on block {HEAD_BLOCK})", flush=True) except Exception as error: traceback.print_exc() LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`" return LOAD_ERROR load_models() # ── Remote conditioning ────────────────────────────────────────────────────── @cache def conditioner(): from gradio_client import Client return Client(CONDITIONER_SPACE) def conditioner_client(ip_token): if not ip_token: return conditioner() from gradio_client import Client return Client(CONDITIONER_SPACE, headers={"x-ip-token": ip_token}) def encode_remote(prompt, ip_token=None): """Call the conditioner Space for the layer-50 Qwen3-VL states ``two_track_sample`` consumes.""" from safetensors import safe_open def call(): return conditioner_client(ip_token).predict( prompt=prompt, image_path=None, last_image_path=None, canvas=f"{CANVAS_W}x{CANVAS_H} · 16:9 full", num_frames=CONDITIONER_FRAMES, rewrite_prompt=False, api_name="/encode", ) try: path, plan = call() except Exception as first: print(f"[cond] retrying with a fresh client after: {first}", flush=True) conditioner.cache_clear() path, plan = call() with safe_open(path, framework="pt") as handle: return handle.get_tensor("prompt_embeds"), plan def _caller_ip_token() -> str | None: from gradio.context import LocalContext request = LocalContext.request.get() return request.headers.get("x-ip-token") if request is not None else None # ── GPU inference ───────────────────────────────────────────────────────────── @spaces.GPU(duration=60, size=GPU_SIZE) # measured 33s end to end (sampler 32s); 60 covers a cold # worker attach + the sliders' max settings (steps 30) @torch.no_grad() # render() is grad-free end to end; without this the VAE decode of the merge # builds a full autograd graph (~24 GB of retained activations) and OOMs the worker def _generate(prompt_embeds, seed, steps, switch, hr_steps, feather, alpha, use_head, box): """Run the four-stage two-track sampler and compose to pixels, all on GPU time. Mirrors ``two_track_h3.render()`` with ``--preset quality``: 20 steps of which the first 16 run at half resolution, the region head picks M from block-25 states at the switch, the HR track takes 6 linear-sigma steps on M only (``switch_upscale="crop"`` round-trips just the bounding box through the VAE), and the merge happens in pixel space (``final="pixel"``) with a feather of 4 cells. ``two_track_sample`` has three return shapes: when the region comes back empty (or switch >= steps) it returns only the plain LR latent, so the pixel merge must be guarded exactly the way ``render()`` guards it. """ import torch from two_track_h3 import pixel_compose, save_clip, two_track_sample from h3_musubi import latent_frames latent_t = latent_frames(NUM_FRAMES) head = HEAD if use_head else None box = tuple(float(v) for v in box) if (box and not use_head) else None embeds = prompt_embeds.to("cuda", torch.bfloat16) latent, info = two_track_sample( DIT, embeds, UPSCALE, height=CANVAS_H, width=CANVAS_W, steps=int(steps), switch=int(switch), seed=int(seed), maps=MAPS, head=head, box=box, block=HEAD_BLOCK, alpha=float(alpha), feather=float(feather), hr_steps=int(hr_steps), latent_t=latent_t, switch_upscale="crop", final="pixel", ) # Stage boundary, as in the vendored code: the sampler's dual-resolution activations are # freed on return but the caching allocator still holds their segments — hand them back # so the merge starts from a clean pool. (The full-clip decode OOM here on the first two # live runs was the missing no_grad on this function: the VAE decode was retaining a # ~24 GB autograd graph that render()'s @torch.no_grad() never builds.) torch.cuda.empty_cache() if "lr" in info: # render()'s guard: the merge inputs exist only on the two-track path pixels = pixel_compose(UPSCALE.vae, latent, info["lr"], info["weight"]) else: pixels = UPSCALE.vae.decode(latent.to("cuda", UPSCALE.dtype)).to("cpu", torch.float16) out_path = os.path.join(OUTPUT_DIR, f"mpeg-diffusion-{int(time.time() * 1000)}.mp4") save_clip(pixels, Path(out_path), fps=FPS) share = float(info["mask"].float().mean()) if info.get("mask") is not None else 0.0 return out_path, share def generate( prompt, use_head=True, box_x0=0.25, box_y0=0.25, box_x1=0.75, box_y1=0.75, seed=0, steps=DEFAULT_STEPS, switch=DEFAULT_SWITCH, hr_steps=DEFAULT_HR_STEPS, feather=DEFAULT_FEATHER, alpha=DEFAULT_ALPHA, progress=gr.Progress(track_tqdm=True), ): """One request: prompt -> mixed-resolution video. Conditioning runs on the conditioner Space (outside GPU booking, charged to the caller's quota via x-ip-token); the sampler runs on this Space's ZeroGPU worker.""" if LOAD_ERROR: raise gr.Error(LOAD_ERROR.replace("**", "").replace("`", "")) if DIT is None: raise gr.Error("The model is still loading — watch the Space logs and retry shortly.") if not prompt or not prompt.strip(): raise gr.Error("A prompt is required.") if int(switch) >= int(steps): raise gr.Error("The region switch step must come before the last step.") box = (float(box_x0), float(box_y0), float(box_x1), float(box_y1)) if box[0] >= box[2] or box[1] >= box[3]: raise gr.Error("The manual region box needs x1 > x0 and y1 > y0.") progress(0.0, desc="Encoding the prompt ...") prompt_embeds, plan = encode_remote(prompt, ip_token=_caller_ip_token()) progress(0.1, desc=f"Two-track sampling {CANVAS_W}x{CANVAS_H} x {NUM_FRAMES} frames ...") started = time.time() out_path, share = _generate( prompt_embeds, int(seed), int(steps), int(switch), int(hr_steps), float(feather), float(alpha), bool(use_head), box, ) seconds = time.time() - started region = "region head" if use_head else f"manual box {tuple(round(v, 2) for v in box)}" report = ( f"{CANVAS_W}x{CANVAS_H} · {NUM_FRAMES} frames ({NUM_FRAMES / FPS:.2f}s) · " f"{int(steps)} steps, switch at {int(switch)}, {int(hr_steps)} HR steps · " f"HR region: {region} covering {share:.0%} of tokens · " f"conditioner {plan.get('seconds', '?')}s · sampler {seconds:.0f}s · seed {int(seed)}" ) print(f"[gen] {report}", flush=True) return out_path, report # ── UI ──────────────────────────────────────────────────────────────────────── INTRO = """# MPEG-Diffusion: Mixed-Resolution Video Generation
[ paper ]   [ code ]   [ weights ]
**MPEG-Diffusion** (Menn et al.) generates video the way video codecs think about it: a **low-resolution track** denoises the whole frame cheaply, a tiny linear head **self-selects the region that deserves detail**, and a **high-resolution track** refines only that region through dual-resolution attention. The tracks merge in pixel space — full-res detail where it matters, ~2x the speed of rendering everything at high resolution. This Space runs the paper's MiniMax-H3 port with the published research weights: the fitted KV up-maps and the v3 region head on the int8 ConvRot base DiT, at the port's *quality preset* (20 steps, region chosen at step 16, 6 HR steps, pixel-space merge). By default the model picks its own detail region; force one with a manual box in *Advanced*.""" CSS = """ .main.fillable { max-width: 1100px !important; } """ EXAMPLE_PROMPTS = [ "A portrait of a young woman holding a red hibiscus flower against a rustic background", "A joyful woman with colorful flowers in her afro, exuding happiness and vibrant style", "A bearded man in a bathrobe uses a smartphone under studio lighting with a soft focus", "A fashionable man in checkered pants and cowboy hat poses outdoors on concrete blocks", "A serene portrait of a woman with curly hair surrounded by autumn foliage and berries", ] with gr.Blocks(title="MPEG-Diffusion") as demo: gr.Markdown(INTRO) with gr.Row(equal_height=True): with gr.Column(): prompt = gr.Textbox( label="Prompt", lines=3, value=EXAMPLE_PROMPTS[0], placeholder="Describe the scene — a clear subject helps the region head pick well ...", ) use_head = gr.Checkbox( label="Let the model choose the detail region (recommended)", value=True, ) run = gr.Button("Generate", variant="primary", size="lg") with gr.Accordion("Advanced", open=False): seed = gr.Number(label="Seed", value=0, precision=0) with gr.Row(): steps = gr.Slider(label="Steps", minimum=10, maximum=30, step=1, value=DEFAULT_STEPS) switch = gr.Slider(label="Region switch step", minimum=8, maximum=24, step=1, value=DEFAULT_SWITCH) with gr.Row(): hr_steps = gr.Slider(label="HR steps", minimum=2, maximum=10, step=1, value=DEFAULT_HR_STEPS) feather = gr.Slider(label="Feather (cells)", minimum=0, maximum=8, step=1, value=DEFAULT_FEATHER) with gr.Row(): alpha = gr.Slider(label="Region threshold α", minimum=0.1, maximum=0.9, step=0.05, value=DEFAULT_ALPHA) with gr.Accordion("Manual region (used when the checkbox is off)", open=False): with gr.Row(): box_x0 = gr.Slider(label="Left x0", minimum=0.0, maximum=1.0, step=0.01, value=0.25) box_y0 = gr.Slider(label="Top y0", minimum=0.0, maximum=1.0, step=0.01, value=0.25) with gr.Row(): box_x1 = gr.Slider(label="Right x1", minimum=0.0, maximum=1.0, step=0.01, value=0.75) box_y1 = gr.Slider(label="Bottom y1", minimum=0.0, maximum=1.0, step=0.01, value=0.75) with gr.Column(): result = gr.Video(label="Mixed-resolution video", autoplay=True) report = gr.Markdown() gr.Examples( examples=[[p] for p in EXAMPLE_PROMPTS], inputs=[prompt], fn=generate, outputs=[result, report], cache_examples=True, cache_mode="lazy", ) inputs = [prompt, use_head, box_x0, box_y0, box_x1, box_y1, seed, steps, switch, hr_steps, feather, alpha] run.click(generate, inputs, [result, report], api_name="generate") prompt.submit(generate, inputs, [result, report], api_name="generate_submit") if __name__ == "__main__": demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, allowed_paths=[OUTPUT_DIR])