mpeg-diffusion / app.py
multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
929cee3 verified
Raw History Blame Contribute Delete
17.9 kB
"""MPEG-Diffusion β€” mixed-resolution text-to-video generation on MiniMax-H3.
This Space implements the MPEG-Diffusion inference recipe end to end: a low-resolution track
denoises the whole frame, a linear region head self-selects where detail matters, and a
high-resolution track refines only that region through dual-resolution attention with
ridge-fitted up-maps, before the two tracks are merged in pixel space.
The reference implementation is the paper's port for MiniMax-H3
(github.com/scrya-com/mpeg-diffusion, Apache-2.0), vendored here as ``lot/`` plus a
subset of its musubi-tuner (``third_party/musubi_tuner/``). The weights are the published
research artifacts (johndpope/MiniMax-H3-MPEG-Diffusion): the fitted KV up-maps and the
v3 region head, applied to the *untrained* ComfyUI int8 ConvRot base DiT β€” exactly the
"quality preset" the port's README documents (`--preset quality`: steps 20, switch 16,
hr_steps 6, switch_upscale="crop", final="pixel", feather 4.0, alpha 0.5, block 25).
The 62 GiB Qwen3-VL text encoder does not fit beside the transformer on a single ZeroGPU
worker, so conditioning is delegated to ``multimodalart/qwen3vl-conditioner`` over the
gradio API β€” the same split every MiniMax-H3 Space in this org uses. Its ``/encode``
returns the layer-50 hidden state ``prompt_embeds (1, L, 5120) bfloat16``, which is
precisely the ``text`` input ``two_track_sample`` consumes.
"""
from __future__ import annotations
import os
# expandable_segments avoids the fragmentation the 21 GB int8 streaming load would otherwise
# hit; must be set before torch is imported. Same for the muted triton autotune cache dir.
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
# The int8 Triton kernels autotune; keep the cache on the ephemeral disk, not a read-only layer.
os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton-cache")
import spaces # noqa: E402 β€” FIRST, before any CUDA-touching import
import sys
import tempfile
import time
import traceback
from functools import cache
from pathlib import Path
import gradio as gr
import torch
SPACE_ROOT = Path(__file__).resolve().parent
# The weights the Space downloads at runtime. The paths are exported *before* any ``lot``
# import because ``lot_paths`` reads them at import time (``procrustes_h3`` reads
# ``LOT_H3_CHECKPOINT`` the moment it is imported).
BASE_REPO = "Comfy-Org/MiniMax-H3"
DIT_FILE = "diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors"
VAE_FILE = "vae/minimax_h3_video_vae_fp16.safetensors"
TT_REPO = "johndpope/MiniMax-H3-MPEG-Diffusion"
MAPS_FILE = "kv_maps/up_maps_tt.safetensors"
REGION_FILE = "region_head_v3/region_head_tt.safetensors"
CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
# The quality preset from the port's README, pinned as defaults. 1344x768 is the canvas the
# maps and the head were fitted/validated on. 22 pixel frames (latent_t 7, 0.92 s at 24 fps)
# is the render length the port itself measures and ships: peak 26.8 GB with every block
# resident. Longer clips blow up the HR dual-resolution attention β€” the author's own note
# says 73 pixel frames OOM'd a 32 GB 5090 without streaming, and 56 frames OOM'd this
# Space's 48 GB worker on the first live run.
CANVAS_W, CANVAS_H = 1344, 768
NUM_FRAMES = 22
# The conditioner enforces a 2 s floor on num_frames, but its prompt_embeds are text-only
# states that do NOT depend on clip length (its _encode never sees num_frames β€” the value
# only gates validation and the optional prompt rewrite). So the /encode request passes 56
# (2.33 s, the shortest grid value over the floor) purely to clear that check, and the
# sampler renders the 22-frame reference clip from those same states.
CONDITIONER_FRAMES = 56
FPS = 24
DEFAULT_STEPS, DEFAULT_SWITCH, DEFAULT_HR_STEPS = 20, 16, 6
DEFAULT_FEATHER, DEFAULT_ALPHA = 4.0, 0.5
OUTPUT_DIR = tempfile.mkdtemp(prefix="mpeg-diffusion-out-")
GPU_SIZE = "large"
DIT: torch.nn.Module | None = None
UPSCALE = None # lot.two_track_h3.VaeUpscaler
MAPS = None # lot.mixed_res.UpMaps
HEAD = None # lot.mixed_res.RegionHead
HEAD_BLOCK = 25
LOAD_ERROR: str | None = None
LOADED_IN: float | None = None
# ── Model loading (module scope, eager) ─────────────────────────────────────
def load_models() -> str | None:
"""Load DiT + VAE + up-maps + region head at startup, `.to("cuda")` so the ZeroGPU
startup pack streams them disk->VRAM on a cold worker (26 GB total)."""
global DIT, UPSCALE, MAPS, HEAD, HEAD_BLOCK, LOAD_ERROR, LOADED_IN
if DIT is not None or LOAD_ERROR is not None:
return LOAD_ERROR
started = time.time()
try:
from huggingface_hub import hf_hub_download
os.environ["LOT_H3_CHECKPOINT"] = hf_hub_download(BASE_REPO, DIT_FILE)
os.environ["LOT_H3_VAE"] = hf_hub_download(BASE_REPO, VAE_FILE)
maps_path = hf_hub_download(TT_REPO, MAPS_FILE)
region_path = hf_hub_download(TT_REPO, REGION_FILE)
sys.path.insert(0, str(SPACE_ROOT / "lot"))
sys.path.insert(0, str(SPACE_ROOT / "third_party"))
from two_track_h3 import load_maps, load_region, load_upscaler # lot/ is on sys.path
from h3_musubi import load_dit
print(f"[load] int8 ConvRot DiT from {BASE_REPO}/{DIT_FILE} ...", flush=True)
DIT = load_dit(os.environ["LOT_H3_CHECKPOINT"], device="cuda", swap=0)
print(f"[load] DiT resident ({time.time() - started:.0f}s), loading VAE ...", flush=True)
UPSCALE = load_upscaler()
print(f"[load] VAE resident, loading fitted up-maps + region head ...", flush=True)
MAPS = load_maps(Path(maps_path))
saved = load_region(Path(region_path))
from mixed_res import RegionHead
HEAD = RegionHead(saved["hidden"]).cuda().eval()
HEAD.load_state_dict(saved["state"])
HEAD_BLOCK = int(saved["block"])
LOADED_IN = time.time() - started
print(f"[load] ready in {LOADED_IN:.0f}s (region head on block {HEAD_BLOCK})", flush=True)
except Exception as error:
traceback.print_exc()
LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`"
return LOAD_ERROR
load_models()
# ── Remote conditioning ──────────────────────────────────────────────────────
@cache
def conditioner():
from gradio_client import Client
return Client(CONDITIONER_SPACE)
def conditioner_client(ip_token):
if not ip_token:
return conditioner()
from gradio_client import Client
return Client(CONDITIONER_SPACE, headers={"x-ip-token": ip_token})
def encode_remote(prompt, ip_token=None):
"""Call the conditioner Space for the layer-50 Qwen3-VL states ``two_track_sample`` consumes."""
from safetensors import safe_open
def call():
return conditioner_client(ip_token).predict(
prompt=prompt,
image_path=None,
last_image_path=None,
canvas=f"{CANVAS_W}x{CANVAS_H} Β· 16:9 full",
num_frames=CONDITIONER_FRAMES,
rewrite_prompt=False,
api_name="/encode",
)
try:
path, plan = call()
except Exception as first:
print(f"[cond] retrying with a fresh client after: {first}", flush=True)
conditioner.cache_clear()
path, plan = call()
with safe_open(path, framework="pt") as handle:
return handle.get_tensor("prompt_embeds"), plan
def _caller_ip_token() -> str | None:
from gradio.context import LocalContext
request = LocalContext.request.get()
return request.headers.get("x-ip-token") if request is not None else None
# ── GPU inference ─────────────────────────────────────────────────────────────
@spaces.GPU(duration=60, size=GPU_SIZE) # measured 33s end to end (sampler 32s); 60 covers a cold
# worker attach + the sliders' max settings (steps 30)
@torch.no_grad() # render() is grad-free end to end; without this the VAE decode of the merge
# builds a full autograd graph (~24 GB of retained activations) and OOMs the worker
def _generate(prompt_embeds, seed, steps, switch, hr_steps, feather, alpha, use_head, box):
"""Run the four-stage two-track sampler and compose to pixels, all on GPU time.
Mirrors ``two_track_h3.render()`` with ``--preset quality``: 20 steps of which the
first 16 run at half resolution, the region head picks M from block-25 states at the
switch, the HR track takes 6 linear-sigma steps on M only (``switch_upscale="crop"``
round-trips just the bounding box through the VAE), and the merge happens in pixel
space (``final="pixel"``) with a feather of 4 cells.
``two_track_sample`` has three return shapes: when the region comes back empty (or
switch >= steps) it returns only the plain LR latent, so the pixel merge must be
guarded exactly the way ``render()`` guards it.
"""
import torch
from two_track_h3 import pixel_compose, save_clip, two_track_sample
from h3_musubi import latent_frames
latent_t = latent_frames(NUM_FRAMES)
head = HEAD if use_head else None
box = tuple(float(v) for v in box) if (box and not use_head) else None
embeds = prompt_embeds.to("cuda", torch.bfloat16)
latent, info = two_track_sample(
DIT, embeds, UPSCALE,
height=CANVAS_H, width=CANVAS_W, steps=int(steps), switch=int(switch), seed=int(seed),
maps=MAPS, head=head, box=box, block=HEAD_BLOCK,
alpha=float(alpha), feather=float(feather), hr_steps=int(hr_steps),
latent_t=latent_t, switch_upscale="crop", final="pixel",
)
# Stage boundary, as in the vendored code: the sampler's dual-resolution activations are
# freed on return but the caching allocator still holds their segments β€” hand them back
# so the merge starts from a clean pool. (The full-clip decode OOM here on the first two
# live runs was the missing no_grad on this function: the VAE decode was retaining a
# ~24 GB autograd graph that render()'s @torch.no_grad() never builds.)
torch.cuda.empty_cache()
if "lr" in info: # render()'s guard: the merge inputs exist only on the two-track path
pixels = pixel_compose(UPSCALE.vae, latent, info["lr"], info["weight"])
else:
pixels = UPSCALE.vae.decode(latent.to("cuda", UPSCALE.dtype)).to("cpu", torch.float16)
out_path = os.path.join(OUTPUT_DIR, f"mpeg-diffusion-{int(time.time() * 1000)}.mp4")
save_clip(pixels, Path(out_path), fps=FPS)
share = float(info["mask"].float().mean()) if info.get("mask") is not None else 0.0
return out_path, share
def generate(
prompt,
use_head=True,
box_x0=0.25,
box_y0=0.25,
box_x1=0.75,
box_y1=0.75,
seed=0,
steps=DEFAULT_STEPS,
switch=DEFAULT_SWITCH,
hr_steps=DEFAULT_HR_STEPS,
feather=DEFAULT_FEATHER,
alpha=DEFAULT_ALPHA,
progress=gr.Progress(track_tqdm=True),
):
"""One request: prompt -> mixed-resolution video. Conditioning runs on the conditioner
Space (outside GPU booking, charged to the caller's quota via x-ip-token); the sampler
runs on this Space's ZeroGPU worker."""
if LOAD_ERROR:
raise gr.Error(LOAD_ERROR.replace("**", "").replace("`", ""))
if DIT is None:
raise gr.Error("The model is still loading β€” watch the Space logs and retry shortly.")
if not prompt or not prompt.strip():
raise gr.Error("A prompt is required.")
if int(switch) >= int(steps):
raise gr.Error("The region switch step must come before the last step.")
box = (float(box_x0), float(box_y0), float(box_x1), float(box_y1))
if box[0] >= box[2] or box[1] >= box[3]:
raise gr.Error("The manual region box needs x1 > x0 and y1 > y0.")
progress(0.0, desc="Encoding the prompt ...")
prompt_embeds, plan = encode_remote(prompt, ip_token=_caller_ip_token())
progress(0.1, desc=f"Two-track sampling {CANVAS_W}x{CANVAS_H} x {NUM_FRAMES} frames ...")
started = time.time()
out_path, share = _generate(
prompt_embeds, int(seed), int(steps), int(switch), int(hr_steps),
float(feather), float(alpha), bool(use_head), box,
)
seconds = time.time() - started
region = "region head" if use_head else f"manual box {tuple(round(v, 2) for v in box)}"
report = (
f"{CANVAS_W}x{CANVAS_H} Β· {NUM_FRAMES} frames ({NUM_FRAMES / FPS:.2f}s) Β· "
f"{int(steps)} steps, switch at {int(switch)}, {int(hr_steps)} HR steps Β· "
f"HR region: {region} covering {share:.0%} of tokens Β· "
f"conditioner {plan.get('seconds', '?')}s Β· sampler {seconds:.0f}s Β· seed {int(seed)}"
)
print(f"[gen] {report}", flush=True)
return out_path, report
# ── UI ────────────────────────────────────────────────────────────────────────
INTRO = """# MPEG-Diffusion: Mixed-Resolution Video Generation
<div align="center">
<a href="https://huggingface.co/papers/2611.12069" target="_blank" rel="noopener"><strong>[ paper ]</strong></a> &nbsp;
<a href="https://github.com/scrya-com/mpeg-diffusion" target="_blank" rel="noopener"><strong>[ code ]</strong></a> &nbsp;
<a href="https://huggingface.co/johndpope/MiniMax-H3-MPEG-Diffusion" target="_blank" rel="noopener"><strong>[ weights ]</strong></a>
</div>
**MPEG-Diffusion** (Menn et al.) generates video the way video codecs think about it: a
**low-resolution track** denoises the whole frame cheaply, a tiny linear head **self-selects
the region that deserves detail**, and a **high-resolution track** refines only that region
through dual-resolution attention. The tracks merge in pixel space β€” full-res detail where it
matters, ~2x the speed of rendering everything at high resolution.
This Space runs the paper's MiniMax-H3 port with the published research weights: the fitted
KV up-maps and the v3 region head on the int8 ConvRot base DiT, at the port's *quality preset*
(20 steps, region chosen at step 16, 6 HR steps, pixel-space merge). By default the model
picks its own detail region; force one with a manual box in *Advanced*."""
CSS = """
.main.fillable { max-width: 1100px !important; }
"""
EXAMPLE_PROMPTS = [
"A portrait of a young woman holding a red hibiscus flower against a rustic background",
"A joyful woman with colorful flowers in her afro, exuding happiness and vibrant style",
"A bearded man in a bathrobe uses a smartphone under studio lighting with a soft focus",
"A fashionable man in checkered pants and cowboy hat poses outdoors on concrete blocks",
"A serene portrait of a woman with curly hair surrounded by autumn foliage and berries",
]
with gr.Blocks(title="MPEG-Diffusion") as demo:
gr.Markdown(INTRO)
with gr.Row(equal_height=True):
with gr.Column():
prompt = gr.Textbox(
label="Prompt",
lines=3,
value=EXAMPLE_PROMPTS[0],
placeholder="Describe the scene β€” a clear subject helps the region head pick well ...",
)
use_head = gr.Checkbox(
label="Let the model choose the detail region (recommended)",
value=True,
)
run = gr.Button("Generate", variant="primary", size="lg")
with gr.Accordion("Advanced", open=False):
seed = gr.Number(label="Seed", value=0, precision=0)
with gr.Row():
steps = gr.Slider(label="Steps", minimum=10, maximum=30, step=1, value=DEFAULT_STEPS)
switch = gr.Slider(label="Region switch step", minimum=8, maximum=24, step=1, value=DEFAULT_SWITCH)
with gr.Row():
hr_steps = gr.Slider(label="HR steps", minimum=2, maximum=10, step=1, value=DEFAULT_HR_STEPS)
feather = gr.Slider(label="Feather (cells)", minimum=0, maximum=8, step=1, value=DEFAULT_FEATHER)
with gr.Row():
alpha = gr.Slider(label="Region threshold Ξ±", minimum=0.1, maximum=0.9, step=0.05, value=DEFAULT_ALPHA)
with gr.Accordion("Manual region (used when the checkbox is off)", open=False):
with gr.Row():
box_x0 = gr.Slider(label="Left x0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
box_y0 = gr.Slider(label="Top y0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
with gr.Row():
box_x1 = gr.Slider(label="Right x1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)
box_y1 = gr.Slider(label="Bottom y1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)
with gr.Column():
result = gr.Video(label="Mixed-resolution video", autoplay=True)
report = gr.Markdown()
gr.Examples(
examples=[[p] for p in EXAMPLE_PROMPTS],
inputs=[prompt],
fn=generate,
outputs=[result, report],
cache_examples=True,
cache_mode="lazy",
)
inputs = [prompt, use_head, box_x0, box_y0, box_x1, box_y1, seed, steps, switch, hr_steps, feather, alpha]
run.click(generate, inputs, [result, report], api_name="generate")
prompt.submit(generate, inputs, [result, report], api_name="generate_submit")
if __name__ == "__main__":
demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, allowed_paths=[OUTPUT_DIR])