Spaces:
Running on Zero
Running on Zero
File size: 17,920 Bytes
6febba8 bb6cd86 6febba8 bb6cd86 6febba8 bb6cd86 6febba8 929cee3 81df4d4 6febba8 81df4d4 d2db40e 6febba8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 | """MPEG-Diffusion β mixed-resolution text-to-video generation on MiniMax-H3.
This Space implements the MPEG-Diffusion inference recipe end to end: a low-resolution track
denoises the whole frame, a linear region head self-selects where detail matters, and a
high-resolution track refines only that region through dual-resolution attention with
ridge-fitted up-maps, before the two tracks are merged in pixel space.
The reference implementation is the paper's port for MiniMax-H3
(github.com/scrya-com/mpeg-diffusion, Apache-2.0), vendored here as ``lot/`` plus a
subset of its musubi-tuner (``third_party/musubi_tuner/``). The weights are the published
research artifacts (johndpope/MiniMax-H3-MPEG-Diffusion): the fitted KV up-maps and the
v3 region head, applied to the *untrained* ComfyUI int8 ConvRot base DiT β exactly the
"quality preset" the port's README documents (`--preset quality`: steps 20, switch 16,
hr_steps 6, switch_upscale="crop", final="pixel", feather 4.0, alpha 0.5, block 25).
The 62 GiB Qwen3-VL text encoder does not fit beside the transformer on a single ZeroGPU
worker, so conditioning is delegated to ``multimodalart/qwen3vl-conditioner`` over the
gradio API β the same split every MiniMax-H3 Space in this org uses. Its ``/encode``
returns the layer-50 hidden state ``prompt_embeds (1, L, 5120) bfloat16``, which is
precisely the ``text`` input ``two_track_sample`` consumes.
"""
from __future__ import annotations
import os
# expandable_segments avoids the fragmentation the 21 GB int8 streaming load would otherwise
# hit; must be set before torch is imported. Same for the muted triton autotune cache dir.
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
# The int8 Triton kernels autotune; keep the cache on the ephemeral disk, not a read-only layer.
os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton-cache")
import spaces # noqa: E402 β FIRST, before any CUDA-touching import
import sys
import tempfile
import time
import traceback
from functools import cache
from pathlib import Path
import gradio as gr
import torch
SPACE_ROOT = Path(__file__).resolve().parent
# The weights the Space downloads at runtime. The paths are exported *before* any ``lot``
# import because ``lot_paths`` reads them at import time (``procrustes_h3`` reads
# ``LOT_H3_CHECKPOINT`` the moment it is imported).
BASE_REPO = "Comfy-Org/MiniMax-H3"
DIT_FILE = "diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors"
VAE_FILE = "vae/minimax_h3_video_vae_fp16.safetensors"
TT_REPO = "johndpope/MiniMax-H3-MPEG-Diffusion"
MAPS_FILE = "kv_maps/up_maps_tt.safetensors"
REGION_FILE = "region_head_v3/region_head_tt.safetensors"
CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
# The quality preset from the port's README, pinned as defaults. 1344x768 is the canvas the
# maps and the head were fitted/validated on. 22 pixel frames (latent_t 7, 0.92 s at 24 fps)
# is the render length the port itself measures and ships: peak 26.8 GB with every block
# resident. Longer clips blow up the HR dual-resolution attention β the author's own note
# says 73 pixel frames OOM'd a 32 GB 5090 without streaming, and 56 frames OOM'd this
# Space's 48 GB worker on the first live run.
CANVAS_W, CANVAS_H = 1344, 768
NUM_FRAMES = 22
# The conditioner enforces a 2 s floor on num_frames, but its prompt_embeds are text-only
# states that do NOT depend on clip length (its _encode never sees num_frames β the value
# only gates validation and the optional prompt rewrite). So the /encode request passes 56
# (2.33 s, the shortest grid value over the floor) purely to clear that check, and the
# sampler renders the 22-frame reference clip from those same states.
CONDITIONER_FRAMES = 56
FPS = 24
DEFAULT_STEPS, DEFAULT_SWITCH, DEFAULT_HR_STEPS = 20, 16, 6
DEFAULT_FEATHER, DEFAULT_ALPHA = 4.0, 0.5
OUTPUT_DIR = tempfile.mkdtemp(prefix="mpeg-diffusion-out-")
GPU_SIZE = "large"
DIT: torch.nn.Module | None = None
UPSCALE = None # lot.two_track_h3.VaeUpscaler
MAPS = None # lot.mixed_res.UpMaps
HEAD = None # lot.mixed_res.RegionHead
HEAD_BLOCK = 25
LOAD_ERROR: str | None = None
LOADED_IN: float | None = None
# ββ Model loading (module scope, eager) βββββββββββββββββββββββββββββββββββββ
def load_models() -> str | None:
"""Load DiT + VAE + up-maps + region head at startup, `.to("cuda")` so the ZeroGPU
startup pack streams them disk->VRAM on a cold worker (26 GB total)."""
global DIT, UPSCALE, MAPS, HEAD, HEAD_BLOCK, LOAD_ERROR, LOADED_IN
if DIT is not None or LOAD_ERROR is not None:
return LOAD_ERROR
started = time.time()
try:
from huggingface_hub import hf_hub_download
os.environ["LOT_H3_CHECKPOINT"] = hf_hub_download(BASE_REPO, DIT_FILE)
os.environ["LOT_H3_VAE"] = hf_hub_download(BASE_REPO, VAE_FILE)
maps_path = hf_hub_download(TT_REPO, MAPS_FILE)
region_path = hf_hub_download(TT_REPO, REGION_FILE)
sys.path.insert(0, str(SPACE_ROOT / "lot"))
sys.path.insert(0, str(SPACE_ROOT / "third_party"))
from two_track_h3 import load_maps, load_region, load_upscaler # lot/ is on sys.path
from h3_musubi import load_dit
print(f"[load] int8 ConvRot DiT from {BASE_REPO}/{DIT_FILE} ...", flush=True)
DIT = load_dit(os.environ["LOT_H3_CHECKPOINT"], device="cuda", swap=0)
print(f"[load] DiT resident ({time.time() - started:.0f}s), loading VAE ...", flush=True)
UPSCALE = load_upscaler()
print(f"[load] VAE resident, loading fitted up-maps + region head ...", flush=True)
MAPS = load_maps(Path(maps_path))
saved = load_region(Path(region_path))
from mixed_res import RegionHead
HEAD = RegionHead(saved["hidden"]).cuda().eval()
HEAD.load_state_dict(saved["state"])
HEAD_BLOCK = int(saved["block"])
LOADED_IN = time.time() - started
print(f"[load] ready in {LOADED_IN:.0f}s (region head on block {HEAD_BLOCK})", flush=True)
except Exception as error:
traceback.print_exc()
LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`"
return LOAD_ERROR
load_models()
# ββ Remote conditioning ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
@cache
def conditioner():
from gradio_client import Client
return Client(CONDITIONER_SPACE)
def conditioner_client(ip_token):
if not ip_token:
return conditioner()
from gradio_client import Client
return Client(CONDITIONER_SPACE, headers={"x-ip-token": ip_token})
def encode_remote(prompt, ip_token=None):
"""Call the conditioner Space for the layer-50 Qwen3-VL states ``two_track_sample`` consumes."""
from safetensors import safe_open
def call():
return conditioner_client(ip_token).predict(
prompt=prompt,
image_path=None,
last_image_path=None,
canvas=f"{CANVAS_W}x{CANVAS_H} Β· 16:9 full",
num_frames=CONDITIONER_FRAMES,
rewrite_prompt=False,
api_name="/encode",
)
try:
path, plan = call()
except Exception as first:
print(f"[cond] retrying with a fresh client after: {first}", flush=True)
conditioner.cache_clear()
path, plan = call()
with safe_open(path, framework="pt") as handle:
return handle.get_tensor("prompt_embeds"), plan
def _caller_ip_token() -> str | None:
from gradio.context import LocalContext
request = LocalContext.request.get()
return request.headers.get("x-ip-token") if request is not None else None
# ββ GPU inference βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
@spaces.GPU(duration=60, size=GPU_SIZE) # measured 33s end to end (sampler 32s); 60 covers a cold
# worker attach + the sliders' max settings (steps 30)
@torch.no_grad() # render() is grad-free end to end; without this the VAE decode of the merge
# builds a full autograd graph (~24 GB of retained activations) and OOMs the worker
def _generate(prompt_embeds, seed, steps, switch, hr_steps, feather, alpha, use_head, box):
"""Run the four-stage two-track sampler and compose to pixels, all on GPU time.
Mirrors ``two_track_h3.render()`` with ``--preset quality``: 20 steps of which the
first 16 run at half resolution, the region head picks M from block-25 states at the
switch, the HR track takes 6 linear-sigma steps on M only (``switch_upscale="crop"``
round-trips just the bounding box through the VAE), and the merge happens in pixel
space (``final="pixel"``) with a feather of 4 cells.
``two_track_sample`` has three return shapes: when the region comes back empty (or
switch >= steps) it returns only the plain LR latent, so the pixel merge must be
guarded exactly the way ``render()`` guards it.
"""
import torch
from two_track_h3 import pixel_compose, save_clip, two_track_sample
from h3_musubi import latent_frames
latent_t = latent_frames(NUM_FRAMES)
head = HEAD if use_head else None
box = tuple(float(v) for v in box) if (box and not use_head) else None
embeds = prompt_embeds.to("cuda", torch.bfloat16)
latent, info = two_track_sample(
DIT, embeds, UPSCALE,
height=CANVAS_H, width=CANVAS_W, steps=int(steps), switch=int(switch), seed=int(seed),
maps=MAPS, head=head, box=box, block=HEAD_BLOCK,
alpha=float(alpha), feather=float(feather), hr_steps=int(hr_steps),
latent_t=latent_t, switch_upscale="crop", final="pixel",
)
# Stage boundary, as in the vendored code: the sampler's dual-resolution activations are
# freed on return but the caching allocator still holds their segments β hand them back
# so the merge starts from a clean pool. (The full-clip decode OOM here on the first two
# live runs was the missing no_grad on this function: the VAE decode was retaining a
# ~24 GB autograd graph that render()'s @torch.no_grad() never builds.)
torch.cuda.empty_cache()
if "lr" in info: # render()'s guard: the merge inputs exist only on the two-track path
pixels = pixel_compose(UPSCALE.vae, latent, info["lr"], info["weight"])
else:
pixels = UPSCALE.vae.decode(latent.to("cuda", UPSCALE.dtype)).to("cpu", torch.float16)
out_path = os.path.join(OUTPUT_DIR, f"mpeg-diffusion-{int(time.time() * 1000)}.mp4")
save_clip(pixels, Path(out_path), fps=FPS)
share = float(info["mask"].float().mean()) if info.get("mask") is not None else 0.0
return out_path, share
def generate(
prompt,
use_head=True,
box_x0=0.25,
box_y0=0.25,
box_x1=0.75,
box_y1=0.75,
seed=0,
steps=DEFAULT_STEPS,
switch=DEFAULT_SWITCH,
hr_steps=DEFAULT_HR_STEPS,
feather=DEFAULT_FEATHER,
alpha=DEFAULT_ALPHA,
progress=gr.Progress(track_tqdm=True),
):
"""One request: prompt -> mixed-resolution video. Conditioning runs on the conditioner
Space (outside GPU booking, charged to the caller's quota via x-ip-token); the sampler
runs on this Space's ZeroGPU worker."""
if LOAD_ERROR:
raise gr.Error(LOAD_ERROR.replace("**", "").replace("`", ""))
if DIT is None:
raise gr.Error("The model is still loading β watch the Space logs and retry shortly.")
if not prompt or not prompt.strip():
raise gr.Error("A prompt is required.")
if int(switch) >= int(steps):
raise gr.Error("The region switch step must come before the last step.")
box = (float(box_x0), float(box_y0), float(box_x1), float(box_y1))
if box[0] >= box[2] or box[1] >= box[3]:
raise gr.Error("The manual region box needs x1 > x0 and y1 > y0.")
progress(0.0, desc="Encoding the prompt ...")
prompt_embeds, plan = encode_remote(prompt, ip_token=_caller_ip_token())
progress(0.1, desc=f"Two-track sampling {CANVAS_W}x{CANVAS_H} x {NUM_FRAMES} frames ...")
started = time.time()
out_path, share = _generate(
prompt_embeds, int(seed), int(steps), int(switch), int(hr_steps),
float(feather), float(alpha), bool(use_head), box,
)
seconds = time.time() - started
region = "region head" if use_head else f"manual box {tuple(round(v, 2) for v in box)}"
report = (
f"{CANVAS_W}x{CANVAS_H} Β· {NUM_FRAMES} frames ({NUM_FRAMES / FPS:.2f}s) Β· "
f"{int(steps)} steps, switch at {int(switch)}, {int(hr_steps)} HR steps Β· "
f"HR region: {region} covering {share:.0%} of tokens Β· "
f"conditioner {plan.get('seconds', '?')}s Β· sampler {seconds:.0f}s Β· seed {int(seed)}"
)
print(f"[gen] {report}", flush=True)
return out_path, report
# ββ UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
INTRO = """# MPEG-Diffusion: Mixed-Resolution Video Generation
<div align="center">
<a href="https://huggingface.co/papers/2611.12069" target="_blank" rel="noopener"><strong>[ paper ]</strong></a>
<a href="https://github.com/scrya-com/mpeg-diffusion" target="_blank" rel="noopener"><strong>[ code ]</strong></a>
<a href="https://huggingface.co/johndpope/MiniMax-H3-MPEG-Diffusion" target="_blank" rel="noopener"><strong>[ weights ]</strong></a>
</div>
**MPEG-Diffusion** (Menn et al.) generates video the way video codecs think about it: a
**low-resolution track** denoises the whole frame cheaply, a tiny linear head **self-selects
the region that deserves detail**, and a **high-resolution track** refines only that region
through dual-resolution attention. The tracks merge in pixel space β full-res detail where it
matters, ~2x the speed of rendering everything at high resolution.
This Space runs the paper's MiniMax-H3 port with the published research weights: the fitted
KV up-maps and the v3 region head on the int8 ConvRot base DiT, at the port's *quality preset*
(20 steps, region chosen at step 16, 6 HR steps, pixel-space merge). By default the model
picks its own detail region; force one with a manual box in *Advanced*."""
CSS = """
.main.fillable { max-width: 1100px !important; }
"""
EXAMPLE_PROMPTS = [
"A portrait of a young woman holding a red hibiscus flower against a rustic background",
"A joyful woman with colorful flowers in her afro, exuding happiness and vibrant style",
"A bearded man in a bathrobe uses a smartphone under studio lighting with a soft focus",
"A fashionable man in checkered pants and cowboy hat poses outdoors on concrete blocks",
"A serene portrait of a woman with curly hair surrounded by autumn foliage and berries",
]
with gr.Blocks(title="MPEG-Diffusion") as demo:
gr.Markdown(INTRO)
with gr.Row(equal_height=True):
with gr.Column():
prompt = gr.Textbox(
label="Prompt",
lines=3,
value=EXAMPLE_PROMPTS[0],
placeholder="Describe the scene β a clear subject helps the region head pick well ...",
)
use_head = gr.Checkbox(
label="Let the model choose the detail region (recommended)",
value=True,
)
run = gr.Button("Generate", variant="primary", size="lg")
with gr.Accordion("Advanced", open=False):
seed = gr.Number(label="Seed", value=0, precision=0)
with gr.Row():
steps = gr.Slider(label="Steps", minimum=10, maximum=30, step=1, value=DEFAULT_STEPS)
switch = gr.Slider(label="Region switch step", minimum=8, maximum=24, step=1, value=DEFAULT_SWITCH)
with gr.Row():
hr_steps = gr.Slider(label="HR steps", minimum=2, maximum=10, step=1, value=DEFAULT_HR_STEPS)
feather = gr.Slider(label="Feather (cells)", minimum=0, maximum=8, step=1, value=DEFAULT_FEATHER)
with gr.Row():
alpha = gr.Slider(label="Region threshold Ξ±", minimum=0.1, maximum=0.9, step=0.05, value=DEFAULT_ALPHA)
with gr.Accordion("Manual region (used when the checkbox is off)", open=False):
with gr.Row():
box_x0 = gr.Slider(label="Left x0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
box_y0 = gr.Slider(label="Top y0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
with gr.Row():
box_x1 = gr.Slider(label="Right x1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)
box_y1 = gr.Slider(label="Bottom y1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)
with gr.Column():
result = gr.Video(label="Mixed-resolution video", autoplay=True)
report = gr.Markdown()
gr.Examples(
examples=[[p] for p in EXAMPLE_PROMPTS],
inputs=[prompt],
fn=generate,
outputs=[result, report],
cache_examples=True,
cache_mode="lazy",
)
inputs = [prompt, use_head, box_x0, box_y0, box_x1, box_y1, seed, steps, switch, hr_steps, feather, alpha]
run.click(generate, inputs, [result, report], api_name="generate")
prompt.submit(generate, inputs, [result, report], api_name="generate_submit")
if __name__ == "__main__":
demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, allowed_paths=[OUTPUT_DIR])
|