File size: 17,920 Bytes
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bb6cd86
 
 
 
 
6febba8
bb6cd86
 
 
 
 
 
 
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bb6cd86
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
929cee3
 
81df4d4
 
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81df4d4
 
 
 
 
d2db40e
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
"""MPEG-Diffusion β€” mixed-resolution text-to-video generation on MiniMax-H3.

This Space implements the MPEG-Diffusion inference recipe end to end: a low-resolution track
denoises the whole frame, a linear region head self-selects where detail matters, and a
high-resolution track refines only that region through dual-resolution attention with
ridge-fitted up-maps, before the two tracks are merged in pixel space.

The reference implementation is the paper's port for MiniMax-H3
(github.com/scrya-com/mpeg-diffusion, Apache-2.0), vendored here as ``lot/`` plus a
subset of its musubi-tuner (``third_party/musubi_tuner/``). The weights are the published
research artifacts (johndpope/MiniMax-H3-MPEG-Diffusion): the fitted KV up-maps and the
v3 region head, applied to the *untrained* ComfyUI int8 ConvRot base DiT β€” exactly the
"quality preset" the port's README documents (`--preset quality`: steps 20, switch 16,
hr_steps 6, switch_upscale="crop", final="pixel", feather 4.0, alpha 0.5, block 25).

The 62 GiB Qwen3-VL text encoder does not fit beside the transformer on a single ZeroGPU
worker, so conditioning is delegated to ``multimodalart/qwen3vl-conditioner`` over the
gradio API β€” the same split every MiniMax-H3 Space in this org uses. Its ``/encode``
returns the layer-50 hidden state ``prompt_embeds (1, L, 5120) bfloat16``, which is
precisely the ``text`` input ``two_track_sample`` consumes.
"""

from __future__ import annotations

import os

# expandable_segments avoids the fragmentation the 21 GB int8 streaming load would otherwise
# hit; must be set before torch is imported. Same for the muted triton autotune cache dir.
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
# The int8 Triton kernels autotune; keep the cache on the ephemeral disk, not a read-only layer.
os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton-cache")

import spaces  # noqa: E402  β€” FIRST, before any CUDA-touching import

import sys
import tempfile
import time
import traceback
from functools import cache
from pathlib import Path

import gradio as gr
import torch

SPACE_ROOT = Path(__file__).resolve().parent

# The weights the Space downloads at runtime. The paths are exported *before* any ``lot``
# import because ``lot_paths`` reads them at import time (``procrustes_h3`` reads
# ``LOT_H3_CHECKPOINT`` the moment it is imported).
BASE_REPO = "Comfy-Org/MiniMax-H3"
DIT_FILE = "diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors"
VAE_FILE = "vae/minimax_h3_video_vae_fp16.safetensors"
TT_REPO = "johndpope/MiniMax-H3-MPEG-Diffusion"
MAPS_FILE = "kv_maps/up_maps_tt.safetensors"
REGION_FILE = "region_head_v3/region_head_tt.safetensors"

CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")

# The quality preset from the port's README, pinned as defaults. 1344x768 is the canvas the
# maps and the head were fitted/validated on. 22 pixel frames (latent_t 7, 0.92 s at 24 fps)
# is the render length the port itself measures and ships: peak 26.8 GB with every block
# resident. Longer clips blow up the HR dual-resolution attention β€” the author's own note
# says 73 pixel frames OOM'd a 32 GB 5090 without streaming, and 56 frames OOM'd this
# Space's 48 GB worker on the first live run.
CANVAS_W, CANVAS_H = 1344, 768
NUM_FRAMES = 22
# The conditioner enforces a 2 s floor on num_frames, but its prompt_embeds are text-only
# states that do NOT depend on clip length (its _encode never sees num_frames β€” the value
# only gates validation and the optional prompt rewrite). So the /encode request passes 56
# (2.33 s, the shortest grid value over the floor) purely to clear that check, and the
# sampler renders the 22-frame reference clip from those same states.
CONDITIONER_FRAMES = 56
FPS = 24
DEFAULT_STEPS, DEFAULT_SWITCH, DEFAULT_HR_STEPS = 20, 16, 6
DEFAULT_FEATHER, DEFAULT_ALPHA = 4.0, 0.5

OUTPUT_DIR = tempfile.mkdtemp(prefix="mpeg-diffusion-out-")

GPU_SIZE = "large"

DIT: torch.nn.Module | None = None
UPSCALE = None  # lot.two_track_h3.VaeUpscaler
MAPS = None  # lot.mixed_res.UpMaps
HEAD = None  # lot.mixed_res.RegionHead
HEAD_BLOCK = 25
LOAD_ERROR: str | None = None
LOADED_IN: float | None = None


# ── Model loading (module scope, eager) ─────────────────────────────────────


def load_models() -> str | None:
    """Load DiT + VAE + up-maps + region head at startup, `.to("cuda")` so the ZeroGPU
    startup pack streams them disk->VRAM on a cold worker (26 GB total)."""
    global DIT, UPSCALE, MAPS, HEAD, HEAD_BLOCK, LOAD_ERROR, LOADED_IN
    if DIT is not None or LOAD_ERROR is not None:
        return LOAD_ERROR
    started = time.time()
    try:
        from huggingface_hub import hf_hub_download

        os.environ["LOT_H3_CHECKPOINT"] = hf_hub_download(BASE_REPO, DIT_FILE)
        os.environ["LOT_H3_VAE"] = hf_hub_download(BASE_REPO, VAE_FILE)
        maps_path = hf_hub_download(TT_REPO, MAPS_FILE)
        region_path = hf_hub_download(TT_REPO, REGION_FILE)

        sys.path.insert(0, str(SPACE_ROOT / "lot"))
        sys.path.insert(0, str(SPACE_ROOT / "third_party"))
        from two_track_h3 import load_maps, load_region, load_upscaler  # lot/ is on sys.path

        from h3_musubi import load_dit

        print(f"[load] int8 ConvRot DiT from {BASE_REPO}/{DIT_FILE} ...", flush=True)
        DIT = load_dit(os.environ["LOT_H3_CHECKPOINT"], device="cuda", swap=0)
        print(f"[load] DiT resident ({time.time() - started:.0f}s), loading VAE ...", flush=True)
        UPSCALE = load_upscaler()
        print(f"[load] VAE resident, loading fitted up-maps + region head ...", flush=True)
        MAPS = load_maps(Path(maps_path))
        saved = load_region(Path(region_path))
        from mixed_res import RegionHead

        HEAD = RegionHead(saved["hidden"]).cuda().eval()
        HEAD.load_state_dict(saved["state"])
        HEAD_BLOCK = int(saved["block"])
        LOADED_IN = time.time() - started
        print(f"[load] ready in {LOADED_IN:.0f}s (region head on block {HEAD_BLOCK})", flush=True)
    except Exception as error:
        traceback.print_exc()
        LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`"
    return LOAD_ERROR


load_models()


# ── Remote conditioning ──────────────────────────────────────────────────────


@cache
def conditioner():
    from gradio_client import Client

    return Client(CONDITIONER_SPACE)


def conditioner_client(ip_token):
    if not ip_token:
        return conditioner()
    from gradio_client import Client

    return Client(CONDITIONER_SPACE, headers={"x-ip-token": ip_token})


def encode_remote(prompt, ip_token=None):
    """Call the conditioner Space for the layer-50 Qwen3-VL states ``two_track_sample`` consumes."""
    from safetensors import safe_open

    def call():
        return conditioner_client(ip_token).predict(
            prompt=prompt,
            image_path=None,
            last_image_path=None,
            canvas=f"{CANVAS_W}x{CANVAS_H} Β· 16:9 full",
            num_frames=CONDITIONER_FRAMES,
            rewrite_prompt=False,
            api_name="/encode",
        )

    try:
        path, plan = call()
    except Exception as first:
        print(f"[cond] retrying with a fresh client after: {first}", flush=True)
        conditioner.cache_clear()
        path, plan = call()

    with safe_open(path, framework="pt") as handle:
        return handle.get_tensor("prompt_embeds"), plan


def _caller_ip_token() -> str | None:
    from gradio.context import LocalContext

    request = LocalContext.request.get()
    return request.headers.get("x-ip-token") if request is not None else None


# ── GPU inference ─────────────────────────────────────────────────────────────


@spaces.GPU(duration=60, size=GPU_SIZE)  # measured 33s end to end (sampler 32s); 60 covers a cold
                                          # worker attach + the sliders' max settings (steps 30)
@torch.no_grad()  # render() is grad-free end to end; without this the VAE decode of the merge
                  # builds a full autograd graph (~24 GB of retained activations) and OOMs the worker
def _generate(prompt_embeds, seed, steps, switch, hr_steps, feather, alpha, use_head, box):
    """Run the four-stage two-track sampler and compose to pixels, all on GPU time.

    Mirrors ``two_track_h3.render()`` with ``--preset quality``: 20 steps of which the
    first 16 run at half resolution, the region head picks M from block-25 states at the
    switch, the HR track takes 6 linear-sigma steps on M only (``switch_upscale="crop"``
    round-trips just the bounding box through the VAE), and the merge happens in pixel
    space (``final="pixel"``) with a feather of 4 cells.

    ``two_track_sample`` has three return shapes: when the region comes back empty (or
    switch >= steps) it returns only the plain LR latent, so the pixel merge must be
    guarded exactly the way ``render()`` guards it.
    """
    import torch

    from two_track_h3 import pixel_compose, save_clip, two_track_sample
    from h3_musubi import latent_frames

    latent_t = latent_frames(NUM_FRAMES)
    head = HEAD if use_head else None
    box = tuple(float(v) for v in box) if (box and not use_head) else None

    embeds = prompt_embeds.to("cuda", torch.bfloat16)

    latent, info = two_track_sample(
        DIT, embeds, UPSCALE,
        height=CANVAS_H, width=CANVAS_W, steps=int(steps), switch=int(switch), seed=int(seed),
        maps=MAPS, head=head, box=box, block=HEAD_BLOCK,
        alpha=float(alpha), feather=float(feather), hr_steps=int(hr_steps),
        latent_t=latent_t, switch_upscale="crop", final="pixel",
    )
    # Stage boundary, as in the vendored code: the sampler's dual-resolution activations are
    # freed on return but the caching allocator still holds their segments β€” hand them back
    # so the merge starts from a clean pool. (The full-clip decode OOM here on the first two
    # live runs was the missing no_grad on this function: the VAE decode was retaining a
    # ~24 GB autograd graph that render()'s @torch.no_grad() never builds.)
    torch.cuda.empty_cache()
    if "lr" in info:  # render()'s guard: the merge inputs exist only on the two-track path
        pixels = pixel_compose(UPSCALE.vae, latent, info["lr"], info["weight"])
    else:
        pixels = UPSCALE.vae.decode(latent.to("cuda", UPSCALE.dtype)).to("cpu", torch.float16)
    out_path = os.path.join(OUTPUT_DIR, f"mpeg-diffusion-{int(time.time() * 1000)}.mp4")
    save_clip(pixels, Path(out_path), fps=FPS)
    share = float(info["mask"].float().mean()) if info.get("mask") is not None else 0.0
    return out_path, share


def generate(
    prompt,
    use_head=True,
    box_x0=0.25,
    box_y0=0.25,
    box_x1=0.75,
    box_y1=0.75,
    seed=0,
    steps=DEFAULT_STEPS,
    switch=DEFAULT_SWITCH,
    hr_steps=DEFAULT_HR_STEPS,
    feather=DEFAULT_FEATHER,
    alpha=DEFAULT_ALPHA,
    progress=gr.Progress(track_tqdm=True),
):
    """One request: prompt -> mixed-resolution video. Conditioning runs on the conditioner
    Space (outside GPU booking, charged to the caller's quota via x-ip-token); the sampler
    runs on this Space's ZeroGPU worker."""
    if LOAD_ERROR:
        raise gr.Error(LOAD_ERROR.replace("**", "").replace("`", ""))
    if DIT is None:
        raise gr.Error("The model is still loading β€” watch the Space logs and retry shortly.")
    if not prompt or not prompt.strip():
        raise gr.Error("A prompt is required.")
    if int(switch) >= int(steps):
        raise gr.Error("The region switch step must come before the last step.")
    box = (float(box_x0), float(box_y0), float(box_x1), float(box_y1))
    if box[0] >= box[2] or box[1] >= box[3]:
        raise gr.Error("The manual region box needs x1 > x0 and y1 > y0.")

    progress(0.0, desc="Encoding the prompt ...")
    prompt_embeds, plan = encode_remote(prompt, ip_token=_caller_ip_token())

    progress(0.1, desc=f"Two-track sampling {CANVAS_W}x{CANVAS_H} x {NUM_FRAMES} frames ...")
    started = time.time()
    out_path, share = _generate(
        prompt_embeds, int(seed), int(steps), int(switch), int(hr_steps),
        float(feather), float(alpha), bool(use_head), box,
    )
    seconds = time.time() - started

    region = "region head" if use_head else f"manual box {tuple(round(v, 2) for v in box)}"
    report = (
        f"{CANVAS_W}x{CANVAS_H} Β· {NUM_FRAMES} frames ({NUM_FRAMES / FPS:.2f}s) Β· "
        f"{int(steps)} steps, switch at {int(switch)}, {int(hr_steps)} HR steps Β· "
        f"HR region: {region} covering {share:.0%} of tokens Β· "
        f"conditioner {plan.get('seconds', '?')}s Β· sampler {seconds:.0f}s Β· seed {int(seed)}"
    )
    print(f"[gen] {report}", flush=True)
    return out_path, report


# ── UI ────────────────────────────────────────────────────────────────────────


INTRO = """# MPEG-Diffusion: Mixed-Resolution Video Generation

<div align="center">
  <a href="https://huggingface.co/papers/2611.12069" target="_blank" rel="noopener"><strong>[ paper ]</strong></a> &nbsp;
  <a href="https://github.com/scrya-com/mpeg-diffusion" target="_blank" rel="noopener"><strong>[ code ]</strong></a> &nbsp;
  <a href="https://huggingface.co/johndpope/MiniMax-H3-MPEG-Diffusion" target="_blank" rel="noopener"><strong>[ weights ]</strong></a>
</div>

**MPEG-Diffusion** (Menn et al.) generates video the way video codecs think about it: a
**low-resolution track** denoises the whole frame cheaply, a tiny linear head **self-selects
the region that deserves detail**, and a **high-resolution track** refines only that region
through dual-resolution attention. The tracks merge in pixel space β€” full-res detail where it
matters, ~2x the speed of rendering everything at high resolution.

This Space runs the paper's MiniMax-H3 port with the published research weights: the fitted
KV up-maps and the v3 region head on the int8 ConvRot base DiT, at the port's *quality preset*
(20 steps, region chosen at step 16, 6 HR steps, pixel-space merge). By default the model
picks its own detail region; force one with a manual box in *Advanced*."""

CSS = """
.main.fillable { max-width: 1100px !important; }
"""

EXAMPLE_PROMPTS = [
    "A portrait of a young woman holding a red hibiscus flower against a rustic background",
    "A joyful woman with colorful flowers in her afro, exuding happiness and vibrant style",
    "A bearded man in a bathrobe uses a smartphone under studio lighting with a soft focus",
    "A fashionable man in checkered pants and cowboy hat poses outdoors on concrete blocks",
    "A serene portrait of a woman with curly hair surrounded by autumn foliage and berries",
]

with gr.Blocks(title="MPEG-Diffusion") as demo:
    gr.Markdown(INTRO)

    with gr.Row(equal_height=True):
        with gr.Column():
            prompt = gr.Textbox(
                label="Prompt",
                lines=3,
                value=EXAMPLE_PROMPTS[0],
                placeholder="Describe the scene β€” a clear subject helps the region head pick well ...",
            )
            use_head = gr.Checkbox(
                label="Let the model choose the detail region (recommended)",
                value=True,
            )
            run = gr.Button("Generate", variant="primary", size="lg")

            with gr.Accordion("Advanced", open=False):
                seed = gr.Number(label="Seed", value=0, precision=0)
                with gr.Row():
                    steps = gr.Slider(label="Steps", minimum=10, maximum=30, step=1, value=DEFAULT_STEPS)
                    switch = gr.Slider(label="Region switch step", minimum=8, maximum=24, step=1, value=DEFAULT_SWITCH)
                with gr.Row():
                    hr_steps = gr.Slider(label="HR steps", minimum=2, maximum=10, step=1, value=DEFAULT_HR_STEPS)
                    feather = gr.Slider(label="Feather (cells)", minimum=0, maximum=8, step=1, value=DEFAULT_FEATHER)
                with gr.Row():
                    alpha = gr.Slider(label="Region threshold Ξ±", minimum=0.1, maximum=0.9, step=0.05, value=DEFAULT_ALPHA)
                with gr.Accordion("Manual region (used when the checkbox is off)", open=False):
                    with gr.Row():
                        box_x0 = gr.Slider(label="Left x0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
                        box_y0 = gr.Slider(label="Top y0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
                    with gr.Row():
                        box_x1 = gr.Slider(label="Right x1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)
                        box_y1 = gr.Slider(label="Bottom y1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)

        with gr.Column():
            result = gr.Video(label="Mixed-resolution video", autoplay=True)
            report = gr.Markdown()

    gr.Examples(
        examples=[[p] for p in EXAMPLE_PROMPTS],
        inputs=[prompt],
        fn=generate,
        outputs=[result, report],
        cache_examples=True,
        cache_mode="lazy",
    )

    inputs = [prompt, use_head, box_x0, box_y0, box_x1, box_y1, seed, steps, switch, hr_steps, feather, alpha]
    run.click(generate, inputs, [result, report], api_name="generate")
    prompt.submit(generate, inputs, [result, report], api_name="generate_submit")

if __name__ == "__main__":
    demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, allowed_paths=[OUTPUT_DIR])