#!/usr/bin/env python3 """MPEG-Diffusion's own arrangement on H3: a native LR track plus HR tokens on a selected region. python3 lot/two_track_h3.py render --out DIR [--steps 20 --switch 18 --hr-steps 6] [--maps DIR/up_maps_tt.pt] [--region DIR/region_head_tt.pt | --box 0.25 0.2 0.75 0.9] python3 lot/two_track_h3.py fit-kv --out DIR [--items 24] # ridge fit of f_up, base model at LR and HR python3 lot/two_track_h3.py region --out DIR [--items 200] # linear region head on the LR track's states Paper: Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu, Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu, MPEG-Diffusion: Mixed-Resolution Video Generation, arXiv:2611.12069. ``mixed_res_h3.py`` puts the paper's pieces on LoT's merged tokens, so its coarse region is only as good as the LoT adapter. Here the LR track is what the paper uses: the unmodified base model at half resolution (no adapter, no LoRA), which is good from the start. H3's rotary positions are normalised by ``sqrt(area)``, so the LR and HR grids already share one coordinate frame. Sampler (their Sec. 4, "Inference"): 1. ``--switch`` steps denoise the whole frame at LR (``W/2 x H/2``). 2. At the switch a region ``M`` is chosen on the LR token grid (``--region`` head, or ``--box``). 3. The clean LR estimate is upscaled, re-noised with fresh noise, and the remaining steps run one packed sequence ``[text, audio, LR video (whole grid), HR video (M only)]`` with dual-resolution attention (``mixed_res.two_track_attention``): LR queries see only LR keys, so the LR track stays the native model; HR queries see their own keys inside ``M`` and ``f_up`` of the LR keys outside. 4. The LR latent is upscaled and the HR latent is pasted over ``M``; the VAE decodes that. The upscaler ``U`` is the VAE itself (decode, bicubic x2, encode): it needs no training and returns a latent the decoder already agrees with. ``--feather`` blends the paste edge in latent space. Not trained here (the paper fine-tunes them): the DiT with the maps, the resolution embedding, ``f_down`` (unused: the LR track is complete), the learned upscaler and the mixer. """ from __future__ import annotations import argparse import contextlib import json import os import random import sys import time from dataclasses import replace from pathlib import Path import torch import torch.nn.functional as F sys.path.insert(0, str(Path(__file__).resolve().parent)) from gpu_guard import refuse_if_busy # noqa: E402 from mixed_res import (RegionHead, TwoTrackPlan, UpMaps, patched_attention, region_mask, # noqa: E402 two_track_attention) from procrustes_h3 import CHECKPOINT # noqa: E402 HEAD_DIM = 128 MID_BLOCK = 25 EXTENT = (1, 2, 2) # ------------------------------------------------------------------------- forward def two_track_forward(dit, lr: torch.Tensor, hr: torch.Tensor | None, plan: TwoTrackPlan | None, t: float, text: torch.Tensor, *, maps: UpMaps | None = None, audio_generator=None, capture: dict | None = None, res_embed: "ResolutionEmbedding | None" = None): """Velocities ``(v_lr, v_hr)`` of the LR latent and of the HR latent on the plan's region. ``lr`` is ``(1, 24, T, h, w)``, ``hr`` is ``(1, 24, T, 2h, 2w)``; ``v_hr`` is zero outside the region. With ``plan=None`` (or an empty region) this is the base model on ``lr`` alone. ``capture={"block": i}`` returns that block's LR video states in ``capture["states"]``; ``capture={"kv": {}}`` collects every block's pre-RoPE LR keys and values. ``res_embed`` adds the paper's learned LR / HR vectors to the video rows. With grad enabled (training) the attention patch is installed for good (``_training_patch``) so activation checkpointing recomputes the same attention in the backward pass. """ from h3_musubi import _layout, _silence, _sync_swap, audio_sigma from musubi_tuner.minimax_h3.packing import (H3RowSegment, _unpack_video_rows, build_position_grid, build_timestep_rows, pack_audio_rows, pack_video_rows) if text.ndim == 2: text = text.unsqueeze(0) _sync_swap(dit) device = dit.video_patch_proj.weight.device sigma_a = float(audio_sigma(1.0 - t)) _b, _c, frames, lat_h, lat_w = lr.shape layout = _layout(text.shape[1], frames, lat_h, lat_w) segment = layout.target_video_segment if segment.stop != layout.row_count: raise RuntimeError("the target video must be the last segment of the packed sequence") lr_count = segment.row_count hr_count = plan.hr_count if plan is not None else 0 if plan is not None and plan.lr_count != lr_count: raise ValueError(f"plan has {plan.lr_count} LR tokens, the latent has {lr_count}") rows = pack_video_rows(lr.to(device, torch.float32)) positions = build_position_grid(layout)[0] hr_rotation = None if hr_count: hr_layout = _layout(text.shape[1], frames, 2 * lat_h, 2 * lat_w) hr_positions = build_position_grid(hr_layout)[0][-4 * lr_count:] rows = torch.cat([rows, pack_video_rows(hr.to(device, torch.float32))[:, plan.hr_index]], dim=1) positions = torch.cat([positions, hr_positions[plan.hr_index.cpu()]]) video = dit.video_patch_proj(rows).to(dit.dtype) if res_embed is not None: video = res_embed(video, lr_count) audio = _silence(frames, sigma_a, device, audio_generator) audio_rows = dit.audio_patch_proj(pack_audio_rows(audio.to(device)).to(torch.float32)).to(dit.dtype) text_rows = dit.token_refiner(dit.condition_proj(text.to(device, dit.dtype))) hidden = torch.cat([text_rows, audio_rows, video], dim=1) segments = layout.segments[:-1] + (H3RowSegment(segment.role, segment.kind, segment.start, segment.start + lr_count + hr_count),) packed = replace(layout, segments=segments, row_count=segment.start + lr_count + hr_count) rotation = dit._rotation_table(positions.unsqueeze(0).to(device), hidden.dtype) tags = torch.ones(1, text.shape[1], dtype=torch.int64) timesteps = build_timestep_rows(packed, text_token_tags=tags, model_t_video=t, model_t_audio=1.0 - sigma_a) temb = dit._timestep_embeddings(timesteps.unique_timesteps, device) patch = contextlib.nullcontext() want_kv = capture is not None and "kv" in capture training = torch.is_grad_enabled() persistent = training or getattr(dit, "_two_track_context", None) is not None if hr_count or want_kv or persistent: if hr_count: hr_rotation = dit._rotation_table(hr_positions.unsqueeze(0).to(device), hidden.dtype) context = dict(plan=plan if plan is not None and hr_count else _empty_plan(frames, lat_h, lat_w, device), maps=maps, video_start=segment.start, hr_rotation=hr_rotation, capture=capture.get("kv") if want_kv else None) if persistent: _training_patch(dit).update(context) else: patch = patched_attention(dit.blocks, attention=two_track_attention, **context) checkpoint = training and dit.gradient_checkpointing with patch: for index, block in enumerate(dit.blocks): if dit.blocks_to_swap: dit.offloader.wait_for_block(index) if checkpoint: hidden = torch.utils.checkpoint.checkpoint(block, hidden, temb, timesteps.block_segments, rotation, use_reentrant=False) else: hidden = block(hidden, temb, timesteps.block_segments, rotation) if dit.blocks_to_swap: dit.offloader.submit_move_blocks_forward(dit.blocks, index) if capture is not None and capture.get("block") == index: states = hidden[0, segment.start:segment.start + lr_count].float() capture["states"] = states if capture.get("grad") else states.detach() hidden = hidden.to(device) final = dit.final_layer shift, scale = final.adaln_proj(temb) index = timesteps.video_timestep_index states = final.norm(hidden[:, segment.start:]) * (1.0 + scale[index]) + shift[index] out = final.video_out(states.to(torch.float32)) v_lr = _unpack_video_rows(out[:, :lr_count], layout.target_video).to(lr.dtype) if not hr_count: return v_lr, None full = out.new_zeros(1, 4 * lr_count, out.shape[-1]) full[:, plan.hr_index] = out[:, lr_count:] return v_lr, _unpack_video_rows(full, hr_layout.target_video).to(hr.dtype) def _empty_plan(frames: int, lat_h: int, lat_w: int, device) -> TwoTrackPlan: return TwoTrackPlan(torch.zeros(frames, lat_h // 2, lat_w // 2, dtype=torch.bool)).to(device) def _training_patch(dit) -> dict: """Route every block's attention through ``two_track_attention`` for the life of the process and return the shared context dict. A context manager would be gone before the backward pass, and checkpointed blocks recompute attention there.""" context = getattr(dit, "_two_track_context", None) if context is None: import types context = {} for layer, block in enumerate(dit.blocks): def forward(self, hidden, rotation=None, _layer=layer): return two_track_attention(self, hidden, rotation, layer=_layer, **context) block.attn.forward = types.MethodType(forward, block.attn) dit._two_track_context = context context.clear() return context class ResolutionEmbedding(torch.nn.Module): """The paper's resolution embedding: one learned vector for LR rows and one for HR rows, zero at init.""" def __init__(self, hidden_size: int): super().__init__() self.lr = torch.nn.Parameter(torch.zeros(hidden_size)) self.hr = torch.nn.Parameter(torch.zeros(hidden_size)) def forward(self, video: torch.Tensor, lr_count: int) -> torch.Tensor: add = torch.cat([self.lr.expand(lr_count, -1), self.hr.expand(video.shape[1] - lr_count, -1)]) return video + add.to(video.dtype).unsqueeze(0) # ------------------------------------------------------------------------ upscaler class VaeUpscaler: """``U``: decode, bicubic x2 per frame, encode. A still or a clip (any latent frame count).""" def __init__(self, vae): self.vae = vae self.dtype = next(vae.parameters()).dtype @torch.no_grad() def _resample(self, latent: torch.Tensor, **interp) -> torch.Tensor: # Long clips (16 s): the decoded clip leaves the GPU at once as fp16, the resize runs 16 frames at a time, # and only the fp16 result goes back for the encode, so the GPU never holds a float32 copy of the clip. pixels = self.vae.decode(latent.to("cuda", self.dtype)).clamp(-1, 1).to("cpu", torch.float16) if latent.shape[2] == 1: pixels = pixels[:, :, :1] b, c, f, h, w = pixels.shape resized = None for start in range(0, f, 16): chunk = pixels[0, :, start:start + 16].transpose(0, 1).to("cuda", torch.float32) # (n, 3, h, w) chunk = F.interpolate(chunk, **interp).clamp(-1, 1).to("cpu", torch.float16) if resized is None: resized = torch.empty(1, c, f, *chunk.shape[-2:], dtype=torch.float16) resized[0, :, start:start + chunk.shape[0]] = chunk.transpose(0, 1) del pixels torch.cuda.empty_cache() pixels = resized.to("cuda", self.dtype) out = self.vae.encode(pixels[:, :, 0] if f == 1 else pixels).float() if out.shape[2] != latent.shape[2]: raise RuntimeError(f"VAE round trip changed latent frames {latent.shape[2]} -> {out.shape[2]}") return out def __call__(self, latent: torch.Tensor) -> torch.Tensor: return self._resample(latent, scale_factor=2, mode="bicubic", align_corners=False) def down(self, latent: torch.Tensor) -> torch.Tensor: """``D``: decode, area x0.5, encode (the LR latent of a cached HR item).""" return self._resample(latent, scale_factor=0.5, mode="area") def mask_to_latent(mask: torch.Tensor, scale: int, feather: float = 0.0) -> torch.Tensor: """LR-token mask ``(T, h, w)`` -> ``(1, 1, T, h*scale, w*scale)`` float, optionally feathered inward. The feather is a ramp inside the region only (outside it the HR track was never denoised): weight ``1/R`` on the edge cell, rising by ``1/R`` per cell of depth, 1 from depth ``R - 1`` on. Every region cell keeps weight > 0, and a region at least ``2R`` cells across reaches full weight in its middle. (The old erode-then-blur left 40% of a 30% box below full weight and zeroed regions 2 tokens wide.)""" up = mask.float().repeat_interleave(scale, 1).repeat_interleave(scale, 2) radius = int(round(feather)) if radius > 1: level = up[:, None] total = level.clone() for _ in range(radius - 1): level = -F.max_pool2d(-level, 3, stride=1, padding=1) # erode one cell (edge of frame counts as inside) total = total + level up = (total / radius)[:, 0] return up[None, None] # ------------------------------------------------------------------------- sampler def box_mask(box, frames: int, height: int, width: int) -> torch.Tensor: x0, y0, x1, y1 = box mask = torch.zeros(frames, height, width, dtype=torch.bool) mask[:, round(y0 * height):round(y1 * height), round(x0 * width):round(x1 * width)] = True return mask @torch.no_grad() def native_sample(dit, text, *, height, width, steps, seed, stop=None, capture_at=None, block=MID_BLOCK, res_embed=None, latent_t=1): """Base-model Euler sampling; returns ``(x, x0_last, states)`` after ``stop`` steps (default all).""" from h3_musubi import sigma_schedule gen = torch.Generator("cuda").manual_seed(seed) x = torch.randn(1, 24, latent_t, height // 16, width // 16, device="cuda", generator=gen) audio_gen = torch.Generator("cuda").manual_seed(seed + 1) sigmas = sigma_schedule(steps) x0 = states = None for index in range(steps if stop is None else stop): sigma, sigma_next = float(sigmas[index]), float(sigmas[index + 1]) capture = {"block": block} if capture_at == index else None velocity, _ = two_track_forward(dit, x, None, None, 1.0 - sigma, text, audio_generator=audio_gen, capture=capture, res_embed=res_embed) if capture is not None: states = capture["states"] x0 = x + sigma * velocity.float() x = x0 + (sigma_next / sigma) * (x - x0) return x, x0, states @torch.no_grad() def two_track_sample(dit, text, upscale, *, height, width, steps, switch, seed, maps=None, head=None, box=None, block=MID_BLOCK, alpha=0.5, floor=None, feather=0.0, on_mask=None, hr_steps=None, res_embed=None, decoder=None, latent_t=1, switch_upscale="vae", final="latent"): """The four-stage sampler of the module docstring. Returns ``(latent, info)`` at ``width x height``. ``decoder`` is ``(U, G)`` from ``train_tt.py decoder``: ``U`` replaces the VAE upscale (at the switch and at the end) and ``G`` merges the tracks (their eqs. 5-6) instead of the feathered paste.""" from h3_musubi import sigma_schedule lat_h, lat_w = height // 32, width // 32 # LR latent; also the HR token grid tok_h, tok_w = lat_h // 2, lat_w // 2 # LR token grid sigmas = sigma_schedule(steps) lr, x0_lr, states = native_sample(dit, text, height=height // 2, width=width // 2, steps=steps, seed=seed, stop=switch, capture_at=switch - 1 if head is not None else None, block=block, res_embed=res_embed, latent_t=latent_t) up = upscale if decoder is None else decoder[0] if head is not None: scores = head(states).float().cpu().reshape(latent_t, tok_h, tok_w) mask = region_mask(scores, alpha=alpha, floor=floor) else: mask = box_mask(box or (0.25, 0.25, 0.75, 0.75), latent_t, tok_h, tok_w) if on_mask is not None: on_mask(mask) tokens = [tok_h * tok_w * latent_t] * switch if not mask.any() or switch >= steps: # No region: finish the LR schedule with the base model, then upscale the clean latent. lr_gen = torch.Generator("cuda").manual_seed(seed + 3) for index in range(switch, steps): sigma, sigma_next = float(sigmas[index]), float(sigmas[index + 1]) velocity, _ = two_track_forward(dit, lr, None, None, 1.0 - sigma, text, audio_generator=lr_gen, res_embed=res_embed) x0 = lr + sigma * velocity.float() lr = x0 + (sigma_next / sigma) * (lr - x0) tokens.append(tok_h * tok_w * latent_t) return up(lr), {"mask": mask, "tokens": tokens} plan = TwoTrackPlan(mask).to("cuda") gen = torch.Generator("cuda").manual_seed(seed + 2) audio_gen = torch.Generator("cuda").manual_seed(seed + 3) sigma = float(sigmas[switch]) # At the switch the HR latent is mostly noise (sigma 0.75 at 16 of 20 on shift 12), so a latent x2 interpolation # can stand in for the VAE round trip (5 s for 7 latent frames on a 5090): switch_upscale="latent". # switch_upscale="crop": the HR track only reads U(x0_lr) inside M, so round-trip just M's bounding box # (all frames, plus a margin) and fill the rest with a latent x2. if switch_upscale == "vae": start = up(x0_lr) else: start = F.interpolate(x0_lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False) if switch_upscale == "crop": box_lr = crop_box(mask, lat_h, lat_w, margin=2) if box_lr is not None: y0, y1, x0, x1 = box_lr start[..., 2 * y0:2 * y1, 2 * x0:2 * x1] = up(x0_lr[..., y0:y1, x0:x1].contiguous()) hr = (1.0 - sigma) * start + sigma * torch.randn(1, 24, latent_t, 2 * lat_h, 2 * lat_w, device="cuda", generator=gen) # After the switch both tracks share the schedule: the model's own, or ``hr_steps`` steps # linear in sigma (H3's shift 12 leaves few low-sigma steps, where the HR detail is made). tail = sigmas[switch:] if not hr_steps else torch.linspace(sigma, 0.0, hr_steps + 1, dtype=torch.float64) for index in range(len(tail) - 1): sigma, sigma_next = float(tail[index]), float(tail[index + 1]) v_lr, v_hr = two_track_forward(dit, lr, hr, plan, 1.0 - sigma, text, maps=maps, audio_generator=audio_gen, res_embed=res_embed) ratio = sigma_next / sigma lr = lr + (1.0 - ratio) * sigma * v_lr.float() hr = hr + (1.0 - ratio) * sigma * v_hr.float() tokens.append(plan.lr_count + plan.hr_count) if decoder is not None: upscaler, mixer = decoder inside = mask_to_latent(mask, 2).to("cuda") # LR latent grid # Outside M the HR latent is still switch-sigma noise (v_hr is zero there). D only feeds M, but the # VAE's receptive field reaches past the edge, so fill the outside with a latent x2 of the LR track. inside_hr = mask_to_latent(mask, 4).to("cuda") context = F.interpolate(lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False) clean_hr = inside_hr * hr + (1.0 - inside_hr) * context filled = (1.0 - inside) * lr + inside * upscale.down(clean_hr) # their eq. 5: fill M with D(z_h) latent = mixer(hr, upscaler(filled), mask_to_latent(mask, 4).to("cuda")) return latent, {"mask": mask, "tokens": tokens} weight = mask_to_latent(mask, 4, feather).to("cuda") if final == "pixel": # No round trip at the end: the caller decodes HR (outside M filled with a latent x2 of LR, so the # VAE sees a plausible neighbourhood at the edge) and LR at half size, and pastes in pixel space. inside = mask_to_latent(mask, 4).to("cuda") context = F.interpolate(lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False) return inside * hr + (1.0 - inside) * context, {"mask": mask, "tokens": tokens, "lr": lr, "weight": weight} latent = weight * hr + (1.0 - weight) * upscale(lr) return latent, {"mask": mask, "tokens": tokens} def crop_box(mask: torch.Tensor, lat_h: int, lat_w: int, margin: int = 2): """Union over frames of M's bounding box on the LR latent grid (one LR token = 2 latent px), grown by ``margin`` latent px and rounded out to even sides, as ``(y0, y1, x0, x1)``; None for an empty mask.""" rows = torch.nonzero(mask.any(0).any(1)).flatten() cols = torch.nonzero(mask.any(0).any(0)).flatten() if not len(rows): return None y0, y1 = max(0, 2 * int(rows[0]) - margin), min(lat_h, 2 * int(rows[-1]) + 2 + margin) x0, x1 = max(0, 2 * int(cols[0]) - margin), min(lat_w, 2 * int(cols[-1]) + 2 + margin) y0, x0 = y0 - y0 % 2, x0 - x0 % 2 y1, x1 = min(lat_h, y1 + y1 % 2), min(lat_w, x1 + x1 % 2) return y0, y1, x0, x1 def pixel_compose(vae, hr_latent: torch.Tensor, lr_latent: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: """Decode HR and LR, upscale the LR pixels x2 (bicubic, per frame), paste with ``weight`` (HR latent grid, ``(1, 1, T, H, W)``) mapped to pixels: x16 in space, each latent frame to the pixel frames it decodes to.""" from h3_musubi import pixel_frames # Long clips: each decode leaves the GPU as fp16 at once, and the paste runs ``chunk`` frames at a time, # so the GPU never holds more than one full clip (16 s at 1344x768 is 2.4 GB in fp16, 4.8 GB in fp32). dtype = next(vae.parameters()).dtype hr_px = vae.decode(hr_latent.to("cuda", dtype)).clamp(-1, 1).to("cpu", torch.float16) lr_px = vae.decode(lr_latent.to("cuda", dtype)).clamp(-1, 1).to("cpu", torch.float16) if hr_latent.shape[2] == 1: hr_px, lr_px = hr_px[:, :, :1], lr_px[:, :, :1] frames = hr_px.shape[2] frame_of = [k for k in range(hr_latent.shape[2]) for _ in range(pixel_frames(k + 1) - pixel_frames(k))] frame_of = (frame_of + [frame_of[-1]] * frames)[:frames] weight = weight[0, 0].float().to("cuda") # (T, H, W) on the HR latent grid out = torch.empty_like(hr_px) chunk = 16 for start in range(0, frames, chunk): stop = min(frames, start + chunk) lr = lr_px[0, :, start:stop].to("cuda", torch.float32).transpose(0, 1) # (n, 3, h, w) lr_up = F.interpolate(lr, scale_factor=2, mode="bicubic", align_corners=False).clamp(-1, 1) w_px = F.interpolate(weight[frame_of[start:stop]][:, None], scale_factor=16, mode="bilinear", align_corners=False) # (n, 1, H, W) hr = hr_px[0, :, start:stop].to("cuda", torch.float32).transpose(0, 1) out[0, :, start:stop] = (w_px * hr + (1.0 - w_px) * lr_up).transpose(0, 1).to("cpu", torch.float16) return out # -------------------------------------------------------------------------- phases def load_base(swap: int): from h3_musubi import load_dit return load_dit(CHECKPOINT, device="cuda", swap=swap) def load_maps(path: Path) -> UpMaps: """``up_maps_tt.pt`` from ``fit-kv``, or the published ``up_maps_tt.safetensors`` (same keys, layers / rank in the metadata).""" path = Path(path) if path.suffix == ".safetensors": from safetensors import safe_open from safetensors.torch import load_file with safe_open(str(path), "pt") as handle: meta = handle.metadata() or {} saved = {"state": load_file(str(path)), "layers": int(meta["layers"]), "rank": int(meta["rank"])} else: saved = torch.load(path, map_location="cpu", weights_only=False) maps = UpMaps(saved["layers"], HEAD_DIM, [EXTENT], rank=saved["rank"]) maps.load_state_dict(saved["state"]) return maps.to("cuda").eval() def load_trained(dit, train_dir: Path, tag: str = ""): """LoRA (on), up-maps and resolution embedding from a ``train_tt.py lora`` run. Returns ``(network, maps, res_embed)``.""" from safetensors import safe_open from h3_musubi import make_lora with safe_open(str(train_dir / f"lora{tag}.safetensors"), "pt") as handle: rank = int((handle.metadata() or {}).get("rank", 16)) network = make_lora(dit, rank) info = network.load_weights(str(train_dir / f"lora{tag}.safetensors")) if info.missing_keys: raise RuntimeError(f"LoRA load missed {len(info.missing_keys)} keys") network.to(device="cuda", dtype=torch.bfloat16).eval() published = train_dir / f"tt_state{tag}.safetensors" # as upload_tt_hf.py publishes it if published.is_file(): from safetensors.torch import load_file with safe_open(str(published), "pt") as handle: meta = handle.metadata() or {} flat = load_file(str(published)) saved = {"layers": int(meta["layers"]), "rank": int(meta["rank"]), "hidden": int(meta["hidden"]), "maps": {k[5:]: v for k, v in flat.items() if k.startswith("maps.")}, "res_embed": {k[10:]: v for k, v in flat.items() if k.startswith("res_embed.")}} else: saved = torch.load(train_dir / f"tt_state{tag}.pt", map_location="cpu", weights_only=False) maps = UpMaps(saved["layers"], HEAD_DIM, [EXTENT], rank=saved["rank"]) maps.load_state_dict(saved["maps"]) res_embed = ResolutionEmbedding(saved["hidden"]) res_embed.load_state_dict(saved["res_embed"]) return network, maps.to("cuda").eval(), res_embed.to("cuda").eval() def load_region(path: Path) -> dict: """``region_head_tt.pt`` (as ``train_tt.py region`` saves it) or ``.safetensors`` (as published).""" path = Path(path) if path.suffix == ".safetensors": from safetensors import safe_open from safetensors.torch import load_file with safe_open(str(path), "pt") as handle: meta = handle.metadata() or {} return {"state": load_file(str(path)), "hidden": int(meta["hidden"]), "block": int(meta["block"])} return torch.load(path, map_location="cpu", weights_only=False) def load_decoder(path: Path): from mixed_res import LatentUpscaler, TwoTrackMixer saved = torch.load(path, map_location="cpu", weights_only=False) upscaler, mixer = LatentUpscaler(width=saved["up_width"]), TwoTrackMixer(width=saved["mix_width"]) upscaler.load_state_dict(saved["upscaler"]) mixer.load_state_dict(saved["mixer"]) return upscaler.cuda().eval(), mixer.cuda().eval() def load_upscaler() -> VaeUpscaler: from h3_musubi import load_vae from lot_paths import LOT_H3_VAE return VaeUpscaler(load_vae(LOT_H3_VAE)) def held_prompts(args): from render_h3 import load_prompts from train_h3 import CACHE, list_items if getattr(args, "te", None): return load_prompts(args.prompts, list(args.te)) held = [te for _latent, te in list_items(args.cache or CACHE, args.holdout)[1]] return load_prompts(args.prompts, held) @torch.no_grad() def render(args) -> None: from PIL import Image, ImageDraw from h3_musubi import sample from render_h3 import to_image from h3_musubi import set_lora dit = load_base(args.swap) upscale = load_upscaler() maps = load_maps(args.maps) if args.maps else None network = res_embed = None if args.train: network, maps, res_embed = load_trained(dit, args.train, args.tag) decoder = load_decoder(args.decoder) if args.decoder else None head = None if args.region: saved = load_region(args.region) head = RegionHead(saved["hidden"]).cuda().eval() head.load_state_dict(saved["state"]) args.block = saved["block"] if args.train and saved.get("train") in (None, "None", ""): print("LOT_H3 kind=warning msg=region head was trained on the base model's states, but --train turns the " "LoRA on for the LR steps too; retrain it with train_tt.py region --train", flush=True) from h3_musubi import latent_frames width, height = args.size latent_t = latent_frames(args.frames) columns = [c for c in ("hr", "lr", "two_track") if c in args.variants] meta = {"timings": {c: [] for c in columns}, "tokens": {c: [] for c in columns}, "hr_share": []} args.out.mkdir(parents=True, exist_ok=True) rows = [] for index, (caption, embeds) in enumerate(held_prompts(args)): embeds = embeds.to("cuda", torch.bfloat16) row = [] for name in columns: mask = None torch.cuda.synchronize() start = time.perf_counter() if network is not None: set_lora(network, name == "two_track") # the baselines are the base model if name == "hr": latent = sample(dit, embeds, height=height, width=width, steps=args.steps, seed=args.seed + index, latent_t=latent_t) count = float((height // 32) * (width // 32) * latent_t) elif name == "lr": latent, _x0, _s = native_sample(dit, embeds, height=height // 2, width=width // 2, steps=args.steps, seed=args.seed + index, latent_t=latent_t) latent = upscale(latent) count = float((height // 64) * (width // 64) * latent_t) else: latent, info = two_track_sample( dit, embeds, upscale, height=height, width=width, steps=args.steps, switch=args.switch, seed=args.seed + index, maps=maps, head=head, box=args.box, block=args.block, alpha=args.alpha, floor=args.floor, feather=args.feather, hr_steps=args.hr_steps, res_embed=res_embed, decoder=decoder, latent_t=latent_t, switch_upscale=args.switch_upscale, final=args.final) mask = info["mask"] count = sum(info["tokens"]) / len(info["tokens"]) meta["hr_share"].append(round(float(mask.float().mean()), 4)) meta["tokens"][name].append(round(count, 1)) # Every variant is timed through its decode to pixels (sampling + VAE), the number a user waits for. if name == "two_track" and args.final == "pixel" and "lr" in info: pixels = pixel_compose(upscale.vae, latent, info["lr"], info["weight"]) else: pixels = upscale.vae.decode(latent.to("cuda", upscale.dtype)).to("cpu", torch.float16) torch.cuda.synchronize() meta["timings"][name].append(round(time.perf_counter() - start, 2)) meta["note"] = "timings include the VAE decode for every variant" if latent_t > 1: save_clip(pixels, args.out / f"p{index}_{name}.mp4") pixels = pixels[:, :, pixels.shape[2] // 2:pixels.shape[2] // 2 + 1] # the grid shows the middle frame image = to_image(pixels) if mask is not None and args.outline: image = outline(image, mask[mask.shape[0] // 2:mask.shape[0] // 2 + 1]) image.save(args.out / f"p{index}_{name}.png") row.append(image) print(f"LOT_H3 kind=two_track_render prompt={index} variant={name} " f"seconds={meta['timings'][name][-1]} mean_tokens={count:.0f}", flush=True) rows.append(row) scale = 0.5 cell = (int(width * scale), int(height * scale)) sheet = Image.new("RGB", (cell[0] * len(columns), 28 + cell[1] * len(rows)), "white") draw = ImageDraw.Draw(sheet) for col, label in enumerate(columns): draw.text((col * cell[0] + 6, 8), label, fill="black") for r, images in enumerate(rows): for col, image in enumerate(images): sheet.paste(image.resize(cell, Image.LANCZOS), (col * cell[0], 28 + r * cell[1])) sheet.save(args.out / "grid.png") meta.update({"columns": columns, "frames": args.frames, "latent_t": latent_t, "steps": args.steps, "switch": args.switch, "canvas": f"{width}x{height}", "maps": str(args.maps), "train": str(args.train), "decoder": str(args.decoder), "region": str(args.region), "box": args.box, "feather": args.feather}) (args.out / "meta.json").write_text(json.dumps(meta, indent=1) + "\n") print(f"LOT_H3 kind=two_track_render_done out={args.out} ok=1", flush=True) @torch.no_grad() def draw_layout(args) -> None: """What the region head picks, drawn: for held-out prompts from each ``--cache``, the base model samples at half resolution; at ``--switch`` the head scores every token. Each row shows the final half-res render (upscaled for display) with the score heatmap and the selected region, and the LoT layout that region implies on the full-resolution token grid (1x1 inside, 2x2 on a one-token ring, 4x4 elsewhere).""" import numpy as np from PIL import Image, ImageDraw from mixed_res import region_layout from render_h3 import load_prompts from train_h3 import list_items dit = load_base(args.swap) upscale = load_upscaler() saved = load_region(args.region) head = RegionHead(saved["hidden"]).cuda().eval() head.load_state_dict(saved["state"]) block = saved["block"] width, height = args.size from safetensors import safe_open from build_pairs_tt import caption_for prompts = [] for cache in args.cache_list or [None]: if cache is None: prompts += load_prompts(args.prompts, None) continue for te in [te for _l, te in list_items(cache, args.holdout)[1]][:args.prompts]: with safe_open(str(te), "pt") as handle: hidden, keep = handle.get_tensor("hidden_states"), handle.get_tensor("attention_mask") prompts.append((caption_for(str(te)) or te.name, hidden[keep].unsqueeze(0))) colors = {1: (74, 222, 128), 4: (139, 180, 240), 16: (192, 132, 252)} rows, meta = [], [] for index, (caption, embeds) in enumerate(prompts): latent, _x0, states = native_sample(dit, embeds.to("cuda", torch.bfloat16), height=height // 2, width=width // 2, steps=args.steps, seed=args.seed + index, capture_at=args.switch - 1, block=block) tok_h, tok_w = height // 64, width // 64 scores = head(states).float().cpu().reshape(1, tok_h, tok_w) mask = region_mask(scores, alpha=args.alpha, floor=args.floor) full = mask.repeat_interleave(2, 1).repeat_interleave(2, 2) layout = region_layout(full, background=4, ring=1) image = to_image_from(upscale.vae.decode(latent.to("cuda", upscale.dtype))).resize((width, height), Image.LANCZOS) # Left: heatmap of the head's probability with the selected region outlined. prob = torch.sigmoid(scores[0]).numpy() heat = Image.fromarray((np.clip(prob, 0, 1) * 255).astype(np.uint8)).resize((width, height), Image.NEAREST) red = Image.merge("RGB", (heat, Image.new("L", heat.size, 40), Image.new("L", heat.size, 40))) left = outline(Image.blend(image, red, 0.45), mask) # Right: the LoT layout, every token drawn at full-resolution token size (32 px). right = image.copy() draw = ImageDraw.Draw(right, "RGBA") for rect in layout.rects: x0, y0 = rect.v * 32, rect.u * 32 area = rect.eh * rect.ew r, g, b = colors.get(area, (167, 139, 250)) draw.rectangle([x0, y0, x0 + rect.ew * 32 - 1, y0 + rect.eh * 32 - 1], outline=(r, g, b, 230), fill=(r, g, b, 40 if area > 1 else 70)) row = Image.new("RGB", (2 * width, height + 30), "white") row.paste(left, (0, 30)) row.paste(right, (width, 30)) label = f"{caption[:110]} | region {float(mask.float().mean()):.0%} of frame, layout {layout.count} of {tok_h * tok_w * 4} tokens" ImageDraw.Draw(row).text((8, 9), label, fill="black") rows.append(row) meta.append({"caption": caption, "region_share": round(float(mask.float().mean()), 4), "layout_tokens": layout.count, "full_tokens": tok_h * tok_w * 4}) print(f"LOT_H3 kind=layout prompt={index} region={float(mask.float().mean()):.2f} tokens={layout.count}", flush=True) args.out.mkdir(parents=True, exist_ok=True) scale = args.sheet_scale cell = (int(2 * width * scale), int((height + 30) * scale)) for start in range(0, len(rows), args.per_sheet): chunk = rows[start:start + args.per_sheet] sheet = Image.new("RGB", (cell[0], cell[1] * len(chunk)), "white") for number, row in enumerate(chunk): sheet.paste(row.resize(cell, Image.LANCZOS), (0, number * cell[1])) sheet.save(args.out / f"layout_{start // args.per_sheet}.jpg", quality=90) (args.out / "layout.json").write_text(json.dumps({"switch": args.switch, "alpha": args.alpha, "region": str(args.region), "items": meta}, indent=1) + "\n") print(f"LOT_H3 kind=layout_done out={args.out} prompts={len(rows)} ok=1", flush=True) def save_clip(pixels: torch.Tensor, path: Path, fps: int = 24) -> None: """(1, 3, F, H, W) in [-1, 1] -> mp4.""" import imageio frames = ((pixels[0].float().clamp(-1, 1) + 1) * 127.5).round().byte().permute(1, 2, 3, 0).cpu().numpy() imageio.mimwrite(str(path), list(frames), fps=fps, quality=8, macro_block_size=1) def to_image_from(pixels): from render_h3 import to_image return to_image(pixels) def outline(image, mask: torch.Tensor): """Draw the region's boundary (one LR token is 64 px).""" from PIL import ImageDraw image = image.copy() draw = ImageDraw.Draw(image) grid = mask[0] cell = image.height // grid.shape[0] for u in range(grid.shape[0]): for v in range(grid.shape[1]): if not grid[u, v]: continue x, y = v * cell, u * cell if u == 0 or not grid[u - 1, v]: draw.line([(x, y), (x + cell, y)], fill="white", width=2) if u == grid.shape[0] - 1 or not grid[u + 1, v]: draw.line([(x, y + cell), (x + cell, y + cell)], fill="white", width=2) if v == 0 or not grid[u, v - 1]: draw.line([(x, y), (x, y + cell)], fill="white", width=2) if v == grid.shape[1] - 1 or not grid[u, v + 1]: draw.line([(x + cell, y), (x + cell, y + cell)], fill="white", width=2) return image def train_items(args): from train_h3 import CACHE, list_items train, _held = list_items(args.cache or CACHE, args.holdout) random.Random(args.seed).shuffle(train) return train[:args.items] def crop_even(x0: torch.Tensor) -> torch.Tensor: """Crop a cached latent so its LR version has an even latent size (HR latent sides multiple of 4).""" height, width = x0.shape[-2] // 4 * 4, x0.shape[-1] // 4 * 4 return x0[..., :height, :width] @torch.no_grad() def fit_kv(args) -> None: """Ridge fit of ``f_up`` (their Sec. 3.1): the base model's keys/values at LR and at HR on the same images and noise level, each LR token paired with its four HR tokens.""" from train_h3 import load_item dit = load_base(args.swap) layers = len(dit.blocks) maps = UpMaps(layers, HEAD_DIM, [EXTENT], rank=args.rank).cuda() moments: dict = {} if args.pairs: from build_pairs_tt import list_pairs, load_pair sources = [("pair", entry) for entry in list_pairs(args.pairs, 8)[0][:args.items]] else: upscale = load_upscaler() sources = [("cache", item) for item in train_items(args)] for index, (kind, item) in enumerate(sources): if kind == "pair": pair = load_pair(item) hr0, lr0, text = pair["hr"], pair["lr"], pair["text"] else: x0, text = load_item(item, random.Random(index), "cuda") hr0 = crop_even(x0) lr0 = upscale.down(hr0) plan = TwoTrackPlan(torch.zeros(1, lr0.shape[-2] // 2, lr0.shape[-1] // 2, dtype=torch.bool)) cells = torch.argsort(plan.cell_order).reshape(plan.lr_count, 4).cuda() # LR token -> its HR rows for sigma in args.sigmas: gen = torch.Generator("cuda").manual_seed(index) kv_lr, kv_hr = {}, {} x_lr = (1.0 - sigma) * lr0 + sigma * torch.randn(lr0.shape, device="cuda", generator=gen) x_hr = (1.0 - sigma) * hr0 + sigma * torch.randn(hr0.shape, device="cuda", generator=gen) two_track_forward(dit, x_lr, None, None, 1.0 - sigma, text, capture={"kv": kv_lr}) two_track_forward(dit, x_hr, None, None, 1.0 - sigma, text, capture={"kv": kv_hr}) for layer in kv_lr: for which, slot in (("k", 0), ("v", 1)): x = kv_lr[layer][slot][0].float().reshape(-1, HEAD_DIM) # (N*H, d) y = kv_hr[layer][slot][0].float()[cells].transpose(1, 2).reshape(-1, 4, HEAD_DIM) delta = (y - x.unsqueeze(1)).reshape(x.shape[0], -1) entry = moments.setdefault((layer, which), [0.0, 0.0, 0, 0.0, 0.0]) entry[0] = entry[0] + x.T @ x entry[1] = entry[1] + x.T @ delta entry[2] += x.shape[0] entry[3] += float(delta.pow(2).sum()) # copy-init residual entry[4] += float(y.pow(2).sum()) print(f"LOT_H3 kind=tt_fit_kv item={index}", flush=True) report = {"k": [], "v": []} for (layer, which), (xtx, xty, count, copy_err, energy) in moments.items(): maps.fit_moments_(layer, EXTENT, which, xtx, xty, count, args.ridge) name = f"l{layer}_2x2_{which}" weight = maps.down[name].float() @ maps.up[name].float() # |Y - XW|^2 = |Y|^2 - 2 tr(W^T X^T Y) + tr(W^T X^T X W), on the training moments. fit_err = copy_err - 2.0 * float((weight * xty).sum()) + float((weight * (xtx @ weight)).sum()) report[which].append((layer, (copy_err / energy) ** 0.5, (max(fit_err, 0.0) / energy) ** 0.5)) for which in ("k", "v"): rows = sorted(report[which]) copy = sum(r[1] for r in rows) / len(rows) fit = sum(r[2] for r in rows) / len(rows) print(f"LOT_H3 kind=tt_fit_kv_residual which={which} copy_rel={copy:.3f} fit_rel={fit:.3f}", flush=True) args.out.mkdir(parents=True, exist_ok=True) torch.save({"state": maps.state_dict(), "rank": args.rank, "layers": layers, "sigmas": args.sigmas, "residual": report}, args.out / "up_maps_tt.pt") print(f"LOT_H3 kind=tt_fit_kv_done out={args.out / 'up_maps_tt.pt'} ok=1", flush=True) def train_region(args) -> None: """Linear head on the LR track's block states at the switch. Label (their self-supervised one, in latent space): the ``--fraction`` of LR tokens where the upscaled LR clean estimate is furthest from the true HR latent.""" from h3_musubi import sigma_schedule from train_h3 import load_item dit = load_base(args.swap) upscale = load_upscaler() sigma = float(sigma_schedule(args.steps)[args.switch - 1]) pairs = [] with torch.no_grad(): for index, item in enumerate(train_items(args)): x0, text = load_item(item, random.Random(index), "cuda") hr0 = crop_even(x0) lr0 = upscale.down(hr0) gen = torch.Generator("cuda").manual_seed(index) x_lr = (1.0 - sigma) * lr0 + sigma * torch.randn(lr0.shape, device="cuda", generator=gen) capture = {"block": args.block} velocity, _ = two_track_forward(dit, x_lr, None, None, 1.0 - sigma, text, capture=capture) error = (upscale(x_lr + sigma * velocity.float()) - hr0).abs().mean(dim=1) # (1, 4h, 4w) latent px error = F.avg_pool2d(error, 4).flatten() # LR tokens label = (error >= torch.quantile(error, 1.0 - args.fraction)).float() pairs.append((capture["states"].cpu(), label.cpu())) print(f"LOT_H3 kind=tt_region_pair item={index} tokens={label.numel()}", flush=True) del dit torch.cuda.empty_cache() head = RegionHead(pairs[0][0].shape[-1]).cuda() optimizer = torch.optim.Adam(head.parameters(), lr=args.lr) held = pairs[: max(1, len(pairs) // 10)] train = pairs[len(held):] or pairs for epoch in range(args.epochs): random.Random(epoch).shuffle(train) total = 0.0 for states, target in train: loss = F.binary_cross_entropy_with_logits(head(states.cuda()), target.cuda()) optimizer.zero_grad(set_to_none=True) loss.backward() optimizer.step() total += float(loss.detach()) with torch.no_grad(): held_bce = sum(float(F.binary_cross_entropy_with_logits(head(s.cuda()), t.cuda())) for s, t in held) print(f"LOT_H3 kind=tt_region_epoch epoch={epoch} train_bce={total / len(train):.4f} " f"held_bce={held_bce / len(held):.4f}", flush=True) args.out.mkdir(parents=True, exist_ok=True) torch.save({"state": head.state_dict(), "hidden": pairs[0][0].shape[-1], "block": args.block, "switch": args.switch, "steps": args.steps, "fraction": args.fraction}, args.out / "region_head_tt.pt") print(f"LOT_H3 kind=tt_region_done out={args.out / 'region_head_tt.pt'} ok=1", flush=True) def main() -> None: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("phase", choices=("render", "fit-kv", "region", "layout")) parser.add_argument("--cache-list", type=Path, nargs="+", default=None, help="layout: caches whose held-out captions are the prompts (--prompts from each)") parser.add_argument("--per-sheet", type=int, default=6, help="layout: rows per output sheet") parser.add_argument("--sheet-scale", type=float, default=0.5, help="layout: sheet size relative to 2x canvas") parser.add_argument("--out", type=Path, required=True) parser.add_argument("--cache", type=Path, default=None, help="still cache with text (prompts and fit data)") parser.add_argument("--holdout", type=int, default=24) parser.add_argument("--te", type=Path, nargs="+", default=None, help="render: caption embeddings (*_minimaxh3_te.safetensors) to use as prompts instead of a " "cache's held-out items, e.g. from the HF dataset johndpope/MiniMax-H3-pexels-people-text") parser.add_argument("--items", type=int, default=24) parser.add_argument("--prompts", type=int, default=3) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--swap", default="auto", help="DiT blocks streamed from CPU; auto (by free VRAM): 0 at 64 GB+, 8 at 28+, 16 at 21+, else 24") parser.add_argument("--steps", type=int, default=20) parser.add_argument("--switch", type=int, default=18, help="LR-only steps before the HR region starts") parser.add_argument("--hr-steps", type=int, default=6, help="steps after the switch, linear in sigma (0: the rest of the schedule)") parser.add_argument("--size", type=int, nargs=2, default=(1344, 768), metavar=("W", "H")) parser.add_argument("--switch-upscale", choices=("vae", "crop", "latent"), default="vae", help="render: HR start at the switch: VAE round trip of the frame, of M's bounding box only " "(crop), or latent x2 interpolation (no VAE; striped regions)") parser.add_argument("--preset", choices=("quality",), default=None, help="render: quality = switch 16, 6 HR steps, crop upscale, pixel paste, and the fitted maps " "($LOT_TT_MAPS) and region head ($LOT_TT_REGION) unless --maps / --region / --box are given") parser.add_argument("--final", choices=("latent", "pixel"), default="latent", help="render: merge the tracks in latent space (a VAE round trip of LR) or in pixel space " "(decode LR at half size, paste after decoding)") parser.add_argument("--frames", type=int, default=1, help="render: pixel frames (1 = still; 22 = 7 latent frames). Clips also save p_.mp4") parser.add_argument("--variants", default="hr,lr,two_track") parser.add_argument("--maps", type=Path, default=None, help="up_maps_tt.pt from fit-kv (default: copy)") parser.add_argument("--train", type=Path, default=None, help="train_tt.py lora output dir: LoRA + fine-tuned maps + resolution embedding (overrides --maps)") parser.add_argument("--tag", default="", help="checkpoint tag in --train, e.g. _1000") parser.add_argument("--decoder", type=Path, default=None, help="decoder.pt from train_tt.py decoder (U and G)") parser.add_argument("--pairs", type=Path, action="append", default=None, help="fit-kv: build_pairs_tt.py dirs (HR + LR latents precomputed; repeatable)") parser.add_argument("--region", type=Path, default=None, help="region_head_tt.pt (default: --box)") parser.add_argument("--box", type=float, nargs=4, default=None, metavar=("X0", "Y0", "X1", "Y1")) parser.add_argument("--alpha", type=float, default=0.5) parser.add_argument("--floor", type=float, default=None) parser.add_argument("--feather", type=float, default=4.0, help="paste-edge ramp inside the region, HR latent pixels (weight 1/R at the edge, 1 from R-1 deep)") parser.add_argument("--outline", action="store_true", help="draw the region boundary on the two_track image") parser.add_argument("--block", type=int, default=MID_BLOCK) parser.add_argument("--fraction", type=float, default=0.25) parser.add_argument("--epochs", type=int, default=20) parser.add_argument("--lr", type=float, default=1e-3) parser.add_argument("--rank", type=int, default=HEAD_DIM, help="rank of the f_up delta (head dim = full)") parser.add_argument("--ridge", type=float, default=1e-2) parser.add_argument("--sigmas", type=float, nargs="+", default=[0.2, 0.4, 0.6, 0.8, 0.95], help="fit-kv: noise levels; cover the sigmas the HR track runs at after the switch") args = parser.parse_args() if args.preset == "quality": args.switch, args.hr_steps, args.switch_upscale, args.final = 16, 6, "crop", "pixel" if args.maps is None and args.train is None: path = os.environ.get("LOT_TT_MAPS") if not path or not Path(path).is_file(): raise SystemExit("--preset quality needs fitted maps: --maps up_maps_tt.pt (fit-kv) or $LOT_TT_MAPS; " "without them the HR region has a diamond-grid grain") args.maps = Path(path) if args.region is None and args.box is None and os.environ.get("LOT_TT_REGION"): args.region = Path(os.environ["LOT_TT_REGION"]) from gpu_guard import auto_swap from h3_musubi import latent_frames as _latent_frames args.swap = auto_swap(args.swap, inference=args.phase in ("render", "layout"), latent_t=_latent_frames(getattr(args, "frames", 1))) if args.size[0] % 64 or args.size[1] % 64: raise SystemExit("--size sides must be multiples of 64 (the LR latent needs whole 2x2 patches)") refuse_if_busy("two_track_h3") {"render": render, "fit-kv": fit_kv, "region": train_region, "layout": draw_layout}[args.phase](args) if __name__ == "__main__": main()