Spaces:
Running on Zero
Running on Zero
Download lot/two_track_h3.py from hugging-apps/mpeg-diffusion: direct link, hf CLI and curl.
- Browser
- Download file 52.3 kB
-
https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/lot/two_track_h3.py
- Command line
-
hf download hf://spaces/hugging-apps/mpeg-diffusion/lot/two_track_h3.py
-
curl -L -o two_track_h3.py https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/lot/two_track_h3.py
52.3 kB
| #!/usr/bin/env python3 | |
| """MPEG-Diffusion's own arrangement on H3: a native LR track plus HR tokens on a selected region. | |
| python3 lot/two_track_h3.py render --out DIR [--steps 20 --switch 18 --hr-steps 6] [--maps DIR/up_maps_tt.pt] | |
| [--region DIR/region_head_tt.pt | --box 0.25 0.2 0.75 0.9] | |
| python3 lot/two_track_h3.py fit-kv --out DIR [--items 24] # ridge fit of f_up, base model at LR and HR | |
| python3 lot/two_track_h3.py region --out DIR [--items 200] # linear region head on the LR track's states | |
| Paper: Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu, Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu, MPEG-Diffusion: Mixed-Resolution Video Generation, arXiv:2611.12069. | |
| ``mixed_res_h3.py`` puts the paper's pieces on LoT's merged tokens, so its coarse region is only as | |
| good as the LoT adapter. Here the LR track is what the paper uses: the unmodified base model at | |
| half resolution (no adapter, no LoRA), which is good from the start. H3's rotary positions are | |
| normalised by ``sqrt(area)``, so the LR and HR grids already share one coordinate frame. | |
| Sampler (their Sec. 4, "Inference"): | |
| 1. ``--switch`` steps denoise the whole frame at LR (``W/2 x H/2``). | |
| 2. At the switch a region ``M`` is chosen on the LR token grid (``--region`` head, or ``--box``). | |
| 3. The clean LR estimate is upscaled, re-noised with fresh noise, and the remaining steps run one | |
| packed sequence ``[text, audio, LR video (whole grid), HR video (M only)]`` with dual-resolution | |
| attention (``mixed_res.two_track_attention``): LR queries see only LR keys, so the LR track stays | |
| the native model; HR queries see their own keys inside ``M`` and ``f_up`` of the LR keys outside. | |
| 4. The LR latent is upscaled and the HR latent is pasted over ``M``; the VAE decodes that. | |
| The upscaler ``U`` is the VAE itself (decode, bicubic x2, encode): it needs no training and returns | |
| a latent the decoder already agrees with. ``--feather`` blends the paste edge in latent space. | |
| Not trained here (the paper fine-tunes them): the DiT with the maps, the resolution embedding, | |
| ``f_down`` (unused: the LR track is complete), the learned upscaler and the mixer. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import contextlib | |
| import json | |
| import os | |
| import random | |
| import sys | |
| import time | |
| from dataclasses import replace | |
| from pathlib import Path | |
| import torch | |
| import torch.nn.functional as F | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from gpu_guard import refuse_if_busy # noqa: E402 | |
| from mixed_res import (RegionHead, TwoTrackPlan, UpMaps, patched_attention, region_mask, # noqa: E402 | |
| two_track_attention) | |
| from procrustes_h3 import CHECKPOINT # noqa: E402 | |
| HEAD_DIM = 128 | |
| MID_BLOCK = 25 | |
| EXTENT = (1, 2, 2) | |
| # ------------------------------------------------------------------------- forward | |
| def two_track_forward(dit, lr: torch.Tensor, hr: torch.Tensor | None, plan: TwoTrackPlan | None, t: float, | |
| text: torch.Tensor, *, maps: UpMaps | None = None, audio_generator=None, | |
| capture: dict | None = None, res_embed: "ResolutionEmbedding | None" = None): | |
| """Velocities ``(v_lr, v_hr)`` of the LR latent and of the HR latent on the plan's region. | |
| ``lr`` is ``(1, 24, T, h, w)``, ``hr`` is ``(1, 24, T, 2h, 2w)``; ``v_hr`` is zero outside the | |
| region. With ``plan=None`` (or an empty region) this is the base model on ``lr`` alone. | |
| ``capture={"block": i}`` returns that block's LR video states in ``capture["states"]``; | |
| ``capture={"kv": {}}`` collects every block's pre-RoPE LR keys and values. | |
| ``res_embed`` adds the paper's learned LR / HR vectors to the video rows. | |
| With grad enabled (training) the attention patch is installed for good (``_training_patch``) | |
| so activation checkpointing recomputes the same attention in the backward pass. | |
| """ | |
| from h3_musubi import _layout, _silence, _sync_swap, audio_sigma | |
| from musubi_tuner.minimax_h3.packing import (H3RowSegment, _unpack_video_rows, build_position_grid, | |
| build_timestep_rows, pack_audio_rows, pack_video_rows) | |
| if text.ndim == 2: | |
| text = text.unsqueeze(0) | |
| _sync_swap(dit) | |
| device = dit.video_patch_proj.weight.device | |
| sigma_a = float(audio_sigma(1.0 - t)) | |
| _b, _c, frames, lat_h, lat_w = lr.shape | |
| layout = _layout(text.shape[1], frames, lat_h, lat_w) | |
| segment = layout.target_video_segment | |
| if segment.stop != layout.row_count: | |
| raise RuntimeError("the target video must be the last segment of the packed sequence") | |
| lr_count = segment.row_count | |
| hr_count = plan.hr_count if plan is not None else 0 | |
| if plan is not None and plan.lr_count != lr_count: | |
| raise ValueError(f"plan has {plan.lr_count} LR tokens, the latent has {lr_count}") | |
| rows = pack_video_rows(lr.to(device, torch.float32)) | |
| positions = build_position_grid(layout)[0] | |
| hr_rotation = None | |
| if hr_count: | |
| hr_layout = _layout(text.shape[1], frames, 2 * lat_h, 2 * lat_w) | |
| hr_positions = build_position_grid(hr_layout)[0][-4 * lr_count:] | |
| rows = torch.cat([rows, pack_video_rows(hr.to(device, torch.float32))[:, plan.hr_index]], dim=1) | |
| positions = torch.cat([positions, hr_positions[plan.hr_index.cpu()]]) | |
| video = dit.video_patch_proj(rows).to(dit.dtype) | |
| if res_embed is not None: | |
| video = res_embed(video, lr_count) | |
| audio = _silence(frames, sigma_a, device, audio_generator) | |
| audio_rows = dit.audio_patch_proj(pack_audio_rows(audio.to(device)).to(torch.float32)).to(dit.dtype) | |
| text_rows = dit.token_refiner(dit.condition_proj(text.to(device, dit.dtype))) | |
| hidden = torch.cat([text_rows, audio_rows, video], dim=1) | |
| segments = layout.segments[:-1] + (H3RowSegment(segment.role, segment.kind, segment.start, | |
| segment.start + lr_count + hr_count),) | |
| packed = replace(layout, segments=segments, row_count=segment.start + lr_count + hr_count) | |
| rotation = dit._rotation_table(positions.unsqueeze(0).to(device), hidden.dtype) | |
| tags = torch.ones(1, text.shape[1], dtype=torch.int64) | |
| timesteps = build_timestep_rows(packed, text_token_tags=tags, model_t_video=t, model_t_audio=1.0 - sigma_a) | |
| temb = dit._timestep_embeddings(timesteps.unique_timesteps, device) | |
| patch = contextlib.nullcontext() | |
| want_kv = capture is not None and "kv" in capture | |
| training = torch.is_grad_enabled() | |
| persistent = training or getattr(dit, "_two_track_context", None) is not None | |
| if hr_count or want_kv or persistent: | |
| if hr_count: | |
| hr_rotation = dit._rotation_table(hr_positions.unsqueeze(0).to(device), hidden.dtype) | |
| context = dict(plan=plan if plan is not None and hr_count else _empty_plan(frames, lat_h, lat_w, device), | |
| maps=maps, video_start=segment.start, hr_rotation=hr_rotation, | |
| capture=capture.get("kv") if want_kv else None) | |
| if persistent: | |
| _training_patch(dit).update(context) | |
| else: | |
| patch = patched_attention(dit.blocks, attention=two_track_attention, **context) | |
| checkpoint = training and dit.gradient_checkpointing | |
| with patch: | |
| for index, block in enumerate(dit.blocks): | |
| if dit.blocks_to_swap: | |
| dit.offloader.wait_for_block(index) | |
| if checkpoint: | |
| hidden = torch.utils.checkpoint.checkpoint(block, hidden, temb, timesteps.block_segments, rotation, | |
| use_reentrant=False) | |
| else: | |
| hidden = block(hidden, temb, timesteps.block_segments, rotation) | |
| if dit.blocks_to_swap: | |
| dit.offloader.submit_move_blocks_forward(dit.blocks, index) | |
| if capture is not None and capture.get("block") == index: | |
| states = hidden[0, segment.start:segment.start + lr_count].float() | |
| capture["states"] = states if capture.get("grad") else states.detach() | |
| hidden = hidden.to(device) | |
| final = dit.final_layer | |
| shift, scale = final.adaln_proj(temb) | |
| index = timesteps.video_timestep_index | |
| states = final.norm(hidden[:, segment.start:]) * (1.0 + scale[index]) + shift[index] | |
| out = final.video_out(states.to(torch.float32)) | |
| v_lr = _unpack_video_rows(out[:, :lr_count], layout.target_video).to(lr.dtype) | |
| if not hr_count: | |
| return v_lr, None | |
| full = out.new_zeros(1, 4 * lr_count, out.shape[-1]) | |
| full[:, plan.hr_index] = out[:, lr_count:] | |
| return v_lr, _unpack_video_rows(full, hr_layout.target_video).to(hr.dtype) | |
| def _empty_plan(frames: int, lat_h: int, lat_w: int, device) -> TwoTrackPlan: | |
| return TwoTrackPlan(torch.zeros(frames, lat_h // 2, lat_w // 2, dtype=torch.bool)).to(device) | |
| def _training_patch(dit) -> dict: | |
| """Route every block's attention through ``two_track_attention`` for the life of the process and | |
| return the shared context dict. A context manager would be gone before the backward pass, and | |
| checkpointed blocks recompute attention there.""" | |
| context = getattr(dit, "_two_track_context", None) | |
| if context is None: | |
| import types | |
| context = {} | |
| for layer, block in enumerate(dit.blocks): | |
| def forward(self, hidden, rotation=None, _layer=layer): | |
| return two_track_attention(self, hidden, rotation, layer=_layer, **context) | |
| block.attn.forward = types.MethodType(forward, block.attn) | |
| dit._two_track_context = context | |
| context.clear() | |
| return context | |
| class ResolutionEmbedding(torch.nn.Module): | |
| """The paper's resolution embedding: one learned vector for LR rows and one for HR rows, zero at init.""" | |
| def __init__(self, hidden_size: int): | |
| super().__init__() | |
| self.lr = torch.nn.Parameter(torch.zeros(hidden_size)) | |
| self.hr = torch.nn.Parameter(torch.zeros(hidden_size)) | |
| def forward(self, video: torch.Tensor, lr_count: int) -> torch.Tensor: | |
| add = torch.cat([self.lr.expand(lr_count, -1), self.hr.expand(video.shape[1] - lr_count, -1)]) | |
| return video + add.to(video.dtype).unsqueeze(0) | |
| # ------------------------------------------------------------------------ upscaler | |
| class VaeUpscaler: | |
| """``U``: decode, bicubic x2 per frame, encode. A still or a clip (any latent frame count).""" | |
| def __init__(self, vae): | |
| self.vae = vae | |
| self.dtype = next(vae.parameters()).dtype | |
| def _resample(self, latent: torch.Tensor, **interp) -> torch.Tensor: | |
| # Long clips (16 s): the decoded clip leaves the GPU at once as fp16, the resize runs 16 frames at a time, | |
| # and only the fp16 result goes back for the encode, so the GPU never holds a float32 copy of the clip. | |
| pixels = self.vae.decode(latent.to("cuda", self.dtype)).clamp(-1, 1).to("cpu", torch.float16) | |
| if latent.shape[2] == 1: | |
| pixels = pixels[:, :, :1] | |
| b, c, f, h, w = pixels.shape | |
| resized = None | |
| for start in range(0, f, 16): | |
| chunk = pixels[0, :, start:start + 16].transpose(0, 1).to("cuda", torch.float32) # (n, 3, h, w) | |
| chunk = F.interpolate(chunk, **interp).clamp(-1, 1).to("cpu", torch.float16) | |
| if resized is None: | |
| resized = torch.empty(1, c, f, *chunk.shape[-2:], dtype=torch.float16) | |
| resized[0, :, start:start + chunk.shape[0]] = chunk.transpose(0, 1) | |
| del pixels | |
| torch.cuda.empty_cache() | |
| pixels = resized.to("cuda", self.dtype) | |
| out = self.vae.encode(pixels[:, :, 0] if f == 1 else pixels).float() | |
| if out.shape[2] != latent.shape[2]: | |
| raise RuntimeError(f"VAE round trip changed latent frames {latent.shape[2]} -> {out.shape[2]}") | |
| return out | |
| def __call__(self, latent: torch.Tensor) -> torch.Tensor: | |
| return self._resample(latent, scale_factor=2, mode="bicubic", align_corners=False) | |
| def down(self, latent: torch.Tensor) -> torch.Tensor: | |
| """``D``: decode, area x0.5, encode (the LR latent of a cached HR item).""" | |
| return self._resample(latent, scale_factor=0.5, mode="area") | |
| def mask_to_latent(mask: torch.Tensor, scale: int, feather: float = 0.0) -> torch.Tensor: | |
| """LR-token mask ``(T, h, w)`` -> ``(1, 1, T, h*scale, w*scale)`` float, optionally feathered inward. | |
| The feather is a ramp inside the region only (outside it the HR track was never denoised): weight | |
| ``1/R`` on the edge cell, rising by ``1/R`` per cell of depth, 1 from depth ``R - 1`` on. Every region | |
| cell keeps weight > 0, and a region at least ``2R`` cells across reaches full weight in its middle. | |
| (The old erode-then-blur left 40% of a 30% box below full weight and zeroed regions 2 tokens wide.)""" | |
| up = mask.float().repeat_interleave(scale, 1).repeat_interleave(scale, 2) | |
| radius = int(round(feather)) | |
| if radius > 1: | |
| level = up[:, None] | |
| total = level.clone() | |
| for _ in range(radius - 1): | |
| level = -F.max_pool2d(-level, 3, stride=1, padding=1) # erode one cell (edge of frame counts as inside) | |
| total = total + level | |
| up = (total / radius)[:, 0] | |
| return up[None, None] | |
| # ------------------------------------------------------------------------- sampler | |
| def box_mask(box, frames: int, height: int, width: int) -> torch.Tensor: | |
| x0, y0, x1, y1 = box | |
| mask = torch.zeros(frames, height, width, dtype=torch.bool) | |
| mask[:, round(y0 * height):round(y1 * height), round(x0 * width):round(x1 * width)] = True | |
| return mask | |
| def native_sample(dit, text, *, height, width, steps, seed, stop=None, capture_at=None, block=MID_BLOCK, | |
| res_embed=None, latent_t=1): | |
| """Base-model Euler sampling; returns ``(x, x0_last, states)`` after ``stop`` steps (default all).""" | |
| from h3_musubi import sigma_schedule | |
| gen = torch.Generator("cuda").manual_seed(seed) | |
| x = torch.randn(1, 24, latent_t, height // 16, width // 16, device="cuda", generator=gen) | |
| audio_gen = torch.Generator("cuda").manual_seed(seed + 1) | |
| sigmas = sigma_schedule(steps) | |
| x0 = states = None | |
| for index in range(steps if stop is None else stop): | |
| sigma, sigma_next = float(sigmas[index]), float(sigmas[index + 1]) | |
| capture = {"block": block} if capture_at == index else None | |
| velocity, _ = two_track_forward(dit, x, None, None, 1.0 - sigma, text, audio_generator=audio_gen, | |
| capture=capture, res_embed=res_embed) | |
| if capture is not None: | |
| states = capture["states"] | |
| x0 = x + sigma * velocity.float() | |
| x = x0 + (sigma_next / sigma) * (x - x0) | |
| return x, x0, states | |
| def two_track_sample(dit, text, upscale, *, height, width, steps, switch, seed, maps=None, head=None, box=None, | |
| block=MID_BLOCK, alpha=0.5, floor=None, feather=0.0, on_mask=None, hr_steps=None, | |
| res_embed=None, decoder=None, latent_t=1, switch_upscale="vae", final="latent"): | |
| """The four-stage sampler of the module docstring. Returns ``(latent, info)`` at ``width x height``. | |
| ``decoder`` is ``(U, G)`` from ``train_tt.py decoder``: ``U`` replaces the VAE upscale (at the | |
| switch and at the end) and ``G`` merges the tracks (their eqs. 5-6) instead of the feathered paste.""" | |
| from h3_musubi import sigma_schedule | |
| lat_h, lat_w = height // 32, width // 32 # LR latent; also the HR token grid | |
| tok_h, tok_w = lat_h // 2, lat_w // 2 # LR token grid | |
| sigmas = sigma_schedule(steps) | |
| lr, x0_lr, states = native_sample(dit, text, height=height // 2, width=width // 2, steps=steps, seed=seed, | |
| stop=switch, capture_at=switch - 1 if head is not None else None, block=block, | |
| res_embed=res_embed, latent_t=latent_t) | |
| up = upscale if decoder is None else decoder[0] | |
| if head is not None: | |
| scores = head(states).float().cpu().reshape(latent_t, tok_h, tok_w) | |
| mask = region_mask(scores, alpha=alpha, floor=floor) | |
| else: | |
| mask = box_mask(box or (0.25, 0.25, 0.75, 0.75), latent_t, tok_h, tok_w) | |
| if on_mask is not None: | |
| on_mask(mask) | |
| tokens = [tok_h * tok_w * latent_t] * switch | |
| if not mask.any() or switch >= steps: | |
| # No region: finish the LR schedule with the base model, then upscale the clean latent. | |
| lr_gen = torch.Generator("cuda").manual_seed(seed + 3) | |
| for index in range(switch, steps): | |
| sigma, sigma_next = float(sigmas[index]), float(sigmas[index + 1]) | |
| velocity, _ = two_track_forward(dit, lr, None, None, 1.0 - sigma, text, audio_generator=lr_gen, | |
| res_embed=res_embed) | |
| x0 = lr + sigma * velocity.float() | |
| lr = x0 + (sigma_next / sigma) * (lr - x0) | |
| tokens.append(tok_h * tok_w * latent_t) | |
| return up(lr), {"mask": mask, "tokens": tokens} | |
| plan = TwoTrackPlan(mask).to("cuda") | |
| gen = torch.Generator("cuda").manual_seed(seed + 2) | |
| audio_gen = torch.Generator("cuda").manual_seed(seed + 3) | |
| sigma = float(sigmas[switch]) | |
| # At the switch the HR latent is mostly noise (sigma 0.75 at 16 of 20 on shift 12), so a latent x2 interpolation | |
| # can stand in for the VAE round trip (5 s for 7 latent frames on a 5090): switch_upscale="latent". | |
| # switch_upscale="crop": the HR track only reads U(x0_lr) inside M, so round-trip just M's bounding box | |
| # (all frames, plus a margin) and fill the rest with a latent x2. | |
| if switch_upscale == "vae": | |
| start = up(x0_lr) | |
| else: | |
| start = F.interpolate(x0_lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False) | |
| if switch_upscale == "crop": | |
| box_lr = crop_box(mask, lat_h, lat_w, margin=2) | |
| if box_lr is not None: | |
| y0, y1, x0, x1 = box_lr | |
| start[..., 2 * y0:2 * y1, 2 * x0:2 * x1] = up(x0_lr[..., y0:y1, x0:x1].contiguous()) | |
| hr = (1.0 - sigma) * start + sigma * torch.randn(1, 24, latent_t, 2 * lat_h, 2 * lat_w, device="cuda", | |
| generator=gen) | |
| # After the switch both tracks share the schedule: the model's own, or ``hr_steps`` steps | |
| # linear in sigma (H3's shift 12 leaves few low-sigma steps, where the HR detail is made). | |
| tail = sigmas[switch:] if not hr_steps else torch.linspace(sigma, 0.0, hr_steps + 1, dtype=torch.float64) | |
| for index in range(len(tail) - 1): | |
| sigma, sigma_next = float(tail[index]), float(tail[index + 1]) | |
| v_lr, v_hr = two_track_forward(dit, lr, hr, plan, 1.0 - sigma, text, maps=maps, audio_generator=audio_gen, | |
| res_embed=res_embed) | |
| ratio = sigma_next / sigma | |
| lr = lr + (1.0 - ratio) * sigma * v_lr.float() | |
| hr = hr + (1.0 - ratio) * sigma * v_hr.float() | |
| tokens.append(plan.lr_count + plan.hr_count) | |
| if decoder is not None: | |
| upscaler, mixer = decoder | |
| inside = mask_to_latent(mask, 2).to("cuda") # LR latent grid | |
| # Outside M the HR latent is still switch-sigma noise (v_hr is zero there). D only feeds M, but the | |
| # VAE's receptive field reaches past the edge, so fill the outside with a latent x2 of the LR track. | |
| inside_hr = mask_to_latent(mask, 4).to("cuda") | |
| context = F.interpolate(lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False) | |
| clean_hr = inside_hr * hr + (1.0 - inside_hr) * context | |
| filled = (1.0 - inside) * lr + inside * upscale.down(clean_hr) # their eq. 5: fill M with D(z_h) | |
| latent = mixer(hr, upscaler(filled), mask_to_latent(mask, 4).to("cuda")) | |
| return latent, {"mask": mask, "tokens": tokens} | |
| weight = mask_to_latent(mask, 4, feather).to("cuda") | |
| if final == "pixel": | |
| # No round trip at the end: the caller decodes HR (outside M filled with a latent x2 of LR, so the | |
| # VAE sees a plausible neighbourhood at the edge) and LR at half size, and pastes in pixel space. | |
| inside = mask_to_latent(mask, 4).to("cuda") | |
| context = F.interpolate(lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False) | |
| return inside * hr + (1.0 - inside) * context, {"mask": mask, "tokens": tokens, "lr": lr, "weight": weight} | |
| latent = weight * hr + (1.0 - weight) * upscale(lr) | |
| return latent, {"mask": mask, "tokens": tokens} | |
| def crop_box(mask: torch.Tensor, lat_h: int, lat_w: int, margin: int = 2): | |
| """Union over frames of M's bounding box on the LR latent grid (one LR token = 2 latent px), grown by | |
| ``margin`` latent px and rounded out to even sides, as ``(y0, y1, x0, x1)``; None for an empty mask.""" | |
| rows = torch.nonzero(mask.any(0).any(1)).flatten() | |
| cols = torch.nonzero(mask.any(0).any(0)).flatten() | |
| if not len(rows): | |
| return None | |
| y0, y1 = max(0, 2 * int(rows[0]) - margin), min(lat_h, 2 * int(rows[-1]) + 2 + margin) | |
| x0, x1 = max(0, 2 * int(cols[0]) - margin), min(lat_w, 2 * int(cols[-1]) + 2 + margin) | |
| y0, x0 = y0 - y0 % 2, x0 - x0 % 2 | |
| y1, x1 = min(lat_h, y1 + y1 % 2), min(lat_w, x1 + x1 % 2) | |
| return y0, y1, x0, x1 | |
| def pixel_compose(vae, hr_latent: torch.Tensor, lr_latent: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: | |
| """Decode HR and LR, upscale the LR pixels x2 (bicubic, per frame), paste with ``weight`` (HR latent grid, | |
| ``(1, 1, T, H, W)``) mapped to pixels: x16 in space, each latent frame to the pixel frames it decodes to.""" | |
| from h3_musubi import pixel_frames | |
| # Long clips: each decode leaves the GPU as fp16 at once, and the paste runs ``chunk`` frames at a time, | |
| # so the GPU never holds more than one full clip (16 s at 1344x768 is 2.4 GB in fp16, 4.8 GB in fp32). | |
| dtype = next(vae.parameters()).dtype | |
| hr_px = vae.decode(hr_latent.to("cuda", dtype)).clamp(-1, 1).to("cpu", torch.float16) | |
| lr_px = vae.decode(lr_latent.to("cuda", dtype)).clamp(-1, 1).to("cpu", torch.float16) | |
| if hr_latent.shape[2] == 1: | |
| hr_px, lr_px = hr_px[:, :, :1], lr_px[:, :, :1] | |
| frames = hr_px.shape[2] | |
| frame_of = [k for k in range(hr_latent.shape[2]) for _ in range(pixel_frames(k + 1) - pixel_frames(k))] | |
| frame_of = (frame_of + [frame_of[-1]] * frames)[:frames] | |
| weight = weight[0, 0].float().to("cuda") # (T, H, W) on the HR latent grid | |
| out = torch.empty_like(hr_px) | |
| chunk = 16 | |
| for start in range(0, frames, chunk): | |
| stop = min(frames, start + chunk) | |
| lr = lr_px[0, :, start:stop].to("cuda", torch.float32).transpose(0, 1) # (n, 3, h, w) | |
| lr_up = F.interpolate(lr, scale_factor=2, mode="bicubic", align_corners=False).clamp(-1, 1) | |
| w_px = F.interpolate(weight[frame_of[start:stop]][:, None], scale_factor=16, mode="bilinear", | |
| align_corners=False) # (n, 1, H, W) | |
| hr = hr_px[0, :, start:stop].to("cuda", torch.float32).transpose(0, 1) | |
| out[0, :, start:stop] = (w_px * hr + (1.0 - w_px) * lr_up).transpose(0, 1).to("cpu", torch.float16) | |
| return out | |
| # -------------------------------------------------------------------------- phases | |
| def load_base(swap: int): | |
| from h3_musubi import load_dit | |
| return load_dit(CHECKPOINT, device="cuda", swap=swap) | |
| def load_maps(path: Path) -> UpMaps: | |
| """``up_maps_tt.pt`` from ``fit-kv``, or the published ``up_maps_tt.safetensors`` (same keys, layers / | |
| rank in the metadata).""" | |
| path = Path(path) | |
| if path.suffix == ".safetensors": | |
| from safetensors import safe_open | |
| from safetensors.torch import load_file | |
| with safe_open(str(path), "pt") as handle: | |
| meta = handle.metadata() or {} | |
| saved = {"state": load_file(str(path)), "layers": int(meta["layers"]), "rank": int(meta["rank"])} | |
| else: | |
| saved = torch.load(path, map_location="cpu", weights_only=False) | |
| maps = UpMaps(saved["layers"], HEAD_DIM, [EXTENT], rank=saved["rank"]) | |
| maps.load_state_dict(saved["state"]) | |
| return maps.to("cuda").eval() | |
| def load_trained(dit, train_dir: Path, tag: str = ""): | |
| """LoRA (on), up-maps and resolution embedding from a ``train_tt.py lora`` run. Returns ``(network, maps, res_embed)``.""" | |
| from safetensors import safe_open | |
| from h3_musubi import make_lora | |
| with safe_open(str(train_dir / f"lora{tag}.safetensors"), "pt") as handle: | |
| rank = int((handle.metadata() or {}).get("rank", 16)) | |
| network = make_lora(dit, rank) | |
| info = network.load_weights(str(train_dir / f"lora{tag}.safetensors")) | |
| if info.missing_keys: | |
| raise RuntimeError(f"LoRA load missed {len(info.missing_keys)} keys") | |
| network.to(device="cuda", dtype=torch.bfloat16).eval() | |
| published = train_dir / f"tt_state{tag}.safetensors" # as upload_tt_hf.py publishes it | |
| if published.is_file(): | |
| from safetensors.torch import load_file | |
| with safe_open(str(published), "pt") as handle: | |
| meta = handle.metadata() or {} | |
| flat = load_file(str(published)) | |
| saved = {"layers": int(meta["layers"]), "rank": int(meta["rank"]), "hidden": int(meta["hidden"]), | |
| "maps": {k[5:]: v for k, v in flat.items() if k.startswith("maps.")}, | |
| "res_embed": {k[10:]: v for k, v in flat.items() if k.startswith("res_embed.")}} | |
| else: | |
| saved = torch.load(train_dir / f"tt_state{tag}.pt", map_location="cpu", weights_only=False) | |
| maps = UpMaps(saved["layers"], HEAD_DIM, [EXTENT], rank=saved["rank"]) | |
| maps.load_state_dict(saved["maps"]) | |
| res_embed = ResolutionEmbedding(saved["hidden"]) | |
| res_embed.load_state_dict(saved["res_embed"]) | |
| return network, maps.to("cuda").eval(), res_embed.to("cuda").eval() | |
| def load_region(path: Path) -> dict: | |
| """``region_head_tt.pt`` (as ``train_tt.py region`` saves it) or ``.safetensors`` (as published).""" | |
| path = Path(path) | |
| if path.suffix == ".safetensors": | |
| from safetensors import safe_open | |
| from safetensors.torch import load_file | |
| with safe_open(str(path), "pt") as handle: | |
| meta = handle.metadata() or {} | |
| return {"state": load_file(str(path)), "hidden": int(meta["hidden"]), "block": int(meta["block"])} | |
| return torch.load(path, map_location="cpu", weights_only=False) | |
| def load_decoder(path: Path): | |
| from mixed_res import LatentUpscaler, TwoTrackMixer | |
| saved = torch.load(path, map_location="cpu", weights_only=False) | |
| upscaler, mixer = LatentUpscaler(width=saved["up_width"]), TwoTrackMixer(width=saved["mix_width"]) | |
| upscaler.load_state_dict(saved["upscaler"]) | |
| mixer.load_state_dict(saved["mixer"]) | |
| return upscaler.cuda().eval(), mixer.cuda().eval() | |
| def load_upscaler() -> VaeUpscaler: | |
| from h3_musubi import load_vae | |
| from lot_paths import LOT_H3_VAE | |
| return VaeUpscaler(load_vae(LOT_H3_VAE)) | |
| def held_prompts(args): | |
| from render_h3 import load_prompts | |
| from train_h3 import CACHE, list_items | |
| if getattr(args, "te", None): | |
| return load_prompts(args.prompts, list(args.te)) | |
| held = [te for _latent, te in list_items(args.cache or CACHE, args.holdout)[1]] | |
| return load_prompts(args.prompts, held) | |
| def render(args) -> None: | |
| from PIL import Image, ImageDraw | |
| from h3_musubi import sample | |
| from render_h3 import to_image | |
| from h3_musubi import set_lora | |
| dit = load_base(args.swap) | |
| upscale = load_upscaler() | |
| maps = load_maps(args.maps) if args.maps else None | |
| network = res_embed = None | |
| if args.train: | |
| network, maps, res_embed = load_trained(dit, args.train, args.tag) | |
| decoder = load_decoder(args.decoder) if args.decoder else None | |
| head = None | |
| if args.region: | |
| saved = load_region(args.region) | |
| head = RegionHead(saved["hidden"]).cuda().eval() | |
| head.load_state_dict(saved["state"]) | |
| args.block = saved["block"] | |
| if args.train and saved.get("train") in (None, "None", ""): | |
| print("LOT_H3 kind=warning msg=region head was trained on the base model's states, but --train turns the " | |
| "LoRA on for the LR steps too; retrain it with train_tt.py region --train", flush=True) | |
| from h3_musubi import latent_frames | |
| width, height = args.size | |
| latent_t = latent_frames(args.frames) | |
| columns = [c for c in ("hr", "lr", "two_track") if c in args.variants] | |
| meta = {"timings": {c: [] for c in columns}, "tokens": {c: [] for c in columns}, "hr_share": []} | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| rows = [] | |
| for index, (caption, embeds) in enumerate(held_prompts(args)): | |
| embeds = embeds.to("cuda", torch.bfloat16) | |
| row = [] | |
| for name in columns: | |
| mask = None | |
| torch.cuda.synchronize() | |
| start = time.perf_counter() | |
| if network is not None: | |
| set_lora(network, name == "two_track") # the baselines are the base model | |
| if name == "hr": | |
| latent = sample(dit, embeds, height=height, width=width, steps=args.steps, seed=args.seed + index, | |
| latent_t=latent_t) | |
| count = float((height // 32) * (width // 32) * latent_t) | |
| elif name == "lr": | |
| latent, _x0, _s = native_sample(dit, embeds, height=height // 2, width=width // 2, steps=args.steps, | |
| seed=args.seed + index, latent_t=latent_t) | |
| latent = upscale(latent) | |
| count = float((height // 64) * (width // 64) * latent_t) | |
| else: | |
| latent, info = two_track_sample( | |
| dit, embeds, upscale, height=height, width=width, steps=args.steps, switch=args.switch, | |
| seed=args.seed + index, maps=maps, head=head, box=args.box, block=args.block, | |
| alpha=args.alpha, floor=args.floor, feather=args.feather, hr_steps=args.hr_steps, | |
| res_embed=res_embed, decoder=decoder, latent_t=latent_t, switch_upscale=args.switch_upscale, | |
| final=args.final) | |
| mask = info["mask"] | |
| count = sum(info["tokens"]) / len(info["tokens"]) | |
| meta["hr_share"].append(round(float(mask.float().mean()), 4)) | |
| meta["tokens"][name].append(round(count, 1)) | |
| # Every variant is timed through its decode to pixels (sampling + VAE), the number a user waits for. | |
| if name == "two_track" and args.final == "pixel" and "lr" in info: | |
| pixels = pixel_compose(upscale.vae, latent, info["lr"], info["weight"]) | |
| else: | |
| pixels = upscale.vae.decode(latent.to("cuda", upscale.dtype)).to("cpu", torch.float16) | |
| torch.cuda.synchronize() | |
| meta["timings"][name].append(round(time.perf_counter() - start, 2)) | |
| meta["note"] = "timings include the VAE decode for every variant" | |
| if latent_t > 1: | |
| save_clip(pixels, args.out / f"p{index}_{name}.mp4") | |
| pixels = pixels[:, :, pixels.shape[2] // 2:pixels.shape[2] // 2 + 1] # the grid shows the middle frame | |
| image = to_image(pixels) | |
| if mask is not None and args.outline: | |
| image = outline(image, mask[mask.shape[0] // 2:mask.shape[0] // 2 + 1]) | |
| image.save(args.out / f"p{index}_{name}.png") | |
| row.append(image) | |
| print(f"LOT_H3 kind=two_track_render prompt={index} variant={name} " | |
| f"seconds={meta['timings'][name][-1]} mean_tokens={count:.0f}", flush=True) | |
| rows.append(row) | |
| scale = 0.5 | |
| cell = (int(width * scale), int(height * scale)) | |
| sheet = Image.new("RGB", (cell[0] * len(columns), 28 + cell[1] * len(rows)), "white") | |
| draw = ImageDraw.Draw(sheet) | |
| for col, label in enumerate(columns): | |
| draw.text((col * cell[0] + 6, 8), label, fill="black") | |
| for r, images in enumerate(rows): | |
| for col, image in enumerate(images): | |
| sheet.paste(image.resize(cell, Image.LANCZOS), (col * cell[0], 28 + r * cell[1])) | |
| sheet.save(args.out / "grid.png") | |
| meta.update({"columns": columns, "frames": args.frames, "latent_t": latent_t, "steps": args.steps, "switch": args.switch, "canvas": f"{width}x{height}", | |
| "maps": str(args.maps), "train": str(args.train), "decoder": str(args.decoder), "region": str(args.region), "box": args.box, "feather": args.feather}) | |
| (args.out / "meta.json").write_text(json.dumps(meta, indent=1) + "\n") | |
| print(f"LOT_H3 kind=two_track_render_done out={args.out} ok=1", flush=True) | |
| def draw_layout(args) -> None: | |
| """What the region head picks, drawn: for held-out prompts from each ``--cache``, the base model samples | |
| at half resolution; at ``--switch`` the head scores every token. Each row shows the final half-res | |
| render (upscaled for display) with the score heatmap and the selected region, and the LoT layout that | |
| region implies on the full-resolution token grid (1x1 inside, 2x2 on a one-token ring, 4x4 elsewhere).""" | |
| import numpy as np | |
| from PIL import Image, ImageDraw | |
| from mixed_res import region_layout | |
| from render_h3 import load_prompts | |
| from train_h3 import list_items | |
| dit = load_base(args.swap) | |
| upscale = load_upscaler() | |
| saved = load_region(args.region) | |
| head = RegionHead(saved["hidden"]).cuda().eval() | |
| head.load_state_dict(saved["state"]) | |
| block = saved["block"] | |
| width, height = args.size | |
| from safetensors import safe_open | |
| from build_pairs_tt import caption_for | |
| prompts = [] | |
| for cache in args.cache_list or [None]: | |
| if cache is None: | |
| prompts += load_prompts(args.prompts, None) | |
| continue | |
| for te in [te for _l, te in list_items(cache, args.holdout)[1]][:args.prompts]: | |
| with safe_open(str(te), "pt") as handle: | |
| hidden, keep = handle.get_tensor("hidden_states"), handle.get_tensor("attention_mask") | |
| prompts.append((caption_for(str(te)) or te.name, hidden[keep].unsqueeze(0))) | |
| colors = {1: (74, 222, 128), 4: (139, 180, 240), 16: (192, 132, 252)} | |
| rows, meta = [], [] | |
| for index, (caption, embeds) in enumerate(prompts): | |
| latent, _x0, states = native_sample(dit, embeds.to("cuda", torch.bfloat16), height=height // 2, | |
| width=width // 2, steps=args.steps, seed=args.seed + index, | |
| capture_at=args.switch - 1, block=block) | |
| tok_h, tok_w = height // 64, width // 64 | |
| scores = head(states).float().cpu().reshape(1, tok_h, tok_w) | |
| mask = region_mask(scores, alpha=args.alpha, floor=args.floor) | |
| full = mask.repeat_interleave(2, 1).repeat_interleave(2, 2) | |
| layout = region_layout(full, background=4, ring=1) | |
| image = to_image_from(upscale.vae.decode(latent.to("cuda", upscale.dtype))).resize((width, height), Image.LANCZOS) | |
| # Left: heatmap of the head's probability with the selected region outlined. | |
| prob = torch.sigmoid(scores[0]).numpy() | |
| heat = Image.fromarray((np.clip(prob, 0, 1) * 255).astype(np.uint8)).resize((width, height), Image.NEAREST) | |
| red = Image.merge("RGB", (heat, Image.new("L", heat.size, 40), Image.new("L", heat.size, 40))) | |
| left = outline(Image.blend(image, red, 0.45), mask) | |
| # Right: the LoT layout, every token drawn at full-resolution token size (32 px). | |
| right = image.copy() | |
| draw = ImageDraw.Draw(right, "RGBA") | |
| for rect in layout.rects: | |
| x0, y0 = rect.v * 32, rect.u * 32 | |
| area = rect.eh * rect.ew | |
| r, g, b = colors.get(area, (167, 139, 250)) | |
| draw.rectangle([x0, y0, x0 + rect.ew * 32 - 1, y0 + rect.eh * 32 - 1], outline=(r, g, b, 230), | |
| fill=(r, g, b, 40 if area > 1 else 70)) | |
| row = Image.new("RGB", (2 * width, height + 30), "white") | |
| row.paste(left, (0, 30)) | |
| row.paste(right, (width, 30)) | |
| label = f"{caption[:110]} | region {float(mask.float().mean()):.0%} of frame, layout {layout.count} of {tok_h * tok_w * 4} tokens" | |
| ImageDraw.Draw(row).text((8, 9), label, fill="black") | |
| rows.append(row) | |
| meta.append({"caption": caption, "region_share": round(float(mask.float().mean()), 4), | |
| "layout_tokens": layout.count, "full_tokens": tok_h * tok_w * 4}) | |
| print(f"LOT_H3 kind=layout prompt={index} region={float(mask.float().mean()):.2f} tokens={layout.count}", | |
| flush=True) | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| scale = args.sheet_scale | |
| cell = (int(2 * width * scale), int((height + 30) * scale)) | |
| for start in range(0, len(rows), args.per_sheet): | |
| chunk = rows[start:start + args.per_sheet] | |
| sheet = Image.new("RGB", (cell[0], cell[1] * len(chunk)), "white") | |
| for number, row in enumerate(chunk): | |
| sheet.paste(row.resize(cell, Image.LANCZOS), (0, number * cell[1])) | |
| sheet.save(args.out / f"layout_{start // args.per_sheet}.jpg", quality=90) | |
| (args.out / "layout.json").write_text(json.dumps({"switch": args.switch, "alpha": args.alpha, | |
| "region": str(args.region), "items": meta}, indent=1) + "\n") | |
| print(f"LOT_H3 kind=layout_done out={args.out} prompts={len(rows)} ok=1", flush=True) | |
| def save_clip(pixels: torch.Tensor, path: Path, fps: int = 24) -> None: | |
| """(1, 3, F, H, W) in [-1, 1] -> mp4.""" | |
| import imageio | |
| frames = ((pixels[0].float().clamp(-1, 1) + 1) * 127.5).round().byte().permute(1, 2, 3, 0).cpu().numpy() | |
| imageio.mimwrite(str(path), list(frames), fps=fps, quality=8, macro_block_size=1) | |
| def to_image_from(pixels): | |
| from render_h3 import to_image | |
| return to_image(pixels) | |
| def outline(image, mask: torch.Tensor): | |
| """Draw the region's boundary (one LR token is 64 px).""" | |
| from PIL import ImageDraw | |
| image = image.copy() | |
| draw = ImageDraw.Draw(image) | |
| grid = mask[0] | |
| cell = image.height // grid.shape[0] | |
| for u in range(grid.shape[0]): | |
| for v in range(grid.shape[1]): | |
| if not grid[u, v]: | |
| continue | |
| x, y = v * cell, u * cell | |
| if u == 0 or not grid[u - 1, v]: | |
| draw.line([(x, y), (x + cell, y)], fill="white", width=2) | |
| if u == grid.shape[0] - 1 or not grid[u + 1, v]: | |
| draw.line([(x, y + cell), (x + cell, y + cell)], fill="white", width=2) | |
| if v == 0 or not grid[u, v - 1]: | |
| draw.line([(x, y), (x, y + cell)], fill="white", width=2) | |
| if v == grid.shape[1] - 1 or not grid[u, v + 1]: | |
| draw.line([(x + cell, y), (x + cell, y + cell)], fill="white", width=2) | |
| return image | |
| def train_items(args): | |
| from train_h3 import CACHE, list_items | |
| train, _held = list_items(args.cache or CACHE, args.holdout) | |
| random.Random(args.seed).shuffle(train) | |
| return train[:args.items] | |
| def crop_even(x0: torch.Tensor) -> torch.Tensor: | |
| """Crop a cached latent so its LR version has an even latent size (HR latent sides multiple of 4).""" | |
| height, width = x0.shape[-2] // 4 * 4, x0.shape[-1] // 4 * 4 | |
| return x0[..., :height, :width] | |
| def fit_kv(args) -> None: | |
| """Ridge fit of ``f_up`` (their Sec. 3.1): the base model's keys/values at LR and at HR on the | |
| same images and noise level, each LR token paired with its four HR tokens.""" | |
| from train_h3 import load_item | |
| dit = load_base(args.swap) | |
| layers = len(dit.blocks) | |
| maps = UpMaps(layers, HEAD_DIM, [EXTENT], rank=args.rank).cuda() | |
| moments: dict = {} | |
| if args.pairs: | |
| from build_pairs_tt import list_pairs, load_pair | |
| sources = [("pair", entry) for entry in list_pairs(args.pairs, 8)[0][:args.items]] | |
| else: | |
| upscale = load_upscaler() | |
| sources = [("cache", item) for item in train_items(args)] | |
| for index, (kind, item) in enumerate(sources): | |
| if kind == "pair": | |
| pair = load_pair(item) | |
| hr0, lr0, text = pair["hr"], pair["lr"], pair["text"] | |
| else: | |
| x0, text = load_item(item, random.Random(index), "cuda") | |
| hr0 = crop_even(x0) | |
| lr0 = upscale.down(hr0) | |
| plan = TwoTrackPlan(torch.zeros(1, lr0.shape[-2] // 2, lr0.shape[-1] // 2, dtype=torch.bool)) | |
| cells = torch.argsort(plan.cell_order).reshape(plan.lr_count, 4).cuda() # LR token -> its HR rows | |
| for sigma in args.sigmas: | |
| gen = torch.Generator("cuda").manual_seed(index) | |
| kv_lr, kv_hr = {}, {} | |
| x_lr = (1.0 - sigma) * lr0 + sigma * torch.randn(lr0.shape, device="cuda", generator=gen) | |
| x_hr = (1.0 - sigma) * hr0 + sigma * torch.randn(hr0.shape, device="cuda", generator=gen) | |
| two_track_forward(dit, x_lr, None, None, 1.0 - sigma, text, capture={"kv": kv_lr}) | |
| two_track_forward(dit, x_hr, None, None, 1.0 - sigma, text, capture={"kv": kv_hr}) | |
| for layer in kv_lr: | |
| for which, slot in (("k", 0), ("v", 1)): | |
| x = kv_lr[layer][slot][0].float().reshape(-1, HEAD_DIM) # (N*H, d) | |
| y = kv_hr[layer][slot][0].float()[cells].transpose(1, 2).reshape(-1, 4, HEAD_DIM) | |
| delta = (y - x.unsqueeze(1)).reshape(x.shape[0], -1) | |
| entry = moments.setdefault((layer, which), [0.0, 0.0, 0, 0.0, 0.0]) | |
| entry[0] = entry[0] + x.T @ x | |
| entry[1] = entry[1] + x.T @ delta | |
| entry[2] += x.shape[0] | |
| entry[3] += float(delta.pow(2).sum()) # copy-init residual | |
| entry[4] += float(y.pow(2).sum()) | |
| print(f"LOT_H3 kind=tt_fit_kv item={index}", flush=True) | |
| report = {"k": [], "v": []} | |
| for (layer, which), (xtx, xty, count, copy_err, energy) in moments.items(): | |
| maps.fit_moments_(layer, EXTENT, which, xtx, xty, count, args.ridge) | |
| name = f"l{layer}_2x2_{which}" | |
| weight = maps.down[name].float() @ maps.up[name].float() | |
| # |Y - XW|^2 = |Y|^2 - 2 tr(W^T X^T Y) + tr(W^T X^T X W), on the training moments. | |
| fit_err = copy_err - 2.0 * float((weight * xty).sum()) + float((weight * (xtx @ weight)).sum()) | |
| report[which].append((layer, (copy_err / energy) ** 0.5, (max(fit_err, 0.0) / energy) ** 0.5)) | |
| for which in ("k", "v"): | |
| rows = sorted(report[which]) | |
| copy = sum(r[1] for r in rows) / len(rows) | |
| fit = sum(r[2] for r in rows) / len(rows) | |
| print(f"LOT_H3 kind=tt_fit_kv_residual which={which} copy_rel={copy:.3f} fit_rel={fit:.3f}", flush=True) | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| torch.save({"state": maps.state_dict(), "rank": args.rank, "layers": layers, "sigmas": args.sigmas, | |
| "residual": report}, args.out / "up_maps_tt.pt") | |
| print(f"LOT_H3 kind=tt_fit_kv_done out={args.out / 'up_maps_tt.pt'} ok=1", flush=True) | |
| def train_region(args) -> None: | |
| """Linear head on the LR track's block states at the switch. Label (their self-supervised one, in | |
| latent space): the ``--fraction`` of LR tokens where the upscaled LR clean estimate is furthest | |
| from the true HR latent.""" | |
| from h3_musubi import sigma_schedule | |
| from train_h3 import load_item | |
| dit = load_base(args.swap) | |
| upscale = load_upscaler() | |
| sigma = float(sigma_schedule(args.steps)[args.switch - 1]) | |
| pairs = [] | |
| with torch.no_grad(): | |
| for index, item in enumerate(train_items(args)): | |
| x0, text = load_item(item, random.Random(index), "cuda") | |
| hr0 = crop_even(x0) | |
| lr0 = upscale.down(hr0) | |
| gen = torch.Generator("cuda").manual_seed(index) | |
| x_lr = (1.0 - sigma) * lr0 + sigma * torch.randn(lr0.shape, device="cuda", generator=gen) | |
| capture = {"block": args.block} | |
| velocity, _ = two_track_forward(dit, x_lr, None, None, 1.0 - sigma, text, capture=capture) | |
| error = (upscale(x_lr + sigma * velocity.float()) - hr0).abs().mean(dim=1) # (1, 4h, 4w) latent px | |
| error = F.avg_pool2d(error, 4).flatten() # LR tokens | |
| label = (error >= torch.quantile(error, 1.0 - args.fraction)).float() | |
| pairs.append((capture["states"].cpu(), label.cpu())) | |
| print(f"LOT_H3 kind=tt_region_pair item={index} tokens={label.numel()}", flush=True) | |
| del dit | |
| torch.cuda.empty_cache() | |
| head = RegionHead(pairs[0][0].shape[-1]).cuda() | |
| optimizer = torch.optim.Adam(head.parameters(), lr=args.lr) | |
| held = pairs[: max(1, len(pairs) // 10)] | |
| train = pairs[len(held):] or pairs | |
| for epoch in range(args.epochs): | |
| random.Random(epoch).shuffle(train) | |
| total = 0.0 | |
| for states, target in train: | |
| loss = F.binary_cross_entropy_with_logits(head(states.cuda()), target.cuda()) | |
| optimizer.zero_grad(set_to_none=True) | |
| loss.backward() | |
| optimizer.step() | |
| total += float(loss.detach()) | |
| with torch.no_grad(): | |
| held_bce = sum(float(F.binary_cross_entropy_with_logits(head(s.cuda()), t.cuda())) for s, t in held) | |
| print(f"LOT_H3 kind=tt_region_epoch epoch={epoch} train_bce={total / len(train):.4f} " | |
| f"held_bce={held_bce / len(held):.4f}", flush=True) | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| torch.save({"state": head.state_dict(), "hidden": pairs[0][0].shape[-1], "block": args.block, | |
| "switch": args.switch, "steps": args.steps, "fraction": args.fraction}, | |
| args.out / "region_head_tt.pt") | |
| print(f"LOT_H3 kind=tt_region_done out={args.out / 'region_head_tt.pt'} ok=1", flush=True) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) | |
| parser.add_argument("phase", choices=("render", "fit-kv", "region", "layout")) | |
| parser.add_argument("--cache-list", type=Path, nargs="+", default=None, | |
| help="layout: caches whose held-out captions are the prompts (--prompts from each)") | |
| parser.add_argument("--per-sheet", type=int, default=6, help="layout: rows per output sheet") | |
| parser.add_argument("--sheet-scale", type=float, default=0.5, help="layout: sheet size relative to 2x canvas") | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--cache", type=Path, default=None, help="still cache with text (prompts and fit data)") | |
| parser.add_argument("--holdout", type=int, default=24) | |
| parser.add_argument("--te", type=Path, nargs="+", default=None, | |
| help="render: caption embeddings (*_minimaxh3_te.safetensors) to use as prompts instead of a " | |
| "cache's held-out items, e.g. from the HF dataset johndpope/MiniMax-H3-pexels-people-text") | |
| parser.add_argument("--items", type=int, default=24) | |
| parser.add_argument("--prompts", type=int, default=3) | |
| parser.add_argument("--seed", type=int, default=0) | |
| parser.add_argument("--swap", default="auto", help="DiT blocks streamed from CPU; auto (by free VRAM): 0 at 64 GB+, 8 at 28+, 16 at 21+, else 24") | |
| parser.add_argument("--steps", type=int, default=20) | |
| parser.add_argument("--switch", type=int, default=18, help="LR-only steps before the HR region starts") | |
| parser.add_argument("--hr-steps", type=int, default=6, | |
| help="steps after the switch, linear in sigma (0: the rest of the schedule)") | |
| parser.add_argument("--size", type=int, nargs=2, default=(1344, 768), metavar=("W", "H")) | |
| parser.add_argument("--switch-upscale", choices=("vae", "crop", "latent"), default="vae", | |
| help="render: HR start at the switch: VAE round trip of the frame, of M's bounding box only " | |
| "(crop), or latent x2 interpolation (no VAE; striped regions)") | |
| parser.add_argument("--preset", choices=("quality",), default=None, | |
| help="render: quality = switch 16, 6 HR steps, crop upscale, pixel paste, and the fitted maps " | |
| "($LOT_TT_MAPS) and region head ($LOT_TT_REGION) unless --maps / --region / --box are given") | |
| parser.add_argument("--final", choices=("latent", "pixel"), default="latent", | |
| help="render: merge the tracks in latent space (a VAE round trip of LR) or in pixel space " | |
| "(decode LR at half size, paste after decoding)") | |
| parser.add_argument("--frames", type=int, default=1, | |
| help="render: pixel frames (1 = still; 22 = 7 latent frames). Clips also save p<i>_<variant>.mp4") | |
| parser.add_argument("--variants", default="hr,lr,two_track") | |
| parser.add_argument("--maps", type=Path, default=None, help="up_maps_tt.pt from fit-kv (default: copy)") | |
| parser.add_argument("--train", type=Path, default=None, | |
| help="train_tt.py lora output dir: LoRA + fine-tuned maps + resolution embedding (overrides --maps)") | |
| parser.add_argument("--tag", default="", help="checkpoint tag in --train, e.g. _1000") | |
| parser.add_argument("--decoder", type=Path, default=None, help="decoder.pt from train_tt.py decoder (U and G)") | |
| parser.add_argument("--pairs", type=Path, action="append", default=None, | |
| help="fit-kv: build_pairs_tt.py dirs (HR + LR latents precomputed; repeatable)") | |
| parser.add_argument("--region", type=Path, default=None, help="region_head_tt.pt (default: --box)") | |
| parser.add_argument("--box", type=float, nargs=4, default=None, metavar=("X0", "Y0", "X1", "Y1")) | |
| parser.add_argument("--alpha", type=float, default=0.5) | |
| parser.add_argument("--floor", type=float, default=None) | |
| parser.add_argument("--feather", type=float, default=4.0, | |
| help="paste-edge ramp inside the region, HR latent pixels (weight 1/R at the edge, 1 from R-1 deep)") | |
| parser.add_argument("--outline", action="store_true", help="draw the region boundary on the two_track image") | |
| parser.add_argument("--block", type=int, default=MID_BLOCK) | |
| parser.add_argument("--fraction", type=float, default=0.25) | |
| parser.add_argument("--epochs", type=int, default=20) | |
| parser.add_argument("--lr", type=float, default=1e-3) | |
| parser.add_argument("--rank", type=int, default=HEAD_DIM, help="rank of the f_up delta (head dim = full)") | |
| parser.add_argument("--ridge", type=float, default=1e-2) | |
| parser.add_argument("--sigmas", type=float, nargs="+", default=[0.2, 0.4, 0.6, 0.8, 0.95], | |
| help="fit-kv: noise levels; cover the sigmas the HR track runs at after the switch") | |
| args = parser.parse_args() | |
| if args.preset == "quality": | |
| args.switch, args.hr_steps, args.switch_upscale, args.final = 16, 6, "crop", "pixel" | |
| if args.maps is None and args.train is None: | |
| path = os.environ.get("LOT_TT_MAPS") | |
| if not path or not Path(path).is_file(): | |
| raise SystemExit("--preset quality needs fitted maps: --maps up_maps_tt.pt (fit-kv) or $LOT_TT_MAPS; " | |
| "without them the HR region has a diamond-grid grain") | |
| args.maps = Path(path) | |
| if args.region is None and args.box is None and os.environ.get("LOT_TT_REGION"): | |
| args.region = Path(os.environ["LOT_TT_REGION"]) | |
| from gpu_guard import auto_swap | |
| from h3_musubi import latent_frames as _latent_frames | |
| args.swap = auto_swap(args.swap, inference=args.phase in ("render", "layout"), | |
| latent_t=_latent_frames(getattr(args, "frames", 1))) | |
| if args.size[0] % 64 or args.size[1] % 64: | |
| raise SystemExit("--size sides must be multiples of 64 (the LR latent needs whole 2x2 patches)") | |
| refuse_if_busy("two_track_h3") | |
| {"render": render, "fit-kv": fit_kv, "region": train_region, "layout": draw_layout}[args.phase](args) | |
| if __name__ == "__main__": | |
| main() | |