Spaces:
Running on Zero
Running on Zero
File size: 6,904 Bytes
6febba8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | """MiniMax-H3 geometry for a Level-of-Token visual stream.
H3's native patch is ``(t, h, w) = (1, 2, 2)`` on 24-channel video latents,
so the uniform token dimension is ``24 * 1 * 2 * 2 = 96``. LoT extents are
counted in those tokens, not in latent pixels. A paper video extent of
``2 x 2`` therefore covers a ``1 x 4 x 4`` block of VAE latents.
The Wan training setup in Appendix A.10 keeps ``e_t = 1`` and draws spatial
extents from ``{1, 2, 4}``, including rectangles. That is the H3 preset.
"""
from __future__ import annotations
import itertools
import torch
from adapter import LotVisualAdapter
from layout import TokenRect, dense_layout, layout_from_rects
H3_LATENT_CHANNELS = 24
H3_PATCH = (1, 2, 2)
H3_TOKEN_DIM = H3_LATENT_CHANNELS * H3_PATCH[0] * H3_PATCH[1] * H3_PATCH[2]
H3_HIDDEN = 5376
H3_HEADS = 56
H3_HEAD_DIM = 128
# Appendix A.10. Temporal extent stays 1; spatial sides are 1, 2, or 4.
H3_SPATIAL_SIDES = (1, 2, 4)
H3_EXTENTS = tuple((1, eh, ew) for eh, ew in itertools.product(H3_SPATIAL_SIDES, repeat=2))
# Image quadtree in Appendix A.3. Squares only, up to 8.
IMAGE_EXTENTS = ((1, 1, 1), (1, 2, 2), (1, 4, 4), (1, 8, 8))
# Wan's per-frame token cap. H3 resolutions differ; this is a budget check,
# not a hard limit of the adapter.
WAN_MAX_TOKENS_PER_FRAME = 3072
def patchify(latent: torch.Tensor, patch: tuple[int, int, int] = H3_PATCH) -> torch.Tensor:
"""``(B, C, T, H, W)`` -> ``(B, T', H', W', C*pt*ph*pw)``.
Channel order inside the token is C-order ``(channel, pt, ph, pw)`` with
``pw`` fastest.
"""
if latent.ndim != 5:
raise ValueError("latent must be (B, C, T, H, W)")
pt, ph, pw = patch
batch, channels, time, height, width = latent.shape
if time % pt or height % ph or width % pw:
raise ValueError(f"latent {(time, height, width)} is not divisible by patch {patch}")
tt, hh, ww = time // pt, height // ph, width // pw
tokens = latent.reshape(batch, channels, tt, pt, hh, ph, ww, pw)
tokens = tokens.permute(0, 2, 4, 6, 1, 3, 5, 7)
return tokens.reshape(batch, tt, hh, ww, channels * pt * ph * pw)
def unpatchify(
tokens: torch.Tensor,
patch: tuple[int, int, int] = H3_PATCH,
channels: int = H3_LATENT_CHANNELS,
) -> torch.Tensor:
"""Inverse of ``patchify``."""
pt, ph, pw = patch
if tokens.ndim != 5 or tokens.shape[-1] != channels * pt * ph * pw:
raise ValueError("token shape does not match channels and patch")
batch, tt, hh, ww, _dense = tokens.shape
tokens = tokens.reshape(batch, tt, hh, ww, channels, pt, ph, pw)
tokens = tokens.permute(0, 4, 1, 5, 2, 6, 3, 7)
return tokens.reshape(batch, channels, tt * pt, hh * ph, ww * pw)
def make_h3_adapter(
hidden_size: int = H3_HIDDEN,
*,
include_time: bool = False,
shape_hidden: int = 256,
extents: tuple[tuple[int, int, int], ...] = H3_EXTENTS,
) -> LotVisualAdapter:
"""Adapter whose token dim matches H3. Weights are not loaded here."""
return LotVisualAdapter(
H3_TOKEN_DIM,
hidden_size,
list(extents),
include_time=include_time,
shape_hidden=shape_hidden,
)
def tokens_per_frame(layout_count: int, time: int) -> float:
if time < 1:
raise ValueError("time must be positive")
return layout_count / time
def over_wan_frame_budget(layout_count: int, time: int) -> bool:
return tokens_per_frame(layout_count, time) > WAN_MAX_TOKENS_PER_FRAME
def gate_frame_layout(height: int = 24, width: int = 42):
"""One frame of three horizontal bands: 1×1, then 2×2, then 4×2.
The default is the 768×1344 timing clip: 24×42 tokens, 462 rectangles.
Width 42 is not divisible by 4, so the coarse band is 4×2, not 4×4.
Rows 0–7 are 1×1, rows 8–15 are 2×2, rows 16–23 are 4×2. The phase-4
smoke canvas (384×640) is ``gate_frame_layout(12, 20)``.
"""
band = height // 3
if height % 3 or band % 4 or width % 2:
raise ValueError(f"token grid {(height, width)} needs height a multiple of 12 and an even width")
rects = []
for u in range(band):
for v in range(width):
rects.append(TokenRect(0, u, v, 1, 1, 1))
for u in range(band, 2 * band, 2):
for v in range(0, width, 2):
rects.append(TokenRect(0, u, v, 1, 2, 2))
for u in range(2 * band, height, 4):
for v in range(0, width, 2):
rects.append(TokenRect(0, u, v, 1, 4, 2))
return layout_from_rects(1, height, width, rects)
def clip_layout(frames: int, height: int = 24, width: int = 42):
"""``gate_frame_layout`` repeated on every latent frame."""
frame = gate_frame_layout(height, width)
rects = [
TokenRect(t, rect.u, rect.v, 1, rect.eh, rect.ew)
for t in range(frames)
for rect in frame.rects
]
return layout_from_rects(frames, height, width, rects)
GRID_LAYOUTS = ("dense", "bands", "center", "uniform2", "uniform4")
def grid_layout(name: str, time: int, height: int, width: int):
"""A LoT layout for any token grid, in 4×4 super-cells; edge leftovers stay 1×1.
``bands``: top third 1×1, middle 2×2, bottom 4×2 (by super-cell row).
``center``: 1×1 near the middle, 2×2 around it, 4×4 at the edges; meant for
portraits and talking heads, where the face sits in the middle.
``uniformN``: every full super-cell N×N. ``dense``: all 1×1.
"""
if name not in GRID_LAYOUTS:
raise ValueError(f"layout {name!r} is not one of {GRID_LAYOUTS}")
if name == "dense":
return dense_layout(time, height, width) # raster order: bit-identical to the stock forward
rows, cols = height // 4, width // 4
def extent(cell_u: int, cell_v: int) -> tuple[int, int, int]:
if name == "dense":
return (1, 1, 1)
if name == "uniform2":
return (1, 2, 2)
if name == "uniform4":
return (1, 4, 4)
if name == "bands":
third = cell_u * 3 // max(rows, 1)
return ((1, 1, 1), (1, 2, 2), (1, 4, 2))[min(third, 2)]
du = (cell_u + 0.5) / max(rows, 1) - 0.5
dv = (cell_v + 0.5) / max(cols, 1) - 0.5
radius = (du * du + dv * dv) ** 0.5 / 0.7071
return (1, 1, 1) if radius < 0.35 else (1, 2, 2) if radius < 0.7 else (1, 4, 4)
rects = []
for t in range(time):
for u in range(height):
for v in range(width):
if u >= rows * 4 or v >= cols * 4:
rects.append(TokenRect(t, u, v, 1, 1, 1)) # edge leftovers
for cu in range(rows):
for cv in range(cols):
_et, eh, ew = extent(cu, cv)
for u in range(cu * 4, cu * 4 + 4, eh):
for v in range(cv * 4, cv * 4 + 4, ew):
rects.append(TokenRect(t, u, v, 1, eh, ew))
return layout_from_rects(time, height, width, rects)
|