File size: 6,904 Bytes
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
"""MiniMax-H3 geometry for a Level-of-Token visual stream.

H3's native patch is ``(t, h, w) = (1, 2, 2)`` on 24-channel video latents,
so the uniform token dimension is ``24 * 1 * 2 * 2 = 96``. LoT extents are
counted in those tokens, not in latent pixels. A paper video extent of
``2 x 2`` therefore covers a ``1 x 4 x 4`` block of VAE latents.

The Wan training setup in Appendix A.10 keeps ``e_t = 1`` and draws spatial
extents from ``{1, 2, 4}``, including rectangles. That is the H3 preset.
"""

from __future__ import annotations

import itertools

import torch

from adapter import LotVisualAdapter
from layout import TokenRect, dense_layout, layout_from_rects


H3_LATENT_CHANNELS = 24
H3_PATCH = (1, 2, 2)
H3_TOKEN_DIM = H3_LATENT_CHANNELS * H3_PATCH[0] * H3_PATCH[1] * H3_PATCH[2]
H3_HIDDEN = 5376
H3_HEADS = 56
H3_HEAD_DIM = 128

# Appendix A.10. Temporal extent stays 1; spatial sides are 1, 2, or 4.
H3_SPATIAL_SIDES = (1, 2, 4)
H3_EXTENTS = tuple((1, eh, ew) for eh, ew in itertools.product(H3_SPATIAL_SIDES, repeat=2))

# Image quadtree in Appendix A.3. Squares only, up to 8.
IMAGE_EXTENTS = ((1, 1, 1), (1, 2, 2), (1, 4, 4), (1, 8, 8))

# Wan's per-frame token cap. H3 resolutions differ; this is a budget check,
# not a hard limit of the adapter.
WAN_MAX_TOKENS_PER_FRAME = 3072


def patchify(latent: torch.Tensor, patch: tuple[int, int, int] = H3_PATCH) -> torch.Tensor:
    """``(B, C, T, H, W)`` -> ``(B, T', H', W', C*pt*ph*pw)``.

    Channel order inside the token is C-order ``(channel, pt, ph, pw)`` with
    ``pw`` fastest.
    """
    if latent.ndim != 5:
        raise ValueError("latent must be (B, C, T, H, W)")
    pt, ph, pw = patch
    batch, channels, time, height, width = latent.shape
    if time % pt or height % ph or width % pw:
        raise ValueError(f"latent {(time, height, width)} is not divisible by patch {patch}")
    tt, hh, ww = time // pt, height // ph, width // pw
    tokens = latent.reshape(batch, channels, tt, pt, hh, ph, ww, pw)
    tokens = tokens.permute(0, 2, 4, 6, 1, 3, 5, 7)
    return tokens.reshape(batch, tt, hh, ww, channels * pt * ph * pw)


def unpatchify(
    tokens: torch.Tensor,
    patch: tuple[int, int, int] = H3_PATCH,
    channels: int = H3_LATENT_CHANNELS,
) -> torch.Tensor:
    """Inverse of ``patchify``."""
    pt, ph, pw = patch
    if tokens.ndim != 5 or tokens.shape[-1] != channels * pt * ph * pw:
        raise ValueError("token shape does not match channels and patch")
    batch, tt, hh, ww, _dense = tokens.shape
    tokens = tokens.reshape(batch, tt, hh, ww, channels, pt, ph, pw)
    tokens = tokens.permute(0, 4, 1, 5, 2, 6, 3, 7)
    return tokens.reshape(batch, channels, tt * pt, hh * ph, ww * pw)


def make_h3_adapter(
    hidden_size: int = H3_HIDDEN,
    *,
    include_time: bool = False,
    shape_hidden: int = 256,
    extents: tuple[tuple[int, int, int], ...] = H3_EXTENTS,
) -> LotVisualAdapter:
    """Adapter whose token dim matches H3. Weights are not loaded here."""
    return LotVisualAdapter(
        H3_TOKEN_DIM,
        hidden_size,
        list(extents),
        include_time=include_time,
        shape_hidden=shape_hidden,
    )


def tokens_per_frame(layout_count: int, time: int) -> float:
    if time < 1:
        raise ValueError("time must be positive")
    return layout_count / time


def over_wan_frame_budget(layout_count: int, time: int) -> bool:
    return tokens_per_frame(layout_count, time) > WAN_MAX_TOKENS_PER_FRAME


def gate_frame_layout(height: int = 24, width: int = 42):
    """One frame of three horizontal bands: 1×1, then 2×2, then 4×2.

    The default is the 768×1344 timing clip: 24×42 tokens, 462 rectangles.
    Width 42 is not divisible by 4, so the coarse band is 4×2, not 4×4.
    Rows 0–7 are 1×1, rows 8–15 are 2×2, rows 16–23 are 4×2. The phase-4
    smoke canvas (384×640) is ``gate_frame_layout(12, 20)``.
    """
    band = height // 3
    if height % 3 or band % 4 or width % 2:
        raise ValueError(f"token grid {(height, width)} needs height a multiple of 12 and an even width")
    rects = []
    for u in range(band):
        for v in range(width):
            rects.append(TokenRect(0, u, v, 1, 1, 1))
    for u in range(band, 2 * band, 2):
        for v in range(0, width, 2):
            rects.append(TokenRect(0, u, v, 1, 2, 2))
    for u in range(2 * band, height, 4):
        for v in range(0, width, 2):
            rects.append(TokenRect(0, u, v, 1, 4, 2))
    return layout_from_rects(1, height, width, rects)


def clip_layout(frames: int, height: int = 24, width: int = 42):
    """``gate_frame_layout`` repeated on every latent frame."""
    frame = gate_frame_layout(height, width)
    rects = [
        TokenRect(t, rect.u, rect.v, 1, rect.eh, rect.ew)
        for t in range(frames)
        for rect in frame.rects
    ]
    return layout_from_rects(frames, height, width, rects)


GRID_LAYOUTS = ("dense", "bands", "center", "uniform2", "uniform4")


def grid_layout(name: str, time: int, height: int, width: int):
    """A LoT layout for any token grid, in 4×4 super-cells; edge leftovers stay 1×1.

    ``bands``: top third 1×1, middle 2×2, bottom 4×2 (by super-cell row).
    ``center``: 1×1 near the middle, 2×2 around it, 4×4 at the edges; meant for
    portraits and talking heads, where the face sits in the middle.
    ``uniformN``: every full super-cell N×N. ``dense``: all 1×1.
    """
    if name not in GRID_LAYOUTS:
        raise ValueError(f"layout {name!r} is not one of {GRID_LAYOUTS}")
    if name == "dense":
        return dense_layout(time, height, width)   # raster order: bit-identical to the stock forward
    rows, cols = height // 4, width // 4

    def extent(cell_u: int, cell_v: int) -> tuple[int, int, int]:
        if name == "dense":
            return (1, 1, 1)
        if name == "uniform2":
            return (1, 2, 2)
        if name == "uniform4":
            return (1, 4, 4)
        if name == "bands":
            third = cell_u * 3 // max(rows, 1)
            return ((1, 1, 1), (1, 2, 2), (1, 4, 2))[min(third, 2)]
        du = (cell_u + 0.5) / max(rows, 1) - 0.5
        dv = (cell_v + 0.5) / max(cols, 1) - 0.5
        radius = (du * du + dv * dv) ** 0.5 / 0.7071
        return (1, 1, 1) if radius < 0.35 else (1, 2, 2) if radius < 0.7 else (1, 4, 4)

    rects = []
    for t in range(time):
        for u in range(height):
            for v in range(width):
                if u >= rows * 4 or v >= cols * 4:
                    rects.append(TokenRect(t, u, v, 1, 1, 1))     # edge leftovers
        for cu in range(rows):
            for cv in range(cols):
                _et, eh, ew = extent(cu, cv)
                for u in range(cu * 4, cu * 4 + 4, eh):
                    for v in range(cv * 4, cv * 4 + 4, ew):
                        rects.append(TokenRect(t, u, v, 1, eh, ew))
    return layout_from_rects(time, height, width, rects)