mpeg-diffusion / lot /mixed_res.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
6febba8 verified
Raw History Blame Contribute Delete
26.9 kB
"""Mixed-resolution additions to LoT, adapted from MPEG-Diffusion (Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu, Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu, 2026).
MPEG-Diffusion: Mixed-Resolution Video Generation, Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu,
Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu (UT Austin, Roblox), arXiv:2611.12069. That paper
keeps two token grids (HR inside a mask, LR elsewhere). LoT has one sequence of
mixed extents. The three pieces here carry its ideas over to that sequence:
1. ``DualKV`` (their dual-resolution attention, Sec. 3). Their Proposition 1:
with one shared K/V, a coarse token can stand in for ``n`` fine tokens under
softmax only when those fine tokens are identical. So 1x1 queries here attend
to a *dense* key/value view: every coarse token's key and value are expanded
to its ``n`` cells by ``f_up(k) = k + U V^T k`` (their "linear delta":
exact copy plus a rank-``r`` correction, zero at init), before RoPE, and the
expanded keys are rotated at the cells' dense positions. Every other query
(text, audio, coarse tokens) keeps LoT's native attention, which already sees
each fine token individually, so no ``f_down`` is needed. With zero deltas a
fine query sees a coarse token as ``n`` copies at its cells.
2. ``RegionHead`` (their self-selected HR region, Sec. 3.4): ``s = w^T LN(h)``
on a middle block's video tokens, smoothed in space and time, thresholded at
``m + alpha (p - m)`` per video (median ``m``, 99th percentile ``p``), all
coarse when ``p`` is under a floor. The mask becomes a per-latent-frame LoT
layout. Labels here are self-supervised in latent space (``region_labels``):
tokens where the coarse layout's clean estimate is furthest from the dense
model's, a latent stand-in for their per-patch LPIPS label.
3. ``SeamMixer`` (their mixed-resolution decoder's mixer ``G``, eq. 6): a small
3D-conv residual on the final latent, applied outside the fine region only,
zero-initialised so it starts as the identity. LoT already returns a dense
latent, so there is no upscaler; the residual targets coarse-token blocking
and the seams between extents.
Nothing here imports model code; ``h3_musubi`` wires it into H3's forward.
"""
from __future__ import annotations
import contextlib
import math
import types
import torch
import torch.nn as nn
import torch.nn.functional as F
from layout import LotLayout, smooth_detail
from layout_sources import layout_from_desired_any
# --------------------------------------------------------------------- 2. dual K/V
class DualKVPlan:
"""Index tensors for one layout: which video rows are 1x1, and how to assemble the dense view.
The expanded order is: the 1x1 tokens (in layout order), then for each larger extent its
tokens, each followed by its ``eh * ew`` cells in row-major order. ``dense_order`` permutes
that into dense ``(t, u, v)`` order, the order of H3's dense video positions.
"""
def __init__(self, layout: LotLayout):
self.layout = layout
fine, groups = [], []
cells = []
for index, rect in enumerate(layout.rects):
if rect.extent == (1, 1, 1):
fine.append(index)
width, plane = layout.width, layout.height * layout.width
for index in fine:
rect = layout.rects[index]
cells.append(rect.t * plane + rect.u * width + rect.v)
for extent, indices in layout.groups():
if extent == (1, 1, 1):
continue
groups.append((extent, torch.tensor(indices, dtype=torch.long)))
for index in indices:
rect = layout.rects[index]
for du in range(rect.eh):
for dv in range(rect.ew):
cells.append(rect.t * plane + (rect.u + du) * width + rect.v + dv)
cells = torch.tensor(cells, dtype=torch.long)
if cells.numel() != layout.dense_count or torch.unique(cells).numel() != layout.dense_count:
raise ValueError("layout does not cover the dense grid exactly once")
self.fine_rows = torch.tensor(fine, dtype=torch.long)
fine_mask = torch.zeros(layout.count, dtype=torch.bool)
fine_mask[self.fine_rows] = True
self.other_rows = torch.nonzero(~fine_mask).squeeze(1)
self.groups = groups
self.dense_order = torch.argsort(cells)
def to(self, device) -> "DualKVPlan":
self.fine_rows = self.fine_rows.to(device)
self.other_rows = self.other_rows.to(device)
self.dense_order = self.dense_order.to(device)
self.groups = [(extent, rows.to(device)) for extent, rows in self.groups]
return self
def _key(layer: int, extent: tuple[int, int, int], which: str) -> str:
return f"l{layer}_{extent[1]}x{extent[2]}_{which}"
class UpMaps(nn.Module):
"""``f_up`` per block, extent, and K or V: ``k -> k + (k V) U`` reshaped to ``n`` cells.
``V`` is ``(d, r)``, ``U`` is ``(r, n * d)``, shared by all heads of a block (the paper fits
one map per layer). ``U`` starts at zero, so the map starts as an exact copy.
"""
def __init__(self, layers: int, head_dim: int, extents, rank: int = 16):
super().__init__()
self.head_dim, self.rank = int(head_dim), int(rank)
self.extents = [tuple(map(int, e)) for e in extents if tuple(e) != (1, 1, 1)]
self.down = nn.ParameterDict()
self.up = nn.ParameterDict()
for layer in range(layers):
for extent in self.extents:
cells = extent[1] * extent[2]
for which in ("k", "v"):
name = _key(layer, extent, which)
down = torch.empty(self.head_dim, self.rank)
nn.init.normal_(down, std=1.0 / math.sqrt(self.head_dim))
self.down[name] = nn.Parameter(down)
self.up[name] = nn.Parameter(torch.zeros(self.rank, cells * self.head_dim))
def expand(self, layer: int, extent: tuple[int, int, int], which: str, rows: torch.Tensor) -> torch.Tensor:
"""``(B, N, H, d)`` -> ``(B, N * n, H, d)``: each token, then its cells row-major."""
name = _key(layer, extent, which)
cells = extent[1] * extent[2]
batch, count, heads, dim = rows.shape
delta = (rows @ self.down[name].to(rows.dtype)) @ self.up[name].to(rows.dtype) # (B, N, H, n*d)
delta = delta.reshape(batch, count, heads, cells, dim).transpose(2, 3) # (B, N, n, H, d)
expanded = rows.unsqueeze(2) + delta
return expanded.reshape(batch, count * cells, heads, dim)
@torch.no_grad()
def fit_(self, layer: int, extent: tuple[int, int, int], which: str,
coarse: torch.Tensor, cells: torch.Tensor, ridge: float = 1e-2) -> float:
"""Ridge fit of the delta (their init), truncated to rank ``r`` by SVD.
``coarse`` is ``(N, d)`` keys (or values) of coarse tokens, ``cells`` is ``(N, n, d)`` the
dense model's keys at the cells they cover. Solves ``min |coarse W - (cells - coarse)|^2 +
ridge |W|^2`` for ``W`` ``(d, n*d)``, keeps the top ``r`` singular directions. Returns
the relative residual of the copy-plus-delta fit."""
name = _key(layer, extent, which)
x = coarse.float()
target = (cells.float() - x.unsqueeze(1)).reshape(x.shape[0], -1)
self.fit_moments_(layer, extent, which, x.T @ x, x.T @ target, x.shape[0], ridge)
fitted = x.unsqueeze(1) + ((x @ self.down[name].float()) @ self.up[name].float()).reshape_as(cells)
return float((fitted - cells.float()).norm() / cells.float().norm().clamp(min=1e-8))
@torch.no_grad()
def fit_moments_(self, layer: int, extent: tuple[int, int, int], which: str,
xtx: torch.Tensor, xty: torch.Tensor, count: int, ridge: float = 1e-2) -> None:
"""``fit_`` from accumulated ``X^T X`` ``(d, d)`` and ``X^T (cells - X)`` ``(d, n*d)`` over ``count`` rows."""
name = _key(layer, extent, which)
gram = xtx.float() + ridge * count * torch.eye(xtx.shape[0], device=xtx.device)
weight = torch.linalg.solve(gram, xty.float()) # (d, n*d)
u, s, vh = torch.linalg.svd(weight, full_matrices=False)
r = self.rank
self.down[name].copy_((u[:, :r] * s[:r].sqrt()).to(self.down[name].dtype))
self.up[name].copy_((s[:r].sqrt().unsqueeze(1) * vh[:r]).to(self.up[name].dtype))
class DualKV:
"""``UpMaps`` plus a per-layout plan cache; what ``h3_musubi.predict(..., dual=)`` takes."""
def __init__(self, maps: UpMaps):
self.maps = maps
self._plans: dict = {}
def plan(self, layout: LotLayout, device) -> DualKVPlan:
key = (layout, str(device))
if key not in self._plans:
if len(self._plans) > 8:
self._plans.clear()
self._plans[key] = DualKVPlan(layout).to(device)
return self._plans[key]
def _rope(states: torch.Tensor, table: torch.Tensor) -> torch.Tensor:
"""H3's split-half RoPE (``model._apply_rope_split_half``), repeated so this file needs no model import."""
pairs = table.shape[-3]
a, b = states[..., :pairs], states[..., pairs:2 * pairs]
first = table[..., 0, 0] * a + table[..., 0, 1] * b # element-wise: no broadcast matmul over heads
second = table[..., 1, 0] * a + table[..., 1, 1] * b
return torch.cat((first, second, states[..., 2 * pairs:]), dim=-1)
def _sdpa(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor) -> torch.Tensor:
"""``(B, L, H, d)`` in, ``(B, L, H * d)`` out."""
out = F.scaled_dot_product_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2))
return out.transpose(1, 2).reshape(query.shape[0], query.shape[1], -1)
def dual_attention(attn, hidden: torch.Tensor, rotation: torch.Tensor, *, layer: int, plan: DualKVPlan,
maps: UpMaps | None, video_start: int, dense_rotation: torch.Tensor,
capture: dict | None = None) -> torch.Tensor:
"""H3 ``Attention.forward`` with the dense K/V view for 1x1 queries (see module docstring).
``hidden`` is the packed ``(1, S, D)`` sequence whose tail from ``video_start`` is the LoT video;
``rotation`` its rotary table; ``dense_rotation`` the table of the dense video positions.
``capture`` (optional) receives this layer's pre-RoPE video keys/values for ``UpMaps.fit_``.
``maps=None`` is H3's own attention (capture only).
"""
batch, length, _ = hidden.shape
query, key, value = attn.qkv_proj(hidden).split(attn.inner_dim, dim=-1)
query = attn.q_norm(query.reshape(batch, length, attn.num_heads, attn.head_dim))
key = attn.k_norm(key.reshape(batch, length, attn.num_heads, attn.head_dim))
value = value.reshape(batch, length, attn.num_heads, attn.head_dim).contiguous()
if capture is not None:
capture[layer] = (key[:, video_start:].detach(), value[:, video_start:].detach())
query_r = _rope(query, rotation).to(value)
key_r = _rope(key, rotation).to(value)
if maps is None:
return attn.out_proj(_sdpa(query_r, key_r, value))
# Native LoT attention for text, audio and coarse queries.
other = torch.cat([torch.arange(video_start, device=hidden.device), video_start + plan.other_rows])
output = hidden.new_zeros(batch, length, attn.inner_dim, dtype=value.dtype)
output[:, other] = _sdpa(query_r[:, other], key_r, value)
if plan.fine_rows.numel():
# Dense view for 1x1 queries: fine keys as they are, coarse keys expanded to their cells.
video_key, video_value = key[:, video_start:], value[:, video_start:]
keys = [video_key[:, plan.fine_rows]]
values = [video_value[:, plan.fine_rows]]
for extent, rows in plan.groups:
keys.append(maps.expand(layer, extent, "k", video_key[:, rows]))
values.append(maps.expand(layer, extent, "v", video_value[:, rows]))
dense_key = torch.cat(keys, dim=1)[:, plan.dense_order]
dense_value = torch.cat(values, dim=1)[:, plan.dense_order]
dense_key = _rope(dense_key, dense_rotation).to(value)
fine = video_start + plan.fine_rows
output[:, fine] = _sdpa(query_r[:, fine],
torch.cat([key_r[:, :video_start], dense_key], dim=1),
torch.cat([value[:, :video_start], dense_value], dim=1))
return attn.out_proj(output)
@contextlib.contextmanager
def patched_attention(blocks, attention=None, **context):
"""Route every block's ``attn.forward`` through ``dual_attention`` for one forward pass.
musubi-tuner's code is unchanged: the instance attribute shadows the class method and is
removed on exit, so checkpointing and block swap see the same modules."""
patched = []
try:
for layer, block in enumerate(blocks):
attn = block.attn
def forward(self, hidden, rotation=None, _layer=layer):
return (attention or dual_attention)(self, hidden, rotation, layer=_layer, **context)
attn.forward = types.MethodType(forward, attn)
patched.append(attn)
yield
finally:
for attn in patched:
del attn.forward
# ------------------------------------------------------------- 4. two native tracks
class TwoTrackPlan:
"""Index tensors for the paper's own arrangement: a complete native LR grid plus HR tokens on a mask.
``mask`` is ``(T, h, w)`` bool on the LR token grid; every masked LR token has ``scale * scale``
HR tokens. ``hr_index`` are their rows in the HR grid's raster order (the order of the HR
track's rows), and ``cell_order`` permutes "LR token, then its cells row-major" into HR raster.
"""
def __init__(self, mask: torch.Tensor, scale: int = 2):
mask = mask.bool().cpu()
time, height, width = mask.shape
self.mask, self.scale = mask, int(scale)
self.lr_count = time * height * width
self.hr_shape = (time, height * scale, width * scale)
up = mask.repeat_interleave(scale, 1).repeat_interleave(scale, 2)
self.hr_index = torch.nonzero(up.flatten()).squeeze(1)
t, u, v = torch.meshgrid(torch.arange(time), torch.arange(height), torch.arange(width), indexing="ij")
du, dv = torch.meshgrid(torch.arange(scale), torch.arange(scale), indexing="ij")
cells = (t.reshape(-1, 1) * (height * scale) + u.reshape(-1, 1) * scale + du.reshape(1, -1)) * (width * scale) \
+ v.reshape(-1, 1) * scale + dv.reshape(1, -1) # (lr, scale^2)
self.cell_order = torch.argsort(cells.flatten())
self.extent = (1, self.scale, self.scale)
@property
def hr_count(self) -> int:
return int(self.hr_index.numel())
def grid_source(self, device) -> torch.Tensor:
"""For each HR-grid cell (raster order), its row in ``cat([LR tokens, HR tokens])``: the LR token that
covers it, or its own HR token inside the mask. One gather builds the whole grid."""
cached = getattr(self, "_grid_source", None)
if cached is None or cached.device != torch.device(device):
source = torch.div(self.cell_order, self.scale * self.scale, rounding_mode="floor")
source[self.hr_index] = self.lr_count + torch.arange(self.hr_count, device=source.device)
self._grid_source = cached = source.to(device)
return cached
def to(self, device) -> "TwoTrackPlan":
self.hr_index = self.hr_index.to(device)
self.cell_order = self.cell_order.to(device)
return self
def two_track_attention(attn, hidden: torch.Tensor, rotation: torch.Tensor, *, layer: int, plan: TwoTrackPlan,
maps: UpMaps | None, video_start: int, hr_rotation: torch.Tensor,
capture: dict | None = None) -> torch.Tensor:
"""Dual-resolution attention (their eq. 3) on ``[text, audio, LR video (whole grid), HR video (mask)]``.
Text, audio and LR queries see only text, audio and LR keys: that part is the unmodified model
at LR, so it needs no ``f_down``. HR queries see text, audio and a full HR grid of keys/values:
their own inside the mask, ``f_up`` of the LR token's outside it (``maps=None`` copies), rotated
at the HR grid's positions (``hr_rotation``, the table of the whole HR grid).
"""
batch, length, _ = hidden.shape
query, key, value = attn.qkv_proj(hidden).split(attn.inner_dim, dim=-1)
query = attn.q_norm(query.reshape(batch, length, attn.num_heads, attn.head_dim))
key = attn.k_norm(key.reshape(batch, length, attn.num_heads, attn.head_dim))
value = value.reshape(batch, length, attn.num_heads, attn.head_dim).contiguous()
split = video_start + plan.lr_count
if capture is not None:
capture[layer] = (key[:, video_start:split].detach(), value[:, video_start:split].detach())
query_r = _rope(query, rotation).to(value)
key_r = _rope(key[:, :split], rotation[:, :split]).to(value) # HR rows' own rotated keys are never read
output = hidden.new_zeros(batch, length, attn.inner_dim, dtype=value.dtype)
output[:, :split] = _sdpa(query_r[:, :split], key_r, value[:, :split])
if plan.hr_count:
lr_key, lr_value = key[:, video_start:split], value[:, video_start:split]
if maps is None:
# One gather builds the HR grid: each cell takes its LR token's key (copy) or its own HR key.
source = plan.grid_source(lr_key.device)
grid_key = torch.cat([lr_key, key[:, split:]], dim=1)[:, source]
grid_value = torch.cat([lr_value, value[:, split:]], dim=1)[:, source]
else:
grid_key = maps.expand(layer, plan.extent, "k", lr_key).to(key.dtype)
grid_value = maps.expand(layer, plan.extent, "v", lr_value).to(value.dtype)
grid_key = grid_key[:, plan.cell_order].index_copy(1, plan.hr_index, key[:, split:])
grid_value = grid_value[:, plan.cell_order].index_copy(1, plan.hr_index, value[:, split:])
grid_key = _rope(grid_key, hr_rotation).to(value)
output[:, split:] = _sdpa(query_r[:, split:],
torch.cat([key_r[:, :video_start], grid_key], dim=1),
torch.cat([value[:, :video_start], grid_value], dim=1))
return attn.out_proj(output)
# ------------------------------------------------------------------ 1. region head
class RegionHead(nn.Module):
"""``s = w^T LN(h)`` per video token, on a middle block's states (their eq. 7)."""
def __init__(self, hidden_size: int):
super().__init__()
self.norm = nn.LayerNorm(hidden_size, elementwise_affine=False)
self.score = nn.Linear(hidden_size, 1)
nn.init.zeros_(self.score.weight)
nn.init.zeros_(self.score.bias)
def forward(self, states: torch.Tensor) -> torch.Tensor:
"""``(..., L, D)`` -> ``(..., L)`` logits."""
return self.score(self.norm(states.float())).squeeze(-1)
def scores_to_cells(scores: torch.Tensor, layout: LotLayout) -> torch.Tensor:
"""Per-token scores ``(L,)`` -> ``(T, H, W)``: every cell takes its token's score."""
field = scores.new_zeros(layout.time, layout.height, layout.width)
for value, rect in zip(scores, layout.rects):
field[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = value
return field
def region_mask(field: torch.Tensor, *, alpha: float = 0.5, floor: float | None = None,
sigma: float = 1.0, temporal: bool = True) -> torch.Tensor:
"""Their selection rule: smooth, then ``M = s~ > m + alpha (p - m)`` per video.
``field`` is ``(T, H, W)`` scores (logits or probabilities). Smoothing is a spatial
Gaussian and, with ``temporal``, a ``[1, 2, 1] / 4`` filter across latent frames. With
``floor``, a video whose 99th percentile is under it stays fully coarse."""
smoothed = smooth_detail(field.float(), sigma)
if temporal and smoothed.shape[0] > 1:
padded = torch.cat([smoothed[:1], smoothed, smoothed[-1:]])
smoothed = 0.25 * padded[:-2] + 0.5 * padded[1:-1] + 0.25 * padded[2:]
flat = smoothed.flatten()
median, top = float(flat.median()), float(torch.quantile(flat, 0.99))
if floor is not None and top < floor:
return torch.zeros_like(smoothed, dtype=torch.bool)
return smoothed > median + alpha * (top - median)
def region_layout(mask: torch.Tensor, *, background: int = 4, ring: int = 1) -> LotLayout:
"""Mask -> per-latent-frame LoT layout: 1x1 inside, 2x2 on a ``ring``-token border, ``background`` elsewhere."""
desired = torch.full(mask.shape, int(background), dtype=torch.long)
if ring > 0 and background > 2:
grown = F.max_pool2d(mask.float()[:, None], 2 * ring + 1, stride=1, padding=ring)[:, 0] > 0.5
desired[grown] = 2
desired[mask] = 1
return layout_from_desired_any(desired)
def region_labels(x0_coarse: torch.Tensor, x0_dense: torch.Tensor, fraction: float = 0.25) -> torch.Tensor:
"""Self-supervised label ``(T, H, W)``: the ``fraction`` of tokens per latent frame where the
coarse layout's clean estimate is furthest from the dense model's (a latent-space stand-in
for their per-patch LPIPS label). Latents are ``(1, C, T, 2H, 2W)``."""
error = (x0_coarse.float() - x0_dense.float()).abs().mean(dim=1)[0] # (T, 2H, 2W)
error = F.avg_pool2d(error[:, None], 2)[:, 0] # token grid
flat = error.flatten(1)
cut = torch.quantile(flat, 1.0 - fraction, dim=1, keepdim=True)
return (flat >= cut).reshape(error.shape)
def token_targets(labels: torch.Tensor, layout: LotLayout) -> torch.Tensor:
"""Cell labels ``(T, H, W)`` -> per-token targets ``(L,)``: the labelled share of its cells."""
return torch.stack([labels[r.t, r.u:r.u + r.eh, r.v:r.v + r.ew].float().mean() for r in layout.rects])
# ------------------------------------------------------------------- 3. seam mixer
def extent_maps(layout: LotLayout) -> torch.Tensor:
"""``(2, T, H, W)``: is-1x1 mask and ``log2`` token area, on the token grid."""
fine = torch.zeros(layout.time, layout.height, layout.width)
area = torch.zeros_like(fine)
for rect in layout.rects:
fine[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = float(rect.extent == (1, 1, 1))
area[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = math.log2(rect.eh * rect.ew)
return torch.stack([fine, area])
class SeamMixer(nn.Module):
"""``z = M z_lot + (1 - M)(z_lot + G(z_lot, maps))`` on the latent (their eq. 6, adapted).
``G`` is three 3x3x3 convs; the last is zero-initialised, so the mixer starts as the identity.
``M`` is the 1x1 region upsampled to latent pixels."""
def __init__(self, channels: int = 24, width: int = 64):
super().__init__()
self.net = nn.Sequential(
nn.Conv3d(channels + 2, width, 3, padding=1), nn.SiLU(),
nn.Conv3d(width, width, 3, padding=1), nn.SiLU(),
nn.Conv3d(width, channels, 3, padding=1),
)
nn.init.zeros_(self.net[-1].weight)
nn.init.zeros_(self.net[-1].bias)
def forward(self, latent: torch.Tensor, layout: LotLayout) -> torch.Tensor:
maps = extent_maps(layout).to(latent.device, latent.dtype)
maps = maps.repeat_interleave(2, -2).repeat_interleave(2, -1).unsqueeze(0) # (1, 2, T, 2H, 2W)
residual = self.net(torch.cat([latent, maps], dim=1))
return latent + (1.0 - maps[:, :1]) * residual
def mixer_loss(mixed: torch.Tensor, target: torch.Tensor, layout: LotLayout) -> torch.Tensor:
"""Latent L1 outside the 1x1 region (their latent term; pixel L1 + LPIPS need the VAE decoder)."""
outside = 1.0 - extent_maps(layout)[0].to(mixed.device)
outside = outside.repeat_interleave(2, -2).repeat_interleave(2, -1)[None, None]
return ((mixed - target).abs() * outside).sum() / (outside.sum() * mixed.shape[1]).clamp(min=1.0)
# ------------------------------------------------- 5. two-track decoder (their Sec. 4.2)
class LatentUpscaler(nn.Module):
"""``U``: LR latent ``(B, C, T, h, w)`` -> ``(B, C, T, 2h, 2w)``, bicubic plus a learned residual.
The last conv is zero-initialised, so an untrained upscaler is bicubic."""
def __init__(self, channels: int = 24, width: int = 128):
super().__init__()
self.net = nn.Sequential(
nn.Conv2d(channels, width, 3, padding=1), nn.SiLU(),
nn.Conv2d(width, width, 3, padding=1), nn.SiLU(),
nn.Conv2d(width, 4 * width, 3, padding=1), nn.PixelShuffle(2), nn.SiLU(),
nn.Conv2d(width, width, 3, padding=1), nn.SiLU(),
nn.Conv2d(width, channels, 3, padding=1),
)
nn.init.zeros_(self.net[-1].weight)
nn.init.zeros_(self.net[-1].bias)
def forward(self, latent: torch.Tensor) -> torch.Tensor:
batch, channels, time, height, width = latent.shape
frames = latent.permute(0, 2, 1, 3, 4).reshape(batch * time, channels, height, width).float()
up = F.interpolate(frames, scale_factor=2, mode="bicubic", align_corners=False) + self.net(frames)
return up.reshape(batch, time, channels, 2 * height, 2 * width).permute(0, 2, 1, 3, 4)
class TwoTrackMixer(nn.Module):
"""``z = M z_h + (1 - M)(z_l_up + G(z_h, z_l_up, M))`` (their eq. 6). ``G`` is a small 3D-conv net,
zero-initialised, so an untrained mixer is a hard paste."""
def __init__(self, channels: int = 24, width: int = 64):
super().__init__()
self.net = nn.Sequential(
nn.Conv3d(2 * channels + 1, width, 3, padding=1), nn.SiLU(),
nn.Conv3d(width, width, 3, padding=1), nn.SiLU(),
nn.Conv3d(width, channels, 3, padding=1),
)
nn.init.zeros_(self.net[-1].weight)
nn.init.zeros_(self.net[-1].bias)
def forward(self, hr: torch.Tensor, lr_up: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""``hr``, ``lr_up`` ``(B, C, T, H, W)``; ``mask`` ``(B, 1, T, H, W)`` in [0, 1] on the HR latent grid."""
hr, lr_up, mask = hr.float(), lr_up.float(), mask.float()
delta = self.net(torch.cat([hr * mask, lr_up, mask], dim=1))
return mask * hr + (1.0 - mask) * (lr_up + delta)
__all__ = [
"DualKVPlan", "UpMaps", "DualKV", "dual_attention", "patched_attention", "TwoTrackPlan", "two_track_attention", "RegionHead", "scores_to_cells",
"region_mask", "region_layout", "region_labels", "token_targets", "SeamMixer", "extent_maps",
"mixer_loss", "LatentUpscaler", "TwoTrackMixer",
]