Spaces:
Running on Zero
Running on Zero
Download lot/mixed_res.py from hugging-apps/mpeg-diffusion: direct link, hf CLI and curl.
- Browser
- Download file 26.9 kB
-
https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/lot/mixed_res.py
- Command line
-
hf download hf://spaces/hugging-apps/mpeg-diffusion/lot/mixed_res.py
-
curl -L -o mixed_res.py https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/lot/mixed_res.py
26.9 kB
| """Mixed-resolution additions to LoT, adapted from MPEG-Diffusion (Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu, Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu, 2026). | |
| MPEG-Diffusion: Mixed-Resolution Video Generation, Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu, | |
| Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu (UT Austin, Roblox), arXiv:2611.12069. That paper | |
| keeps two token grids (HR inside a mask, LR elsewhere). LoT has one sequence of | |
| mixed extents. The three pieces here carry its ideas over to that sequence: | |
| 1. ``DualKV`` (their dual-resolution attention, Sec. 3). Their Proposition 1: | |
| with one shared K/V, a coarse token can stand in for ``n`` fine tokens under | |
| softmax only when those fine tokens are identical. So 1x1 queries here attend | |
| to a *dense* key/value view: every coarse token's key and value are expanded | |
| to its ``n`` cells by ``f_up(k) = k + U V^T k`` (their "linear delta": | |
| exact copy plus a rank-``r`` correction, zero at init), before RoPE, and the | |
| expanded keys are rotated at the cells' dense positions. Every other query | |
| (text, audio, coarse tokens) keeps LoT's native attention, which already sees | |
| each fine token individually, so no ``f_down`` is needed. With zero deltas a | |
| fine query sees a coarse token as ``n`` copies at its cells. | |
| 2. ``RegionHead`` (their self-selected HR region, Sec. 3.4): ``s = w^T LN(h)`` | |
| on a middle block's video tokens, smoothed in space and time, thresholded at | |
| ``m + alpha (p - m)`` per video (median ``m``, 99th percentile ``p``), all | |
| coarse when ``p`` is under a floor. The mask becomes a per-latent-frame LoT | |
| layout. Labels here are self-supervised in latent space (``region_labels``): | |
| tokens where the coarse layout's clean estimate is furthest from the dense | |
| model's, a latent stand-in for their per-patch LPIPS label. | |
| 3. ``SeamMixer`` (their mixed-resolution decoder's mixer ``G``, eq. 6): a small | |
| 3D-conv residual on the final latent, applied outside the fine region only, | |
| zero-initialised so it starts as the identity. LoT already returns a dense | |
| latent, so there is no upscaler; the residual targets coarse-token blocking | |
| and the seams between extents. | |
| Nothing here imports model code; ``h3_musubi`` wires it into H3's forward. | |
| """ | |
| from __future__ import annotations | |
| import contextlib | |
| import math | |
| import types | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from layout import LotLayout, smooth_detail | |
| from layout_sources import layout_from_desired_any | |
| # --------------------------------------------------------------------- 2. dual K/V | |
| class DualKVPlan: | |
| """Index tensors for one layout: which video rows are 1x1, and how to assemble the dense view. | |
| The expanded order is: the 1x1 tokens (in layout order), then for each larger extent its | |
| tokens, each followed by its ``eh * ew`` cells in row-major order. ``dense_order`` permutes | |
| that into dense ``(t, u, v)`` order, the order of H3's dense video positions. | |
| """ | |
| def __init__(self, layout: LotLayout): | |
| self.layout = layout | |
| fine, groups = [], [] | |
| cells = [] | |
| for index, rect in enumerate(layout.rects): | |
| if rect.extent == (1, 1, 1): | |
| fine.append(index) | |
| width, plane = layout.width, layout.height * layout.width | |
| for index in fine: | |
| rect = layout.rects[index] | |
| cells.append(rect.t * plane + rect.u * width + rect.v) | |
| for extent, indices in layout.groups(): | |
| if extent == (1, 1, 1): | |
| continue | |
| groups.append((extent, torch.tensor(indices, dtype=torch.long))) | |
| for index in indices: | |
| rect = layout.rects[index] | |
| for du in range(rect.eh): | |
| for dv in range(rect.ew): | |
| cells.append(rect.t * plane + (rect.u + du) * width + rect.v + dv) | |
| cells = torch.tensor(cells, dtype=torch.long) | |
| if cells.numel() != layout.dense_count or torch.unique(cells).numel() != layout.dense_count: | |
| raise ValueError("layout does not cover the dense grid exactly once") | |
| self.fine_rows = torch.tensor(fine, dtype=torch.long) | |
| fine_mask = torch.zeros(layout.count, dtype=torch.bool) | |
| fine_mask[self.fine_rows] = True | |
| self.other_rows = torch.nonzero(~fine_mask).squeeze(1) | |
| self.groups = groups | |
| self.dense_order = torch.argsort(cells) | |
| def to(self, device) -> "DualKVPlan": | |
| self.fine_rows = self.fine_rows.to(device) | |
| self.other_rows = self.other_rows.to(device) | |
| self.dense_order = self.dense_order.to(device) | |
| self.groups = [(extent, rows.to(device)) for extent, rows in self.groups] | |
| return self | |
| def _key(layer: int, extent: tuple[int, int, int], which: str) -> str: | |
| return f"l{layer}_{extent[1]}x{extent[2]}_{which}" | |
| class UpMaps(nn.Module): | |
| """``f_up`` per block, extent, and K or V: ``k -> k + (k V) U`` reshaped to ``n`` cells. | |
| ``V`` is ``(d, r)``, ``U`` is ``(r, n * d)``, shared by all heads of a block (the paper fits | |
| one map per layer). ``U`` starts at zero, so the map starts as an exact copy. | |
| """ | |
| def __init__(self, layers: int, head_dim: int, extents, rank: int = 16): | |
| super().__init__() | |
| self.head_dim, self.rank = int(head_dim), int(rank) | |
| self.extents = [tuple(map(int, e)) for e in extents if tuple(e) != (1, 1, 1)] | |
| self.down = nn.ParameterDict() | |
| self.up = nn.ParameterDict() | |
| for layer in range(layers): | |
| for extent in self.extents: | |
| cells = extent[1] * extent[2] | |
| for which in ("k", "v"): | |
| name = _key(layer, extent, which) | |
| down = torch.empty(self.head_dim, self.rank) | |
| nn.init.normal_(down, std=1.0 / math.sqrt(self.head_dim)) | |
| self.down[name] = nn.Parameter(down) | |
| self.up[name] = nn.Parameter(torch.zeros(self.rank, cells * self.head_dim)) | |
| def expand(self, layer: int, extent: tuple[int, int, int], which: str, rows: torch.Tensor) -> torch.Tensor: | |
| """``(B, N, H, d)`` -> ``(B, N * n, H, d)``: each token, then its cells row-major.""" | |
| name = _key(layer, extent, which) | |
| cells = extent[1] * extent[2] | |
| batch, count, heads, dim = rows.shape | |
| delta = (rows @ self.down[name].to(rows.dtype)) @ self.up[name].to(rows.dtype) # (B, N, H, n*d) | |
| delta = delta.reshape(batch, count, heads, cells, dim).transpose(2, 3) # (B, N, n, H, d) | |
| expanded = rows.unsqueeze(2) + delta | |
| return expanded.reshape(batch, count * cells, heads, dim) | |
| def fit_(self, layer: int, extent: tuple[int, int, int], which: str, | |
| coarse: torch.Tensor, cells: torch.Tensor, ridge: float = 1e-2) -> float: | |
| """Ridge fit of the delta (their init), truncated to rank ``r`` by SVD. | |
| ``coarse`` is ``(N, d)`` keys (or values) of coarse tokens, ``cells`` is ``(N, n, d)`` the | |
| dense model's keys at the cells they cover. Solves ``min |coarse W - (cells - coarse)|^2 + | |
| ridge |W|^2`` for ``W`` ``(d, n*d)``, keeps the top ``r`` singular directions. Returns | |
| the relative residual of the copy-plus-delta fit.""" | |
| name = _key(layer, extent, which) | |
| x = coarse.float() | |
| target = (cells.float() - x.unsqueeze(1)).reshape(x.shape[0], -1) | |
| self.fit_moments_(layer, extent, which, x.T @ x, x.T @ target, x.shape[0], ridge) | |
| fitted = x.unsqueeze(1) + ((x @ self.down[name].float()) @ self.up[name].float()).reshape_as(cells) | |
| return float((fitted - cells.float()).norm() / cells.float().norm().clamp(min=1e-8)) | |
| def fit_moments_(self, layer: int, extent: tuple[int, int, int], which: str, | |
| xtx: torch.Tensor, xty: torch.Tensor, count: int, ridge: float = 1e-2) -> None: | |
| """``fit_`` from accumulated ``X^T X`` ``(d, d)`` and ``X^T (cells - X)`` ``(d, n*d)`` over ``count`` rows.""" | |
| name = _key(layer, extent, which) | |
| gram = xtx.float() + ridge * count * torch.eye(xtx.shape[0], device=xtx.device) | |
| weight = torch.linalg.solve(gram, xty.float()) # (d, n*d) | |
| u, s, vh = torch.linalg.svd(weight, full_matrices=False) | |
| r = self.rank | |
| self.down[name].copy_((u[:, :r] * s[:r].sqrt()).to(self.down[name].dtype)) | |
| self.up[name].copy_((s[:r].sqrt().unsqueeze(1) * vh[:r]).to(self.up[name].dtype)) | |
| class DualKV: | |
| """``UpMaps`` plus a per-layout plan cache; what ``h3_musubi.predict(..., dual=)`` takes.""" | |
| def __init__(self, maps: UpMaps): | |
| self.maps = maps | |
| self._plans: dict = {} | |
| def plan(self, layout: LotLayout, device) -> DualKVPlan: | |
| key = (layout, str(device)) | |
| if key not in self._plans: | |
| if len(self._plans) > 8: | |
| self._plans.clear() | |
| self._plans[key] = DualKVPlan(layout).to(device) | |
| return self._plans[key] | |
| def _rope(states: torch.Tensor, table: torch.Tensor) -> torch.Tensor: | |
| """H3's split-half RoPE (``model._apply_rope_split_half``), repeated so this file needs no model import.""" | |
| pairs = table.shape[-3] | |
| a, b = states[..., :pairs], states[..., pairs:2 * pairs] | |
| first = table[..., 0, 0] * a + table[..., 0, 1] * b # element-wise: no broadcast matmul over heads | |
| second = table[..., 1, 0] * a + table[..., 1, 1] * b | |
| return torch.cat((first, second, states[..., 2 * pairs:]), dim=-1) | |
| def _sdpa(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor) -> torch.Tensor: | |
| """``(B, L, H, d)`` in, ``(B, L, H * d)`` out.""" | |
| out = F.scaled_dot_product_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2)) | |
| return out.transpose(1, 2).reshape(query.shape[0], query.shape[1], -1) | |
| def dual_attention(attn, hidden: torch.Tensor, rotation: torch.Tensor, *, layer: int, plan: DualKVPlan, | |
| maps: UpMaps | None, video_start: int, dense_rotation: torch.Tensor, | |
| capture: dict | None = None) -> torch.Tensor: | |
| """H3 ``Attention.forward`` with the dense K/V view for 1x1 queries (see module docstring). | |
| ``hidden`` is the packed ``(1, S, D)`` sequence whose tail from ``video_start`` is the LoT video; | |
| ``rotation`` its rotary table; ``dense_rotation`` the table of the dense video positions. | |
| ``capture`` (optional) receives this layer's pre-RoPE video keys/values for ``UpMaps.fit_``. | |
| ``maps=None`` is H3's own attention (capture only). | |
| """ | |
| batch, length, _ = hidden.shape | |
| query, key, value = attn.qkv_proj(hidden).split(attn.inner_dim, dim=-1) | |
| query = attn.q_norm(query.reshape(batch, length, attn.num_heads, attn.head_dim)) | |
| key = attn.k_norm(key.reshape(batch, length, attn.num_heads, attn.head_dim)) | |
| value = value.reshape(batch, length, attn.num_heads, attn.head_dim).contiguous() | |
| if capture is not None: | |
| capture[layer] = (key[:, video_start:].detach(), value[:, video_start:].detach()) | |
| query_r = _rope(query, rotation).to(value) | |
| key_r = _rope(key, rotation).to(value) | |
| if maps is None: | |
| return attn.out_proj(_sdpa(query_r, key_r, value)) | |
| # Native LoT attention for text, audio and coarse queries. | |
| other = torch.cat([torch.arange(video_start, device=hidden.device), video_start + plan.other_rows]) | |
| output = hidden.new_zeros(batch, length, attn.inner_dim, dtype=value.dtype) | |
| output[:, other] = _sdpa(query_r[:, other], key_r, value) | |
| if plan.fine_rows.numel(): | |
| # Dense view for 1x1 queries: fine keys as they are, coarse keys expanded to their cells. | |
| video_key, video_value = key[:, video_start:], value[:, video_start:] | |
| keys = [video_key[:, plan.fine_rows]] | |
| values = [video_value[:, plan.fine_rows]] | |
| for extent, rows in plan.groups: | |
| keys.append(maps.expand(layer, extent, "k", video_key[:, rows])) | |
| values.append(maps.expand(layer, extent, "v", video_value[:, rows])) | |
| dense_key = torch.cat(keys, dim=1)[:, plan.dense_order] | |
| dense_value = torch.cat(values, dim=1)[:, plan.dense_order] | |
| dense_key = _rope(dense_key, dense_rotation).to(value) | |
| fine = video_start + plan.fine_rows | |
| output[:, fine] = _sdpa(query_r[:, fine], | |
| torch.cat([key_r[:, :video_start], dense_key], dim=1), | |
| torch.cat([value[:, :video_start], dense_value], dim=1)) | |
| return attn.out_proj(output) | |
| def patched_attention(blocks, attention=None, **context): | |
| """Route every block's ``attn.forward`` through ``dual_attention`` for one forward pass. | |
| musubi-tuner's code is unchanged: the instance attribute shadows the class method and is | |
| removed on exit, so checkpointing and block swap see the same modules.""" | |
| patched = [] | |
| try: | |
| for layer, block in enumerate(blocks): | |
| attn = block.attn | |
| def forward(self, hidden, rotation=None, _layer=layer): | |
| return (attention or dual_attention)(self, hidden, rotation, layer=_layer, **context) | |
| attn.forward = types.MethodType(forward, attn) | |
| patched.append(attn) | |
| yield | |
| finally: | |
| for attn in patched: | |
| del attn.forward | |
| # ------------------------------------------------------------- 4. two native tracks | |
| class TwoTrackPlan: | |
| """Index tensors for the paper's own arrangement: a complete native LR grid plus HR tokens on a mask. | |
| ``mask`` is ``(T, h, w)`` bool on the LR token grid; every masked LR token has ``scale * scale`` | |
| HR tokens. ``hr_index`` are their rows in the HR grid's raster order (the order of the HR | |
| track's rows), and ``cell_order`` permutes "LR token, then its cells row-major" into HR raster. | |
| """ | |
| def __init__(self, mask: torch.Tensor, scale: int = 2): | |
| mask = mask.bool().cpu() | |
| time, height, width = mask.shape | |
| self.mask, self.scale = mask, int(scale) | |
| self.lr_count = time * height * width | |
| self.hr_shape = (time, height * scale, width * scale) | |
| up = mask.repeat_interleave(scale, 1).repeat_interleave(scale, 2) | |
| self.hr_index = torch.nonzero(up.flatten()).squeeze(1) | |
| t, u, v = torch.meshgrid(torch.arange(time), torch.arange(height), torch.arange(width), indexing="ij") | |
| du, dv = torch.meshgrid(torch.arange(scale), torch.arange(scale), indexing="ij") | |
| cells = (t.reshape(-1, 1) * (height * scale) + u.reshape(-1, 1) * scale + du.reshape(1, -1)) * (width * scale) \ | |
| + v.reshape(-1, 1) * scale + dv.reshape(1, -1) # (lr, scale^2) | |
| self.cell_order = torch.argsort(cells.flatten()) | |
| self.extent = (1, self.scale, self.scale) | |
| def hr_count(self) -> int: | |
| return int(self.hr_index.numel()) | |
| def grid_source(self, device) -> torch.Tensor: | |
| """For each HR-grid cell (raster order), its row in ``cat([LR tokens, HR tokens])``: the LR token that | |
| covers it, or its own HR token inside the mask. One gather builds the whole grid.""" | |
| cached = getattr(self, "_grid_source", None) | |
| if cached is None or cached.device != torch.device(device): | |
| source = torch.div(self.cell_order, self.scale * self.scale, rounding_mode="floor") | |
| source[self.hr_index] = self.lr_count + torch.arange(self.hr_count, device=source.device) | |
| self._grid_source = cached = source.to(device) | |
| return cached | |
| def to(self, device) -> "TwoTrackPlan": | |
| self.hr_index = self.hr_index.to(device) | |
| self.cell_order = self.cell_order.to(device) | |
| return self | |
| def two_track_attention(attn, hidden: torch.Tensor, rotation: torch.Tensor, *, layer: int, plan: TwoTrackPlan, | |
| maps: UpMaps | None, video_start: int, hr_rotation: torch.Tensor, | |
| capture: dict | None = None) -> torch.Tensor: | |
| """Dual-resolution attention (their eq. 3) on ``[text, audio, LR video (whole grid), HR video (mask)]``. | |
| Text, audio and LR queries see only text, audio and LR keys: that part is the unmodified model | |
| at LR, so it needs no ``f_down``. HR queries see text, audio and a full HR grid of keys/values: | |
| their own inside the mask, ``f_up`` of the LR token's outside it (``maps=None`` copies), rotated | |
| at the HR grid's positions (``hr_rotation``, the table of the whole HR grid). | |
| """ | |
| batch, length, _ = hidden.shape | |
| query, key, value = attn.qkv_proj(hidden).split(attn.inner_dim, dim=-1) | |
| query = attn.q_norm(query.reshape(batch, length, attn.num_heads, attn.head_dim)) | |
| key = attn.k_norm(key.reshape(batch, length, attn.num_heads, attn.head_dim)) | |
| value = value.reshape(batch, length, attn.num_heads, attn.head_dim).contiguous() | |
| split = video_start + plan.lr_count | |
| if capture is not None: | |
| capture[layer] = (key[:, video_start:split].detach(), value[:, video_start:split].detach()) | |
| query_r = _rope(query, rotation).to(value) | |
| key_r = _rope(key[:, :split], rotation[:, :split]).to(value) # HR rows' own rotated keys are never read | |
| output = hidden.new_zeros(batch, length, attn.inner_dim, dtype=value.dtype) | |
| output[:, :split] = _sdpa(query_r[:, :split], key_r, value[:, :split]) | |
| if plan.hr_count: | |
| lr_key, lr_value = key[:, video_start:split], value[:, video_start:split] | |
| if maps is None: | |
| # One gather builds the HR grid: each cell takes its LR token's key (copy) or its own HR key. | |
| source = plan.grid_source(lr_key.device) | |
| grid_key = torch.cat([lr_key, key[:, split:]], dim=1)[:, source] | |
| grid_value = torch.cat([lr_value, value[:, split:]], dim=1)[:, source] | |
| else: | |
| grid_key = maps.expand(layer, plan.extent, "k", lr_key).to(key.dtype) | |
| grid_value = maps.expand(layer, plan.extent, "v", lr_value).to(value.dtype) | |
| grid_key = grid_key[:, plan.cell_order].index_copy(1, plan.hr_index, key[:, split:]) | |
| grid_value = grid_value[:, plan.cell_order].index_copy(1, plan.hr_index, value[:, split:]) | |
| grid_key = _rope(grid_key, hr_rotation).to(value) | |
| output[:, split:] = _sdpa(query_r[:, split:], | |
| torch.cat([key_r[:, :video_start], grid_key], dim=1), | |
| torch.cat([value[:, :video_start], grid_value], dim=1)) | |
| return attn.out_proj(output) | |
| # ------------------------------------------------------------------ 1. region head | |
| class RegionHead(nn.Module): | |
| """``s = w^T LN(h)`` per video token, on a middle block's states (their eq. 7).""" | |
| def __init__(self, hidden_size: int): | |
| super().__init__() | |
| self.norm = nn.LayerNorm(hidden_size, elementwise_affine=False) | |
| self.score = nn.Linear(hidden_size, 1) | |
| nn.init.zeros_(self.score.weight) | |
| nn.init.zeros_(self.score.bias) | |
| def forward(self, states: torch.Tensor) -> torch.Tensor: | |
| """``(..., L, D)`` -> ``(..., L)`` logits.""" | |
| return self.score(self.norm(states.float())).squeeze(-1) | |
| def scores_to_cells(scores: torch.Tensor, layout: LotLayout) -> torch.Tensor: | |
| """Per-token scores ``(L,)`` -> ``(T, H, W)``: every cell takes its token's score.""" | |
| field = scores.new_zeros(layout.time, layout.height, layout.width) | |
| for value, rect in zip(scores, layout.rects): | |
| field[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = value | |
| return field | |
| def region_mask(field: torch.Tensor, *, alpha: float = 0.5, floor: float | None = None, | |
| sigma: float = 1.0, temporal: bool = True) -> torch.Tensor: | |
| """Their selection rule: smooth, then ``M = s~ > m + alpha (p - m)`` per video. | |
| ``field`` is ``(T, H, W)`` scores (logits or probabilities). Smoothing is a spatial | |
| Gaussian and, with ``temporal``, a ``[1, 2, 1] / 4`` filter across latent frames. With | |
| ``floor``, a video whose 99th percentile is under it stays fully coarse.""" | |
| smoothed = smooth_detail(field.float(), sigma) | |
| if temporal and smoothed.shape[0] > 1: | |
| padded = torch.cat([smoothed[:1], smoothed, smoothed[-1:]]) | |
| smoothed = 0.25 * padded[:-2] + 0.5 * padded[1:-1] + 0.25 * padded[2:] | |
| flat = smoothed.flatten() | |
| median, top = float(flat.median()), float(torch.quantile(flat, 0.99)) | |
| if floor is not None and top < floor: | |
| return torch.zeros_like(smoothed, dtype=torch.bool) | |
| return smoothed > median + alpha * (top - median) | |
| def region_layout(mask: torch.Tensor, *, background: int = 4, ring: int = 1) -> LotLayout: | |
| """Mask -> per-latent-frame LoT layout: 1x1 inside, 2x2 on a ``ring``-token border, ``background`` elsewhere.""" | |
| desired = torch.full(mask.shape, int(background), dtype=torch.long) | |
| if ring > 0 and background > 2: | |
| grown = F.max_pool2d(mask.float()[:, None], 2 * ring + 1, stride=1, padding=ring)[:, 0] > 0.5 | |
| desired[grown] = 2 | |
| desired[mask] = 1 | |
| return layout_from_desired_any(desired) | |
| def region_labels(x0_coarse: torch.Tensor, x0_dense: torch.Tensor, fraction: float = 0.25) -> torch.Tensor: | |
| """Self-supervised label ``(T, H, W)``: the ``fraction`` of tokens per latent frame where the | |
| coarse layout's clean estimate is furthest from the dense model's (a latent-space stand-in | |
| for their per-patch LPIPS label). Latents are ``(1, C, T, 2H, 2W)``.""" | |
| error = (x0_coarse.float() - x0_dense.float()).abs().mean(dim=1)[0] # (T, 2H, 2W) | |
| error = F.avg_pool2d(error[:, None], 2)[:, 0] # token grid | |
| flat = error.flatten(1) | |
| cut = torch.quantile(flat, 1.0 - fraction, dim=1, keepdim=True) | |
| return (flat >= cut).reshape(error.shape) | |
| def token_targets(labels: torch.Tensor, layout: LotLayout) -> torch.Tensor: | |
| """Cell labels ``(T, H, W)`` -> per-token targets ``(L,)``: the labelled share of its cells.""" | |
| return torch.stack([labels[r.t, r.u:r.u + r.eh, r.v:r.v + r.ew].float().mean() for r in layout.rects]) | |
| # ------------------------------------------------------------------- 3. seam mixer | |
| def extent_maps(layout: LotLayout) -> torch.Tensor: | |
| """``(2, T, H, W)``: is-1x1 mask and ``log2`` token area, on the token grid.""" | |
| fine = torch.zeros(layout.time, layout.height, layout.width) | |
| area = torch.zeros_like(fine) | |
| for rect in layout.rects: | |
| fine[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = float(rect.extent == (1, 1, 1)) | |
| area[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = math.log2(rect.eh * rect.ew) | |
| return torch.stack([fine, area]) | |
| class SeamMixer(nn.Module): | |
| """``z = M z_lot + (1 - M)(z_lot + G(z_lot, maps))`` on the latent (their eq. 6, adapted). | |
| ``G`` is three 3x3x3 convs; the last is zero-initialised, so the mixer starts as the identity. | |
| ``M`` is the 1x1 region upsampled to latent pixels.""" | |
| def __init__(self, channels: int = 24, width: int = 64): | |
| super().__init__() | |
| self.net = nn.Sequential( | |
| nn.Conv3d(channels + 2, width, 3, padding=1), nn.SiLU(), | |
| nn.Conv3d(width, width, 3, padding=1), nn.SiLU(), | |
| nn.Conv3d(width, channels, 3, padding=1), | |
| ) | |
| nn.init.zeros_(self.net[-1].weight) | |
| nn.init.zeros_(self.net[-1].bias) | |
| def forward(self, latent: torch.Tensor, layout: LotLayout) -> torch.Tensor: | |
| maps = extent_maps(layout).to(latent.device, latent.dtype) | |
| maps = maps.repeat_interleave(2, -2).repeat_interleave(2, -1).unsqueeze(0) # (1, 2, T, 2H, 2W) | |
| residual = self.net(torch.cat([latent, maps], dim=1)) | |
| return latent + (1.0 - maps[:, :1]) * residual | |
| def mixer_loss(mixed: torch.Tensor, target: torch.Tensor, layout: LotLayout) -> torch.Tensor: | |
| """Latent L1 outside the 1x1 region (their latent term; pixel L1 + LPIPS need the VAE decoder).""" | |
| outside = 1.0 - extent_maps(layout)[0].to(mixed.device) | |
| outside = outside.repeat_interleave(2, -2).repeat_interleave(2, -1)[None, None] | |
| return ((mixed - target).abs() * outside).sum() / (outside.sum() * mixed.shape[1]).clamp(min=1.0) | |
| # ------------------------------------------------- 5. two-track decoder (their Sec. 4.2) | |
| class LatentUpscaler(nn.Module): | |
| """``U``: LR latent ``(B, C, T, h, w)`` -> ``(B, C, T, 2h, 2w)``, bicubic plus a learned residual. | |
| The last conv is zero-initialised, so an untrained upscaler is bicubic.""" | |
| def __init__(self, channels: int = 24, width: int = 128): | |
| super().__init__() | |
| self.net = nn.Sequential( | |
| nn.Conv2d(channels, width, 3, padding=1), nn.SiLU(), | |
| nn.Conv2d(width, width, 3, padding=1), nn.SiLU(), | |
| nn.Conv2d(width, 4 * width, 3, padding=1), nn.PixelShuffle(2), nn.SiLU(), | |
| nn.Conv2d(width, width, 3, padding=1), nn.SiLU(), | |
| nn.Conv2d(width, channels, 3, padding=1), | |
| ) | |
| nn.init.zeros_(self.net[-1].weight) | |
| nn.init.zeros_(self.net[-1].bias) | |
| def forward(self, latent: torch.Tensor) -> torch.Tensor: | |
| batch, channels, time, height, width = latent.shape | |
| frames = latent.permute(0, 2, 1, 3, 4).reshape(batch * time, channels, height, width).float() | |
| up = F.interpolate(frames, scale_factor=2, mode="bicubic", align_corners=False) + self.net(frames) | |
| return up.reshape(batch, time, channels, 2 * height, 2 * width).permute(0, 2, 1, 3, 4) | |
| class TwoTrackMixer(nn.Module): | |
| """``z = M z_h + (1 - M)(z_l_up + G(z_h, z_l_up, M))`` (their eq. 6). ``G`` is a small 3D-conv net, | |
| zero-initialised, so an untrained mixer is a hard paste.""" | |
| def __init__(self, channels: int = 24, width: int = 64): | |
| super().__init__() | |
| self.net = nn.Sequential( | |
| nn.Conv3d(2 * channels + 1, width, 3, padding=1), nn.SiLU(), | |
| nn.Conv3d(width, width, 3, padding=1), nn.SiLU(), | |
| nn.Conv3d(width, channels, 3, padding=1), | |
| ) | |
| nn.init.zeros_(self.net[-1].weight) | |
| nn.init.zeros_(self.net[-1].bias) | |
| def forward(self, hr: torch.Tensor, lr_up: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: | |
| """``hr``, ``lr_up`` ``(B, C, T, H, W)``; ``mask`` ``(B, 1, T, H, W)`` in [0, 1] on the HR latent grid.""" | |
| hr, lr_up, mask = hr.float(), lr_up.float(), mask.float() | |
| delta = self.net(torch.cat([hr * mask, lr_up, mask], dim=1)) | |
| return mask * hr + (1.0 - mask) * (lr_up + delta) | |
| __all__ = [ | |
| "DualKVPlan", "UpMaps", "DualKV", "dual_attention", "patched_attention", "TwoTrackPlan", "two_track_attention", "RegionHead", "scores_to_cells", | |
| "region_mask", "region_layout", "region_labels", "token_targets", "SeamMixer", "extent_maps", | |
| "mixer_loss", "LatentUpscaler", "TwoTrackMixer", | |
| ] | |