Spaces:
Running on Zero
Running on Zero
File size: 13,104 Bytes
6febba8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 | """Level-of-Token DiT adapter.
The backbone still runs attention. This module replaces only the visual
token pack, the positional centers, and the velocity head:
tokens (B, T, H, W, D) in scaled y-space
-> per-extent input head + shape MLP (eqs. 11–12)
-> backbone(hidden, centers, sigma) -> hidden
-> per-extent output head (eq. 10)
-> LakonLab asymmetric-velocity recovery (eq. 9)
-> dense velocity on the uniform lattice
Text, audio, and the transformer's own blocks stay inside ``backbone``.
It must return one hidden vector per LoT token, in ``layout.rects`` order.
"""
from __future__ import annotations
import torch
from torch import nn
from flow import (
apply_extent_scales,
asymmetric_target,
fit_procrustes,
gather_extent,
mean_basis,
recover_dense_velocity,
scatter_extent,
)
from layout import LotLayout, shape_features
def _key(extent: tuple[int, int, int]) -> str:
et, eh, ew = extent
return f"{et}x{eh}x{ew}"
class ExtentBank(nn.Module):
"""Fixed per-extent lifts ``A`` and data scales ``s``.
Bases start as the mean lift and stay frozen. ``fit_`` replaces one
extent with a Procrustes fit; training does not update these buffers.
"""
def __init__(self, token_dim: int, extents: list[tuple[int, int, int]]):
super().__init__()
if not extents:
raise ValueError("at least one extent is required")
self.token_dim = int(token_dim)
self.extents = [tuple(map(int, extent)) for extent in extents]
seen = set()
for extent in self.extents:
if extent in seen:
raise ValueError(f"duplicate extent {extent}")
seen.add(extent)
if min(extent) < 1:
raise ValueError(f"bad extent {extent}")
key = _key(extent)
self.register_buffer(f"A_{key}", mean_basis(self.token_dim, extent))
self.register_buffer(f"s_{key}", torch.ones(()))
def basis(self, extent: tuple[int, int, int]) -> torch.Tensor:
return getattr(self, f"A_{_key(extent)}")
def scale(self, extent: tuple[int, int, int]) -> torch.Tensor:
return getattr(self, f"s_{_key(extent)}")
def scales(self) -> dict[tuple[int, int, int], torch.Tensor]:
return {extent: self.scale(extent) for extent in self.extents}
def dense_dim(self, extent: tuple[int, int, int]) -> int:
et, eh, ew = extent
return self.token_dim * et * eh * ew
@torch.no_grad()
def fit_(self, extent: tuple[int, int, int], dense: torch.Tensor, reference: torch.Tensor) -> None:
"""Replace ``A`` and ``s`` for one extent. Inputs are detached."""
if tuple(extent) not in self.extents:
raise KeyError(f"extent {extent} is not in this bank")
if dense.shape[-1] != self.dense_dim(extent) or reference.shape[-1] != self.token_dim:
raise ValueError(
f"expected dense (N, {self.dense_dim(extent)}) and reference (N, {self.token_dim})"
)
basis, scale = fit_procrustes(dense, reference)
slot = self.basis(extent)
slot.copy_(basis.to(device=slot.device, dtype=slot.dtype))
scale_slot = self.scale(extent)
scale_slot.copy_(scale.to(device=scale_slot.device, dtype=scale_slot.dtype))
def scale_clean(self, x0: torch.Tensor, layout: LotLayout) -> torch.Tensor:
"""Eq. 16. ``x0`` is ``(B, T, H, W, D)`` in the pretrained latent space."""
self._check_layout(layout)
return apply_extent_scales(x0, layout, self.scales(), invert=False)
def unscale(self, y0: torch.Tensor, layout: LotLayout) -> torch.Tensor:
"""Map a y-space clean estimate back to the pretrained latent."""
self._check_layout(layout)
return apply_extent_scales(y0, layout, self.scales(), invert=True)
def _check_layout(self, layout: LotLayout) -> None:
missing = sorted({rect.extent for rect in layout.rects} - set(self.extents))
if missing:
raise KeyError(f"layout uses extents with no basis: {missing}")
class LotVisualAdapter(nn.Module):
"""Extent heads and shape MLP wrapped around an external backbone."""
def __init__(
self,
token_dim: int,
hidden_size: int,
extents: list[tuple[int, int, int]],
*,
include_time: bool = False,
shape_hidden: int = 256,
):
super().__init__()
self.token_dim = int(token_dim)
self.hidden_size = int(hidden_size)
self.include_time = bool(include_time)
self.bank = ExtentBank(self.token_dim, extents)
feat_dim = 5 if self.include_time else 4
width = max(feat_dim, int(shape_hidden))
self.shape_mlp = nn.Sequential(
nn.Linear(feat_dim, width),
nn.SiLU(),
nn.Linear(width, self.hidden_size),
)
nn.init.zeros_(self.shape_mlp[-1].weight)
nn.init.zeros_(self.shape_mlp[-1].bias)
self.in_proj = nn.ModuleDict()
self.out_proj = nn.ModuleDict()
for extent in self.bank.extents:
dense_dim = self.bank.dense_dim(extent)
self.in_proj[_key(extent)] = nn.Linear(dense_dim, self.hidden_size)
self.out_proj[_key(extent)] = nn.Linear(self.hidden_size, dense_dim)
self.register_buffer("pretrained_in", torch.empty(self.hidden_size, self.token_dim))
self.register_buffer("pretrained_out", torch.empty(self.token_dim, self.hidden_size))
self.register_buffer("pretrained_bias_in", torch.empty(self.hidden_size))
self.register_buffer("pretrained_bias_out", torch.empty(self.token_dim))
self.register_buffer("pretrained_ready", torch.zeros((), dtype=torch.int32))
@torch.no_grad()
def init_from_pretrained(
self,
weight_in: torch.Tensor,
weight_out: torch.Tensor,
bias_in: torch.Tensor | None = None,
bias_out: torch.Tensor | None = None,
) -> None:
"""Initialize heads as ``W_in,e = W_in A^T`` and ``W_e = A W_out``.
``weight_in`` is ``(hidden, D)`` and ``weight_out`` is ``(D, hidden)``,
matching ``nn.Linear`` layout. Output bias is lifted by ``A`` so a
unit extent reproduces the pretrained bias and a larger extent starts
on the same subspace.
"""
hidden, token_dim = self.hidden_size, self.token_dim
if tuple(weight_in.shape) != (hidden, token_dim):
raise ValueError(f"weight_in shape {tuple(weight_in.shape)} != {(hidden, token_dim)}")
if tuple(weight_out.shape) != (token_dim, hidden):
raise ValueError(f"weight_out shape {tuple(weight_out.shape)} != {(token_dim, hidden)}")
self.pretrained_in.copy_(weight_in.detach().to(dtype=self.pretrained_in.dtype, device=self.pretrained_in.device).clone())
self.pretrained_out.copy_(weight_out.detach().to(dtype=self.pretrained_out.dtype, device=self.pretrained_out.device).clone())
if bias_in is None:
self.pretrained_bias_in.zero_()
else:
self.pretrained_bias_in.copy_(bias_in.detach().to(dtype=self.pretrained_bias_in.dtype, device=self.pretrained_bias_in.device).clone())
if bias_out is None:
self.pretrained_bias_out.zero_()
else:
self.pretrained_bias_out.copy_(bias_out.detach().to(dtype=self.pretrained_bias_out.dtype, device=self.pretrained_bias_out.device).clone())
self.pretrained_ready.fill_(1)
for extent in self.bank.extents:
key = _key(extent)
in_layer = self.in_proj[key]
out_layer = self.out_proj[key]
basis = self.bank.basis(extent).to(dtype=in_layer.weight.dtype, device=in_layer.weight.device)
weight_in_m = weight_in.to(dtype=in_layer.weight.dtype, device=in_layer.weight.device)
weight_out_m = weight_out.to(dtype=out_layer.weight.dtype, device=out_layer.weight.device)
in_layer.weight.copy_(weight_in_m @ basis.T)
out_layer.weight.copy_(basis @ weight_out_m)
if bias_in is None:
in_layer.bias.zero_()
else:
in_layer.bias.copy_(bias_in.to(dtype=in_layer.bias.dtype, device=in_layer.bias.device))
if bias_out is None:
out_layer.bias.zero_()
else:
lifted = basis @ bias_out.to(dtype=basis.dtype, device=basis.device)
out_layer.bias.copy_(lifted.to(dtype=out_layer.bias.dtype))
@torch.no_grad()
def fit_extent(self, extent: tuple[int, int, int], dense: torch.Tensor, reference: torch.Tensor) -> None:
"""Fit one basis, then rebuild every head from the stored pretrained maps.
``fit_`` on the bank alone updates ``A`` and leaves the heads on the
previous subspace. Call this instead once ``init_from_pretrained`` has run.
"""
self.bank.fit_(extent, dense, reference)
if int(self.pretrained_ready) != 1:
return
self.init_from_pretrained(
self.pretrained_in,
self.pretrained_out,
self.pretrained_bias_in,
self.pretrained_bias_out,
)
def embed(self, tokens: torch.Tensor, layout: LotLayout) -> tuple[torch.Tensor, torch.Tensor]:
"""Return ``(B, L, hidden)`` and ``(L, 3)`` finest-grid centers."""
self._check_tokens(tokens, layout)
hidden = tokens.new_zeros(tokens.shape[0], layout.count, self.hidden_size)
for extent, indices in layout.groups():
rects = [layout.rects[index] for index in indices]
dense = gather_extent(tokens, rects)
projected = self.in_proj[_key(extent)](dense)
features = shape_features(extent, self.include_time).to(device=tokens.device, dtype=tokens.dtype)
projected = projected + self.shape_mlp(features)
hidden[:, indices] = projected
centers = layout.centers().to(device=tokens.device)
return hidden, centers
def velocity_from_states(
self,
states: torch.Tensor,
tokens: torch.Tensor,
sigma: torch.Tensor | float,
layout: LotLayout,
*,
x0_minus_eps: bool = False,
) -> torch.Tensor:
"""Extent heads plus eq. 9, scattered onto the uniform lattice.
Eq. 9 assumes the head predicts ``P eps - x0`` and returns ``eps - x0``.
H3's head predicts the negation (``x0 - eps``; H3's scheduler steps ``x0 = x_t + sigma * v``).
``x0_minus_eps=True`` flips the head before recovery and the velocity
after, so the result stays in the caller's convention. A unit extent has
no complement, so 1x1 parity cannot see a wrong sign here.
"""
sign = -1.0 if x0_minus_eps else 1.0
self._check_tokens(tokens, layout)
if states.shape != (tokens.shape[0], layout.count, self.hidden_size):
raise ValueError(
f"backbone returned {tuple(states.shape)}, expected "
f"{(tokens.shape[0], layout.count, self.hidden_size)}"
)
velocity = tokens.new_zeros(tokens.shape)
for extent, indices in layout.groups():
rects = [layout.rects[index] for index in indices]
u_a = sign * self.out_proj[_key(extent)](states[:, indices])
x_t = gather_extent(tokens, rects)
recovered = recover_dense_velocity(u_a, x_t, self.bank.basis(extent), sigma)
scatter_extent(velocity, sign * recovered, rects)
return velocity
def forward(self, tokens, sigma, layout: LotLayout, backbone, backbone_kwargs=None):
"""Dense y-space velocity. ``backbone(hidden, centers, sigma, **kwargs)``."""
hidden, centers = self.embed(tokens, layout)
kwargs = {} if backbone_kwargs is None else backbone_kwargs
states = backbone(hidden, centers, sigma, **kwargs)
return self.velocity_from_states(states, tokens, sigma, layout)
def asymmetric_targets(
self,
y0: torch.Tensor,
noise: torch.Tensor,
layout: LotLayout,
) -> list[tuple[list[int], torch.Tensor]]:
"""Per-extent eq. 8 targets, for tests and for a direct head loss."""
self.bank._check_layout(layout)
targets = []
for extent, indices in layout.groups():
rects = [layout.rects[index] for index in indices]
clean = gather_extent(y0, rects)
eps = gather_extent(noise, rects)
targets.append((indices, asymmetric_target(clean, eps, self.bank.basis(extent))))
return targets
def _check_tokens(self, tokens: torch.Tensor, layout: LotLayout) -> None:
expected = (layout.time, layout.height, layout.width, self.token_dim)
if tokens.ndim != 5 or tuple(tokens.shape[1:]) != expected:
raise ValueError(f"tokens shape {tuple(tokens.shape)} != (B, {expected[0]}, {expected[1]}, {expected[2]}, {expected[3]})")
self.bank._check_layout(layout)
|