Spaces:
Running on Zero
Running on Zero
File size: 10,019 Bytes
b9c6027 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 | #
# For licensing see accompanying LICENSE file.
# Copyright (C) 2026 Apple Inc. All Rights Reserved.
#
"""KV refinement network (Sec. 3.3 of the paper).
After the final denoising step of a frame, the DiT has written *noisy* keys and
values into the cache slot of that frame. Instead of a second, clean DiT pass
over the frame, this network maps (noisy K/V, clean-latent features) to refined
K/V and overwrites the slot:
* a shared per-token stem ``phi = E(patch_embed(x0))`` (one small transformer
over the frame's tokens);
* one light head per DiT layer:
``K' = renorm(K + dK)`` (post-RoPE, per-head norm preserved) and
``V' = (V + dV) * exp(s_V)``.
All residual branches are zero-initialised, so an untrained network is the
identity (the no-refinement baseline). The heads of all layers are evaluated as
batched ``baddbmm`` GEMMs, which keeps the network launch-efficient.
"""
from __future__ import annotations
from typing import List, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.attention import SDPBackend, sdpa_kernel
__all__ = ["KVRefinementNetwork"]
def sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor:
half = dim // 2
position = position.type(torch.float64)
div = torch.arange(half, dtype=torch.float64, device=position.device).div(half)
sinusoid = torch.outer(position, torch.pow(10000.0, -div))
return torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1).float()
def _sdpa(q, k, v):
"""``[B, H, L, D]`` attention; cuDNN for half precision on CUDA, math kernel otherwise."""
if q.is_cuda and q.dtype in (torch.float16, torch.bfloat16):
with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
return F.scaled_dot_product_attention(q, k, v)
return F.scaled_dot_product_attention(q, k, v)
class TimestepEmbedder(nn.Module):
def __init__(self, d_a: int, freq_dim: int = 256):
super().__init__()
self.freq_dim = freq_dim
self.mlp = nn.Sequential(nn.Linear(freq_dim, d_a), nn.SiLU(), nn.Linear(d_a, d_a))
def forward(self, t: torch.Tensor) -> torch.Tensor:
emb = sinusoidal_embedding_1d(self.freq_dim, t.flatten().float())
return self.mlp(emb.to(self.mlp[0].weight.dtype))
class Modulation(nn.Module):
"""adaLN: ``e [B, d]`` -> ``n`` vectors ``[B, 1, d]``; zero-initialised."""
def __init__(self, d_a: int, n: int):
super().__init__()
self.n = n
self.lin = nn.Linear(d_a, n * d_a)
nn.init.zeros_(self.lin.weight)
nn.init.zeros_(self.lin.bias)
def forward(self, e: torch.Tensor) -> Tuple[torch.Tensor, ...]:
return self.lin(F.silu(e)).unsqueeze(1).chunk(self.n, dim=-1)
class StemBlock(nn.Module):
"""Pre-norm spatial transformer block with adaLN and zero-initialised gates."""
def __init__(self, d_a: int, n_heads: int = 8, mlp_mult: int = 4):
super().__init__()
self.n_heads = n_heads
self.head_dim = d_a // n_heads
self.norm1 = nn.LayerNorm(d_a, elementwise_affine=False)
self.qkv = nn.Linear(d_a, 3 * d_a)
self.proj = nn.Linear(d_a, d_a)
self.norm2 = nn.LayerNorm(d_a, elementwise_affine=False)
self.mlp = nn.Sequential(
nn.Linear(d_a, d_a * mlp_mult), nn.GELU(approximate="tanh"), nn.Linear(d_a * mlp_mult, d_a))
self.mod = Modulation(d_a, 6)
def forward(self, x: torch.Tensor, e: torch.Tensor) -> torch.Tensor:
b, s, _ = x.shape
sh1, sc1, g1, sh2, sc2, g2 = self.mod(e)
h = self.norm1(x) * (1 + sc1) + sh1
q, k, v = self.qkv(h).reshape(b, s, 3, self.n_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0)
x = x + g1 * self.proj(_sdpa(q, k, v).transpose(1, 2).reshape(b, s, -1))
h = self.norm2(x) * (1 + sc2) + sh2
return x + g2 * self.mlp(h)
class GroupGEMM(nn.Module):
"""Refinement heads for a group of layers sharing one width ``(d_v, d_k)``.
Weights of all members are stacked on a leading dimension; modulation and the
V log-scale remain per layer.
"""
def __init__(self, num_members: int, dim: int, d_a: int, d_v: int, d_k: int,
num_heads: int, head_dim: int, mlp_mult: int = 2):
super().__init__()
self.dim, self.d_a, self.d_v, self.d_k = dim, d_a, d_v, d_k
self.num_heads, self.head_dim = num_heads, head_dim
n = int(num_members)
def weight(i, o, zero=False):
return nn.Parameter(torch.zeros(n, i, o) if zero else torch.randn(n, i, o) * (i ** -0.5))
def bias(o):
return nn.Parameter(torch.zeros(n, 1, o))
self.proj_k_w, self.proj_k_b = weight(dim, d_k), bias(d_k)
self.proj_v_w, self.proj_v_b = weight(dim, d_v), bias(d_v)
self.fuse_w, self.fuse_b = weight(d_k + 2 * d_v, d_v), bias(d_v)
self.m0_w, self.m0_b = weight(d_v, d_v * mlp_mult), bias(d_v * mlp_mult)
self.m2_w, self.m2_b = weight(d_v * mlp_mult, d_v), bias(d_v)
self.mod_w, self.mod_b = weight(d_a, 2 * d_v, zero=True), bias(2 * d_v)
self.dK_w, self.dK_b = weight(d_v, dim, zero=True), bias(dim)
self.dV_w, self.dV_b = weight(d_v, dim, zero=True), bias(dim)
self.v_logscale = nn.Parameter(torch.zeros(n, 1, dim))
def _renorm_per_head(self, x, ref):
b, s, _ = x.shape
xh = x.reshape(b, s, self.num_heads, self.head_dim)
rh = ref.reshape(b, s, self.num_heads, self.head_dim)
xh = xh * (rh.norm(dim=-1, keepdim=True) / (xh.norm(dim=-1, keepdim=True) + 1e-6))
return xh.reshape(b, s, -1)
def forward(self, nK, nV, phi_v, e, layers=slice(None)):
"""nK, nV: ``[M, S, dim]``; phi_v: ``[M, S, d_v]``; e: ``[*, d_a]``.
``layers`` selects the ``M`` members the inputs belong to (default: all), e.g. one
rank's share of the layers when the refinement is split across GPUs.
"""
b = nK.shape[0]
pk = torch.baddbmm(self.proj_k_b[layers], nK, self.proj_k_w[layers])
pv = torch.baddbmm(self.proj_v_b[layers], nV, self.proj_v_w[layers])
h = torch.baddbmm(self.fuse_b[layers], torch.cat([pk, pv, phi_v], dim=-1), self.fuse_w[layers])
e1 = F.silu(e).reshape(1, 1, self.d_a).expand(b, 1, self.d_a).contiguous()
shift, scale = torch.baddbmm(self.mod_b[layers], e1, self.mod_w[layers]).chunk(2, dim=-1)
h = F.layer_norm(h, (self.d_v,)) * (1 + scale) + shift
h = torch.baddbmm(self.m0_b[layers], h, self.m0_w[layers])
h = torch.baddbmm(self.m2_b[layers], F.gelu(h, approximate="tanh"), self.m2_w[layers])
k = self._renorm_per_head(nK + torch.baddbmm(self.dK_b[layers], h, self.dK_w[layers]), ref=nK)
v = (nV + torch.baddbmm(self.dV_b[layers], h, self.dV_w[layers])) * torch.exp(self.v_logscale[layers])
return k, v
class GroupedKVRepairHeads(nn.Module):
"""The refinement heads of all layers, evaluated together as one stacked ``GroupGEMM``.
``gemms`` and ``phi_proj`` keep the layout of the released state dicts
(``heads_grouped.gemms.0.*``, ``heads_grouped.phi_proj.<d_a>.*``).
"""
def __init__(self, num_blocks: int, dim: int, d_a: int, num_heads: int, head_dim: int, mlp_mult: int = 2):
super().__init__()
self.num_blocks = num_blocks
self.gemms = nn.ModuleList([GroupGEMM(num_blocks, dim, d_a, d_a, d_a, num_heads, head_dim, mlp_mult)])
self.phi_proj = nn.ModuleDict({str(d_a): nn.Linear(d_a, d_a, bias=False)})
def forward(self, nK_all, nV_all, phi, e):
"""nK_all, nV_all: ``[L, S, dim]``; phi: ``[1, S, d_a]`` or ``[S, d_a]``."""
phi2 = phi[0] if phi.dim() == 3 else phi
gemm = self.gemms[0]
phi_v = self.phi_proj[str(gemm.d_v)](phi2).unsqueeze(0).expand(self.num_blocks, -1, -1)
return gemm(nK_all, nV_all, phi_v, e)
class KVRefinementNetwork(nn.Module):
"""Shared clean-latent stem plus one refinement head per DiT layer.
Per frame::
e = net.embed_timestep(t_ref, 1, device, dtype)
phi = net.compute_phi(patch_embed(x0), e)
K, V = net.repair_all(K_noisy, V_noisy, phi, e) # [layers, tokens, dim]
"""
def __init__(self, dim: int = 1536, num_heads: int = 12, head_dim: int = 128, num_blocks: int = 30,
d_a: int = 768, n_stem: int = 4, stem_n_heads: int = 8,
stem_mlp_mult: int = 4, head_mlp_mult: int = 2, freq_dim: int = 256):
super().__init__()
assert dim == num_heads * head_dim, (dim, num_heads, head_dim)
self.dim, self.num_heads, self.head_dim = dim, num_heads, head_dim
self.num_blocks, self.d_a, self.n_stem = num_blocks, d_a, n_stem
self.t_embed = TimestepEmbedder(d_a, freq_dim)
self.stem_in = nn.Linear(dim, d_a)
self.stem_blocks = nn.ModuleList([StemBlock(d_a, stem_n_heads, stem_mlp_mult) for _ in range(n_stem)])
self.stem_norm = nn.LayerNorm(d_a, elementwise_affine=True)
self.heads_grouped = GroupedKVRepairHeads(num_blocks, dim, d_a, num_heads, head_dim, head_mlp_mult)
def embed_timestep(self, timestep, batch_size: int, device, dtype) -> torch.Tensor:
if not torch.is_tensor(timestep):
timestep = torch.tensor([float(timestep)], device=device)
timestep = timestep.to(device).float().reshape(-1)
if timestep.numel() == 1:
timestep = timestep.expand(batch_size)
return self.t_embed(timestep).to(dtype)
def compute_phi(self, stem_tokens: torch.Tensor, e: torch.Tensor) -> torch.Tensor:
"""stem_tokens: ``[B, S, dim]`` patch-embedded clean latent -> ``[B, S, d_a]``."""
x = self.stem_in(stem_tokens)
for block in self.stem_blocks:
x = block(x, e)
return self.stem_norm(x)
def repair_all(self, nK_all, nV_all, phi, e):
return self.heads_grouped(nK_all, nV_all, phi, e)
@property
def repaired_block_indices(self) -> List[int]:
return list(range(self.num_blocks))
|