Spaces:
Running on Zero
Running on Zero
Download fastframes/models/kv_refinement.py from mrfakename/fastframes-l: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/mrfakename/fastframes-l/resolve/main/fastframes/models/kv_refinement.py
- Command line
-
hf download hf://spaces/mrfakename/fastframes-l/fastframes/models/kv_refinement.py
-
curl -L -o kv_refinement.py https://huggingface.co/spaces/mrfakename/fastframes-l/resolve/main/fastframes/models/kv_refinement.py
10 kB
| # | |
| # For licensing see accompanying LICENSE file. | |
| # Copyright (C) 2026 Apple Inc. All Rights Reserved. | |
| # | |
| """KV refinement network (Sec. 3.3 of the paper). | |
| After the final denoising step of a frame, the DiT has written *noisy* keys and | |
| values into the cache slot of that frame. Instead of a second, clean DiT pass | |
| over the frame, this network maps (noisy K/V, clean-latent features) to refined | |
| K/V and overwrites the slot: | |
| * a shared per-token stem ``phi = E(patch_embed(x0))`` (one small transformer | |
| over the frame's tokens); | |
| * one light head per DiT layer: | |
| ``K' = renorm(K + dK)`` (post-RoPE, per-head norm preserved) and | |
| ``V' = (V + dV) * exp(s_V)``. | |
| All residual branches are zero-initialised, so an untrained network is the | |
| identity (the no-refinement baseline). The heads of all layers are evaluated as | |
| batched ``baddbmm`` GEMMs, which keeps the network launch-efficient. | |
| """ | |
| from __future__ import annotations | |
| from typing import List, Tuple | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.nn.attention import SDPBackend, sdpa_kernel | |
| __all__ = ["KVRefinementNetwork"] | |
| def sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor: | |
| half = dim // 2 | |
| position = position.type(torch.float64) | |
| div = torch.arange(half, dtype=torch.float64, device=position.device).div(half) | |
| sinusoid = torch.outer(position, torch.pow(10000.0, -div)) | |
| return torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1).float() | |
| def _sdpa(q, k, v): | |
| """``[B, H, L, D]`` attention; cuDNN for half precision on CUDA, math kernel otherwise.""" | |
| if q.is_cuda and q.dtype in (torch.float16, torch.bfloat16): | |
| with sdpa_kernel(SDPBackend.CUDNN_ATTENTION): | |
| return F.scaled_dot_product_attention(q, k, v) | |
| return F.scaled_dot_product_attention(q, k, v) | |
| class TimestepEmbedder(nn.Module): | |
| def __init__(self, d_a: int, freq_dim: int = 256): | |
| super().__init__() | |
| self.freq_dim = freq_dim | |
| self.mlp = nn.Sequential(nn.Linear(freq_dim, d_a), nn.SiLU(), nn.Linear(d_a, d_a)) | |
| def forward(self, t: torch.Tensor) -> torch.Tensor: | |
| emb = sinusoidal_embedding_1d(self.freq_dim, t.flatten().float()) | |
| return self.mlp(emb.to(self.mlp[0].weight.dtype)) | |
| class Modulation(nn.Module): | |
| """adaLN: ``e [B, d]`` -> ``n`` vectors ``[B, 1, d]``; zero-initialised.""" | |
| def __init__(self, d_a: int, n: int): | |
| super().__init__() | |
| self.n = n | |
| self.lin = nn.Linear(d_a, n * d_a) | |
| nn.init.zeros_(self.lin.weight) | |
| nn.init.zeros_(self.lin.bias) | |
| def forward(self, e: torch.Tensor) -> Tuple[torch.Tensor, ...]: | |
| return self.lin(F.silu(e)).unsqueeze(1).chunk(self.n, dim=-1) | |
| class StemBlock(nn.Module): | |
| """Pre-norm spatial transformer block with adaLN and zero-initialised gates.""" | |
| def __init__(self, d_a: int, n_heads: int = 8, mlp_mult: int = 4): | |
| super().__init__() | |
| self.n_heads = n_heads | |
| self.head_dim = d_a // n_heads | |
| self.norm1 = nn.LayerNorm(d_a, elementwise_affine=False) | |
| self.qkv = nn.Linear(d_a, 3 * d_a) | |
| self.proj = nn.Linear(d_a, d_a) | |
| self.norm2 = nn.LayerNorm(d_a, elementwise_affine=False) | |
| self.mlp = nn.Sequential( | |
| nn.Linear(d_a, d_a * mlp_mult), nn.GELU(approximate="tanh"), nn.Linear(d_a * mlp_mult, d_a)) | |
| self.mod = Modulation(d_a, 6) | |
| def forward(self, x: torch.Tensor, e: torch.Tensor) -> torch.Tensor: | |
| b, s, _ = x.shape | |
| sh1, sc1, g1, sh2, sc2, g2 = self.mod(e) | |
| h = self.norm1(x) * (1 + sc1) + sh1 | |
| q, k, v = self.qkv(h).reshape(b, s, 3, self.n_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0) | |
| x = x + g1 * self.proj(_sdpa(q, k, v).transpose(1, 2).reshape(b, s, -1)) | |
| h = self.norm2(x) * (1 + sc2) + sh2 | |
| return x + g2 * self.mlp(h) | |
| class GroupGEMM(nn.Module): | |
| """Refinement heads for a group of layers sharing one width ``(d_v, d_k)``. | |
| Weights of all members are stacked on a leading dimension; modulation and the | |
| V log-scale remain per layer. | |
| """ | |
| def __init__(self, num_members: int, dim: int, d_a: int, d_v: int, d_k: int, | |
| num_heads: int, head_dim: int, mlp_mult: int = 2): | |
| super().__init__() | |
| self.dim, self.d_a, self.d_v, self.d_k = dim, d_a, d_v, d_k | |
| self.num_heads, self.head_dim = num_heads, head_dim | |
| n = int(num_members) | |
| def weight(i, o, zero=False): | |
| return nn.Parameter(torch.zeros(n, i, o) if zero else torch.randn(n, i, o) * (i ** -0.5)) | |
| def bias(o): | |
| return nn.Parameter(torch.zeros(n, 1, o)) | |
| self.proj_k_w, self.proj_k_b = weight(dim, d_k), bias(d_k) | |
| self.proj_v_w, self.proj_v_b = weight(dim, d_v), bias(d_v) | |
| self.fuse_w, self.fuse_b = weight(d_k + 2 * d_v, d_v), bias(d_v) | |
| self.m0_w, self.m0_b = weight(d_v, d_v * mlp_mult), bias(d_v * mlp_mult) | |
| self.m2_w, self.m2_b = weight(d_v * mlp_mult, d_v), bias(d_v) | |
| self.mod_w, self.mod_b = weight(d_a, 2 * d_v, zero=True), bias(2 * d_v) | |
| self.dK_w, self.dK_b = weight(d_v, dim, zero=True), bias(dim) | |
| self.dV_w, self.dV_b = weight(d_v, dim, zero=True), bias(dim) | |
| self.v_logscale = nn.Parameter(torch.zeros(n, 1, dim)) | |
| def _renorm_per_head(self, x, ref): | |
| b, s, _ = x.shape | |
| xh = x.reshape(b, s, self.num_heads, self.head_dim) | |
| rh = ref.reshape(b, s, self.num_heads, self.head_dim) | |
| xh = xh * (rh.norm(dim=-1, keepdim=True) / (xh.norm(dim=-1, keepdim=True) + 1e-6)) | |
| return xh.reshape(b, s, -1) | |
| def forward(self, nK, nV, phi_v, e, layers=slice(None)): | |
| """nK, nV: ``[M, S, dim]``; phi_v: ``[M, S, d_v]``; e: ``[*, d_a]``. | |
| ``layers`` selects the ``M`` members the inputs belong to (default: all), e.g. one | |
| rank's share of the layers when the refinement is split across GPUs. | |
| """ | |
| b = nK.shape[0] | |
| pk = torch.baddbmm(self.proj_k_b[layers], nK, self.proj_k_w[layers]) | |
| pv = torch.baddbmm(self.proj_v_b[layers], nV, self.proj_v_w[layers]) | |
| h = torch.baddbmm(self.fuse_b[layers], torch.cat([pk, pv, phi_v], dim=-1), self.fuse_w[layers]) | |
| e1 = F.silu(e).reshape(1, 1, self.d_a).expand(b, 1, self.d_a).contiguous() | |
| shift, scale = torch.baddbmm(self.mod_b[layers], e1, self.mod_w[layers]).chunk(2, dim=-1) | |
| h = F.layer_norm(h, (self.d_v,)) * (1 + scale) + shift | |
| h = torch.baddbmm(self.m0_b[layers], h, self.m0_w[layers]) | |
| h = torch.baddbmm(self.m2_b[layers], F.gelu(h, approximate="tanh"), self.m2_w[layers]) | |
| k = self._renorm_per_head(nK + torch.baddbmm(self.dK_b[layers], h, self.dK_w[layers]), ref=nK) | |
| v = (nV + torch.baddbmm(self.dV_b[layers], h, self.dV_w[layers])) * torch.exp(self.v_logscale[layers]) | |
| return k, v | |
| class GroupedKVRepairHeads(nn.Module): | |
| """The refinement heads of all layers, evaluated together as one stacked ``GroupGEMM``. | |
| ``gemms`` and ``phi_proj`` keep the layout of the released state dicts | |
| (``heads_grouped.gemms.0.*``, ``heads_grouped.phi_proj.<d_a>.*``). | |
| """ | |
| def __init__(self, num_blocks: int, dim: int, d_a: int, num_heads: int, head_dim: int, mlp_mult: int = 2): | |
| super().__init__() | |
| self.num_blocks = num_blocks | |
| self.gemms = nn.ModuleList([GroupGEMM(num_blocks, dim, d_a, d_a, d_a, num_heads, head_dim, mlp_mult)]) | |
| self.phi_proj = nn.ModuleDict({str(d_a): nn.Linear(d_a, d_a, bias=False)}) | |
| def forward(self, nK_all, nV_all, phi, e): | |
| """nK_all, nV_all: ``[L, S, dim]``; phi: ``[1, S, d_a]`` or ``[S, d_a]``.""" | |
| phi2 = phi[0] if phi.dim() == 3 else phi | |
| gemm = self.gemms[0] | |
| phi_v = self.phi_proj[str(gemm.d_v)](phi2).unsqueeze(0).expand(self.num_blocks, -1, -1) | |
| return gemm(nK_all, nV_all, phi_v, e) | |
| class KVRefinementNetwork(nn.Module): | |
| """Shared clean-latent stem plus one refinement head per DiT layer. | |
| Per frame:: | |
| e = net.embed_timestep(t_ref, 1, device, dtype) | |
| phi = net.compute_phi(patch_embed(x0), e) | |
| K, V = net.repair_all(K_noisy, V_noisy, phi, e) # [layers, tokens, dim] | |
| """ | |
| def __init__(self, dim: int = 1536, num_heads: int = 12, head_dim: int = 128, num_blocks: int = 30, | |
| d_a: int = 768, n_stem: int = 4, stem_n_heads: int = 8, | |
| stem_mlp_mult: int = 4, head_mlp_mult: int = 2, freq_dim: int = 256): | |
| super().__init__() | |
| assert dim == num_heads * head_dim, (dim, num_heads, head_dim) | |
| self.dim, self.num_heads, self.head_dim = dim, num_heads, head_dim | |
| self.num_blocks, self.d_a, self.n_stem = num_blocks, d_a, n_stem | |
| self.t_embed = TimestepEmbedder(d_a, freq_dim) | |
| self.stem_in = nn.Linear(dim, d_a) | |
| self.stem_blocks = nn.ModuleList([StemBlock(d_a, stem_n_heads, stem_mlp_mult) for _ in range(n_stem)]) | |
| self.stem_norm = nn.LayerNorm(d_a, elementwise_affine=True) | |
| self.heads_grouped = GroupedKVRepairHeads(num_blocks, dim, d_a, num_heads, head_dim, head_mlp_mult) | |
| def embed_timestep(self, timestep, batch_size: int, device, dtype) -> torch.Tensor: | |
| if not torch.is_tensor(timestep): | |
| timestep = torch.tensor([float(timestep)], device=device) | |
| timestep = timestep.to(device).float().reshape(-1) | |
| if timestep.numel() == 1: | |
| timestep = timestep.expand(batch_size) | |
| return self.t_embed(timestep).to(dtype) | |
| def compute_phi(self, stem_tokens: torch.Tensor, e: torch.Tensor) -> torch.Tensor: | |
| """stem_tokens: ``[B, S, dim]`` patch-embedded clean latent -> ``[B, S, d_a]``.""" | |
| x = self.stem_in(stem_tokens) | |
| for block in self.stem_blocks: | |
| x = block(x, e) | |
| return self.stem_norm(x) | |
| def repair_all(self, nK_all, nV_all, phi, e): | |
| return self.heads_grouped(nK_all, nV_all, phi, e) | |
| def repaired_block_indices(self) -> List[int]: | |
| return list(range(self.num_blocks)) | |