File size: 10,019 Bytes
b9c6027
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
#
# For licensing see accompanying LICENSE file.
# Copyright (C) 2026 Apple Inc. All Rights Reserved.
#
"""KV refinement network (Sec. 3.3 of the paper).

After the final denoising step of a frame, the DiT has written *noisy* keys and
values into the cache slot of that frame. Instead of a second, clean DiT pass
over the frame, this network maps (noisy K/V, clean-latent features) to refined
K/V and overwrites the slot:

  * a shared per-token stem ``phi = E(patch_embed(x0))`` (one small transformer
    over the frame's tokens);
  * one light head per DiT layer:
    ``K' = renorm(K + dK)`` (post-RoPE, per-head norm preserved) and
    ``V' = (V + dV) * exp(s_V)``.

All residual branches are zero-initialised, so an untrained network is the
identity (the no-refinement baseline). The heads of all layers are evaluated as
batched ``baddbmm`` GEMMs, which keeps the network launch-efficient.
"""
from __future__ import annotations

from typing import List, Tuple

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.attention import SDPBackend, sdpa_kernel

__all__ = ["KVRefinementNetwork"]


def sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor:
    half = dim // 2
    position = position.type(torch.float64)
    div = torch.arange(half, dtype=torch.float64, device=position.device).div(half)
    sinusoid = torch.outer(position, torch.pow(10000.0, -div))
    return torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1).float()


def _sdpa(q, k, v):
    """``[B, H, L, D]`` attention; cuDNN for half precision on CUDA, math kernel otherwise."""
    if q.is_cuda and q.dtype in (torch.float16, torch.bfloat16):
        with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
            return F.scaled_dot_product_attention(q, k, v)
    return F.scaled_dot_product_attention(q, k, v)


class TimestepEmbedder(nn.Module):

    def __init__(self, d_a: int, freq_dim: int = 256):
        super().__init__()
        self.freq_dim = freq_dim
        self.mlp = nn.Sequential(nn.Linear(freq_dim, d_a), nn.SiLU(), nn.Linear(d_a, d_a))

    def forward(self, t: torch.Tensor) -> torch.Tensor:
        emb = sinusoidal_embedding_1d(self.freq_dim, t.flatten().float())
        return self.mlp(emb.to(self.mlp[0].weight.dtype))


class Modulation(nn.Module):
    """adaLN: ``e [B, d]`` -> ``n`` vectors ``[B, 1, d]``; zero-initialised."""

    def __init__(self, d_a: int, n: int):
        super().__init__()
        self.n = n
        self.lin = nn.Linear(d_a, n * d_a)
        nn.init.zeros_(self.lin.weight)
        nn.init.zeros_(self.lin.bias)

    def forward(self, e: torch.Tensor) -> Tuple[torch.Tensor, ...]:
        return self.lin(F.silu(e)).unsqueeze(1).chunk(self.n, dim=-1)


class StemBlock(nn.Module):
    """Pre-norm spatial transformer block with adaLN and zero-initialised gates."""

    def __init__(self, d_a: int, n_heads: int = 8, mlp_mult: int = 4):
        super().__init__()
        self.n_heads = n_heads
        self.head_dim = d_a // n_heads
        self.norm1 = nn.LayerNorm(d_a, elementwise_affine=False)
        self.qkv = nn.Linear(d_a, 3 * d_a)
        self.proj = nn.Linear(d_a, d_a)
        self.norm2 = nn.LayerNorm(d_a, elementwise_affine=False)
        self.mlp = nn.Sequential(
            nn.Linear(d_a, d_a * mlp_mult), nn.GELU(approximate="tanh"), nn.Linear(d_a * mlp_mult, d_a))
        self.mod = Modulation(d_a, 6)

    def forward(self, x: torch.Tensor, e: torch.Tensor) -> torch.Tensor:
        b, s, _ = x.shape
        sh1, sc1, g1, sh2, sc2, g2 = self.mod(e)
        h = self.norm1(x) * (1 + sc1) + sh1
        q, k, v = self.qkv(h).reshape(b, s, 3, self.n_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0)
        x = x + g1 * self.proj(_sdpa(q, k, v).transpose(1, 2).reshape(b, s, -1))
        h = self.norm2(x) * (1 + sc2) + sh2
        return x + g2 * self.mlp(h)


class GroupGEMM(nn.Module):
    """Refinement heads for a group of layers sharing one width ``(d_v, d_k)``.

    Weights of all members are stacked on a leading dimension; modulation and the
    V log-scale remain per layer.
    """

    def __init__(self, num_members: int, dim: int, d_a: int, d_v: int, d_k: int,
                 num_heads: int, head_dim: int, mlp_mult: int = 2):
        super().__init__()
        self.dim, self.d_a, self.d_v, self.d_k = dim, d_a, d_v, d_k
        self.num_heads, self.head_dim = num_heads, head_dim
        n = int(num_members)

        def weight(i, o, zero=False):
            return nn.Parameter(torch.zeros(n, i, o) if zero else torch.randn(n, i, o) * (i ** -0.5))

        def bias(o):
            return nn.Parameter(torch.zeros(n, 1, o))

        self.proj_k_w, self.proj_k_b = weight(dim, d_k), bias(d_k)
        self.proj_v_w, self.proj_v_b = weight(dim, d_v), bias(d_v)
        self.fuse_w, self.fuse_b = weight(d_k + 2 * d_v, d_v), bias(d_v)
        self.m0_w, self.m0_b = weight(d_v, d_v * mlp_mult), bias(d_v * mlp_mult)
        self.m2_w, self.m2_b = weight(d_v * mlp_mult, d_v), bias(d_v)
        self.mod_w, self.mod_b = weight(d_a, 2 * d_v, zero=True), bias(2 * d_v)
        self.dK_w, self.dK_b = weight(d_v, dim, zero=True), bias(dim)
        self.dV_w, self.dV_b = weight(d_v, dim, zero=True), bias(dim)
        self.v_logscale = nn.Parameter(torch.zeros(n, 1, dim))

    def _renorm_per_head(self, x, ref):
        b, s, _ = x.shape
        xh = x.reshape(b, s, self.num_heads, self.head_dim)
        rh = ref.reshape(b, s, self.num_heads, self.head_dim)
        xh = xh * (rh.norm(dim=-1, keepdim=True) / (xh.norm(dim=-1, keepdim=True) + 1e-6))
        return xh.reshape(b, s, -1)

    def forward(self, nK, nV, phi_v, e, layers=slice(None)):
        """nK, nV: ``[M, S, dim]``; phi_v: ``[M, S, d_v]``; e: ``[*, d_a]``.

        ``layers`` selects the ``M`` members the inputs belong to (default: all), e.g. one
        rank's share of the layers when the refinement is split across GPUs.
        """
        b = nK.shape[0]
        pk = torch.baddbmm(self.proj_k_b[layers], nK, self.proj_k_w[layers])
        pv = torch.baddbmm(self.proj_v_b[layers], nV, self.proj_v_w[layers])
        h = torch.baddbmm(self.fuse_b[layers], torch.cat([pk, pv, phi_v], dim=-1), self.fuse_w[layers])
        e1 = F.silu(e).reshape(1, 1, self.d_a).expand(b, 1, self.d_a).contiguous()
        shift, scale = torch.baddbmm(self.mod_b[layers], e1, self.mod_w[layers]).chunk(2, dim=-1)
        h = F.layer_norm(h, (self.d_v,)) * (1 + scale) + shift
        h = torch.baddbmm(self.m0_b[layers], h, self.m0_w[layers])
        h = torch.baddbmm(self.m2_b[layers], F.gelu(h, approximate="tanh"), self.m2_w[layers])
        k = self._renorm_per_head(nK + torch.baddbmm(self.dK_b[layers], h, self.dK_w[layers]), ref=nK)
        v = (nV + torch.baddbmm(self.dV_b[layers], h, self.dV_w[layers])) * torch.exp(self.v_logscale[layers])
        return k, v


class GroupedKVRepairHeads(nn.Module):
    """The refinement heads of all layers, evaluated together as one stacked ``GroupGEMM``.

    ``gemms`` and ``phi_proj`` keep the layout of the released state dicts
    (``heads_grouped.gemms.0.*``, ``heads_grouped.phi_proj.<d_a>.*``).
    """

    def __init__(self, num_blocks: int, dim: int, d_a: int, num_heads: int, head_dim: int, mlp_mult: int = 2):
        super().__init__()
        self.num_blocks = num_blocks
        self.gemms = nn.ModuleList([GroupGEMM(num_blocks, dim, d_a, d_a, d_a, num_heads, head_dim, mlp_mult)])
        self.phi_proj = nn.ModuleDict({str(d_a): nn.Linear(d_a, d_a, bias=False)})

    def forward(self, nK_all, nV_all, phi, e):
        """nK_all, nV_all: ``[L, S, dim]``; phi: ``[1, S, d_a]`` or ``[S, d_a]``."""
        phi2 = phi[0] if phi.dim() == 3 else phi
        gemm = self.gemms[0]
        phi_v = self.phi_proj[str(gemm.d_v)](phi2).unsqueeze(0).expand(self.num_blocks, -1, -1)
        return gemm(nK_all, nV_all, phi_v, e)


class KVRefinementNetwork(nn.Module):
    """Shared clean-latent stem plus one refinement head per DiT layer.

    Per frame::

        e = net.embed_timestep(t_ref, 1, device, dtype)
        phi = net.compute_phi(patch_embed(x0), e)
        K, V = net.repair_all(K_noisy, V_noisy, phi, e)   # [layers, tokens, dim]
    """

    def __init__(self, dim: int = 1536, num_heads: int = 12, head_dim: int = 128, num_blocks: int = 30,
                 d_a: int = 768, n_stem: int = 4, stem_n_heads: int = 8,
                 stem_mlp_mult: int = 4, head_mlp_mult: int = 2, freq_dim: int = 256):
        super().__init__()
        assert dim == num_heads * head_dim, (dim, num_heads, head_dim)
        self.dim, self.num_heads, self.head_dim = dim, num_heads, head_dim
        self.num_blocks, self.d_a, self.n_stem = num_blocks, d_a, n_stem
        self.t_embed = TimestepEmbedder(d_a, freq_dim)
        self.stem_in = nn.Linear(dim, d_a)
        self.stem_blocks = nn.ModuleList([StemBlock(d_a, stem_n_heads, stem_mlp_mult) for _ in range(n_stem)])
        self.stem_norm = nn.LayerNorm(d_a, elementwise_affine=True)
        self.heads_grouped = GroupedKVRepairHeads(num_blocks, dim, d_a, num_heads, head_dim, head_mlp_mult)

    def embed_timestep(self, timestep, batch_size: int, device, dtype) -> torch.Tensor:
        if not torch.is_tensor(timestep):
            timestep = torch.tensor([float(timestep)], device=device)
        timestep = timestep.to(device).float().reshape(-1)
        if timestep.numel() == 1:
            timestep = timestep.expand(batch_size)
        return self.t_embed(timestep).to(dtype)

    def compute_phi(self, stem_tokens: torch.Tensor, e: torch.Tensor) -> torch.Tensor:
        """stem_tokens: ``[B, S, dim]`` patch-embedded clean latent -> ``[B, S, d_a]``."""
        x = self.stem_in(stem_tokens)
        for block in self.stem_blocks:
            x = block(x, e)
        return self.stem_norm(x)

    def repair_all(self, nK_all, nV_all, phi, e):
        return self.heads_grouped(nK_all, nV_all, phi, e)

    @property
    def repaired_block_indices(self) -> List[int]:
        return list(range(self.num_blocks))