Instructions to use pratikgx/FastFrames-S-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/FastFrames-S-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/FastFrames-S-MLX --local-dir FastFrames-S-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download fastframes_mlx/dit.py from pratikgx/FastFrames-S-MLX: direct link, hf CLI and curl.
- Browser
- Download file 10.7 kB
-
https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/dit.py
- Command line
-
hf download hf://pratikgx/FastFrames-S-MLX/fastframes_mlx/dit.py
-
curl -L -o dit.py https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/dit.py
10.7 kB
| """Frame-wise causal Wan2.1 DiT used as the FastFrames generator. | |
| This is the inference path in ``CausalWanModel.forward_frame``: one latent | |
| frame attends to its own tokens and to the KV cache of earlier frames. The | |
| pre-head hidden state is what the pixel decoder reads. RoPE follows the | |
| complex reference (FastFrames-S sets ``fast_ops=False``) but the multiply is | |
| float32 so it can run on the GPU. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| from .config import WanConfig | |
| from .ops import gelu_tanh, layer_norm, rms_norm, sdpa, silu, sinusoidal_embedding_1d | |
| from .rope import RopeTables | |
| def _zeros(shape): | |
| return mx.zeros(shape) | |
| class _RMS(nn.Module): | |
| def __init__(self, dim, eps): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = mx.ones((dim,)) | |
| def __call__(self, x): | |
| return rms_norm(x, self.weight, self.eps) | |
| class _AffineLN(nn.Module): | |
| def __init__(self, dim, eps): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = mx.ones((dim,)) | |
| self.bias = mx.zeros((dim,)) | |
| def __call__(self, x): | |
| return layer_norm(x, self.eps, self.weight, self.bias) | |
| class SelfAttention(nn.Module): | |
| def __init__(self, dim, num_heads, eps): | |
| super().__init__() | |
| self.num_heads = num_heads | |
| self.head_dim = dim // num_heads | |
| self.q = nn.Linear(dim, dim) | |
| self.k = nn.Linear(dim, dim) | |
| self.v = nn.Linear(dim, dim) | |
| self.o = nn.Linear(dim, dim) | |
| self.norm_q = _RMS(dim, eps) | |
| self.norm_k = _RMS(dim, eps) | |
| def qkv(self, x): | |
| b, s, _ = x.shape | |
| q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim) | |
| k = self.norm_k(self.k(x)).reshape(b, s, self.num_heads, self.head_dim) | |
| v = self.v(x).reshape(b, s, self.num_heads, self.head_dim) | |
| return q, k, v | |
| def text_kv(self, context): | |
| b = context.shape[0] | |
| k = self.norm_k(self.k(context)).reshape(b, -1, self.num_heads, self.head_dim) | |
| v = self.v(context).reshape(b, -1, self.num_heads, self.head_dim) | |
| return k, v | |
| class CrossAttention(SelfAttention): | |
| def __call__(self, x, cache): | |
| b, s, _ = x.shape | |
| q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim) | |
| k, v = cache | |
| out = sdpa(q, k, v).reshape(b, s, -1) | |
| return self.o(out) | |
| class FeedForward(nn.Module): | |
| def __init__(self, dim, ffn_dim): | |
| super().__init__() | |
| self.fc1 = nn.Linear(dim, ffn_dim) | |
| self.fc2 = nn.Linear(ffn_dim, dim) | |
| def __call__(self, x): | |
| return self.fc2(gelu_tanh(self.fc1(x))) | |
| class CausalBlock(nn.Module): | |
| def __init__(self, dim, ffn_dim, num_heads, eps): | |
| super().__init__() | |
| self.eps = eps | |
| self.self_attn = SelfAttention(dim, num_heads, eps) | |
| self.cross_attn = CrossAttention(dim, num_heads, eps) | |
| self.norm3 = _AffineLN(dim, eps) | |
| self.ffn = FeedForward(dim, ffn_dim) | |
| self.modulation = _zeros((1, 6, dim)) | |
| def forward_frame(self, x, e, grid, rope: RopeTables, cross_kv, kv_cache, frame: int): | |
| """``x`` is ``[B, tokens, dim]``. ``e`` is ``[B, 1, 6, dim]``.""" | |
| tokens = x.shape[1] | |
| frames, height, width = grid | |
| mod = self.modulation.reshape(1, 1, 6, -1) + e | |
| parts = mx.split(mod, 6, axis=2) | |
| e0, e1, e2, e3, e4, e5 = [p.reshape(p.shape[0], p.shape[1], 1, p.shape[-1]) for p in parts] | |
| h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) | |
| h = (h * (1.0 + e1) + e0).reshape(x.shape[0], tokens, -1) | |
| q, k, v = self.self_attn.qkv(h) | |
| q = rope.apply(q, frames, height, width, start_frame=frame) | |
| k = rope.apply(k, frames, height, width, start_frame=frame) | |
| kv_cache.write(frame, k, v) | |
| k_pre, v_pre = kv_cache.prefix(frame) | |
| y = self.self_attn.o(sdpa(q, k_pre, v_pre).reshape(x.shape[0], tokens, -1)) | |
| x = x + (y.reshape(x.shape[0], 1, tokens, -1) * e2).reshape(x.shape[0], tokens, -1) | |
| x = x + self.cross_attn(self.norm3(x), cross_kv) | |
| y = self.ffn((layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) * (1.0 + e4) + e3).reshape(x.shape[0], tokens, -1)) | |
| y = (y.reshape(x.shape[0], 1, tokens, -1) * e5).reshape(x.shape[0], tokens, -1) | |
| return x + y | |
| class CausalHead(nn.Module): | |
| def __init__(self, dim, out_dim, patch_size, eps): | |
| super().__init__() | |
| self.eps = eps | |
| self.patch_size = patch_size | |
| self.out_dim = out_dim | |
| self.head = nn.Linear(dim, math.prod(patch_size) * out_dim) | |
| self.modulation = _zeros((1, 2, dim)) | |
| def __call__(self, x, e): | |
| """``x`` ``[B, tokens, dim]``, ``e`` ``[B, 1, 1, dim]``.""" | |
| tokens = x.shape[1] | |
| mod = self.modulation.reshape(1, 1, 2, -1) + e | |
| shift, scale = mx.split(mod, 2, axis=2) | |
| h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) | |
| h = (h * (1.0 + scale) + shift).reshape(x.shape[0], tokens, -1) | |
| return self.head(h) | |
| def unpatchify(tokens, grid, patch_size, out_dim): | |
| """``[B, tokens, prod(patch)*C]`` -> ``[B, C, F, H, W]`` in Wan layout.""" | |
| b = tokens.shape[0] | |
| f, h, w = grid | |
| pt, ph, pw = patch_size | |
| u = tokens.reshape(b, f, h, w, pt, ph, pw, out_dim) | |
| u = mx.transpose(u, (0, 7, 1, 4, 2, 5, 3, 6)) | |
| return u.reshape(b, out_dim, f * pt, h * ph, w * pw) | |
| class PatchEmbed(nn.Module): | |
| def __init__(self, in_dim, dim, patch_size): | |
| super().__init__() | |
| self.patch_size = tuple(patch_size) | |
| pt, ph, pw = self.patch_size | |
| # MLX conv weight is [out, kd, kh, kw, in]. | |
| self.weight = _zeros((dim, pt, ph, pw, in_dim)) | |
| self.bias = _zeros((dim,)) | |
| def __call__(self, latent): | |
| """``latent`` is ``[B, 1, C, H, W]`` -> ``[B, tokens, dim]``.""" | |
| from .ops import conv3d_ncdhw | |
| x = mx.transpose(latent, (0, 2, 1, 3, 4)) | |
| y = conv3d_ncdhw(x, self.weight, self.bias, stride=self.patch_size) | |
| y = mx.transpose(y, (0, 2, 3, 4, 1)) | |
| return y.reshape(y.shape[0], -1, y.shape[-1]) | |
| class KVCache: | |
| """Per-layer list of per-frame ``(k, v)`` with shape ``[B, tokens, heads, dim]``.""" | |
| def __init__(self, num_frames: int): | |
| self.num_frames = num_frames | |
| self.slots = [None] * num_frames | |
| def write(self, frame, k, v): | |
| mx.eval(k, v) | |
| self.slots[frame] = (k, v) | |
| def prefix(self, frame): | |
| ks = [self.slots[i][0] for i in range(frame + 1)] | |
| vs = [self.slots[i][1] for i in range(frame + 1)] | |
| if len(ks) == 1: | |
| return ks[0], vs[0] | |
| return mx.concatenate(ks, axis=1), mx.concatenate(vs, axis=1) | |
| def slot(self, frame): | |
| return self.slots[frame] | |
| class LayerCache: | |
| def __init__(self, num_layers, num_frames): | |
| self.layers = [KVCache(num_frames) for _ in range(num_layers)] | |
| def __getitem__(self, index): | |
| return self.layers[index] | |
| class CausalWanModel(nn.Module): | |
| def __init__(self, config: WanConfig = WanConfig()): | |
| super().__init__() | |
| self.config = config | |
| dim = config.dim | |
| self.dim = dim | |
| self.num_heads = config.num_heads | |
| self.num_layers = config.num_layers | |
| self.patch_size = config.patch_size | |
| self.text_len = config.text_len | |
| self.freq_dim = config.freq_dim | |
| self.out_dim = config.out_dim | |
| self.eps = config.eps | |
| self.patch_embedding = PatchEmbed(config.in_dim, dim, config.patch_size) | |
| self.text_embed_fc1 = nn.Linear(config.text_dim, dim) | |
| self.text_embed_fc2 = nn.Linear(dim, dim) | |
| self.time_embed_fc1 = nn.Linear(config.freq_dim, dim) | |
| self.time_embed_fc2 = nn.Linear(dim, dim) | |
| self.time_proj_fc = nn.Linear(dim, dim * 6) | |
| self.blocks = [ | |
| CausalBlock(dim, config.ffn_dim, config.num_heads, config.eps) for _ in range(config.num_layers) | |
| ] | |
| self.head = CausalHead(dim, config.out_dim, config.patch_size, config.eps) | |
| # Plain object so the cos/sin tables are not registered as parameters. | |
| self.rope = RopeTables(config.head_dim) | |
| def embed_text(self, prompt_embeds): | |
| if isinstance(prompt_embeds, (list, tuple)): | |
| rows = [] | |
| for row in prompt_embeds: | |
| if row.shape[0] < self.text_len: | |
| pad = mx.zeros((self.text_len - row.shape[0], row.shape[-1]), dtype=row.dtype) | |
| row = mx.concatenate([row, pad], axis=0) | |
| rows.append(row[: self.text_len]) | |
| prompt_embeds = mx.stack(rows, axis=0) | |
| elif prompt_embeds.shape[1] < self.text_len: | |
| pad = mx.zeros( | |
| (prompt_embeds.shape[0], self.text_len - prompt_embeds.shape[1], prompt_embeds.shape[2]), | |
| dtype=prompt_embeds.dtype, | |
| ) | |
| prompt_embeds = mx.concatenate([prompt_embeds, pad], axis=1) | |
| elif prompt_embeds.shape[1] > self.text_len: | |
| prompt_embeds = prompt_embeds[:, : self.text_len] | |
| return self.text_embed_fc2(gelu_tanh(self.text_embed_fc1(prompt_embeds))) | |
| def time_conditioning(self, t, dtype): | |
| """``t`` is ``[B, F]`` -> ``(e0 [B,F,6,C], e_head [B,F,1,C])``.""" | |
| emb = sinusoidal_embedding_1d(self.freq_dim, t.reshape(-1)).astype(dtype) | |
| e = self.time_embed_fc2(silu(self.time_embed_fc1(emb))) | |
| e0 = self.time_proj_fc(silu(e)).reshape(*t.shape, 6, self.dim) | |
| e_head = e.reshape(*t.shape, 1, self.dim) | |
| return e0, e_head | |
| def text_cache(self, prompt_embeds): | |
| context = self.embed_text(prompt_embeds) | |
| caches = [] | |
| for block in self.blocks: | |
| k, v = block.cross_attn.text_kv(context) | |
| mx.eval(k, v) | |
| caches.append((k, v)) | |
| return caches | |
| def forward_frame(self, latent, e0, e_head, text_cache, kv_cache: LayerCache, frame: int): | |
| """Denoise ``latent`` ``[B, 1, 16, H, W]``. | |
| Returns flow ``[B, 1, 16, H, W]`` and pre-head states ``[B, tokens, dim]``. | |
| """ | |
| _, _, _, height, width = latent.shape | |
| x = self.patch_embedding(latent) | |
| grid = (1, height // self.patch_size[1], width // self.patch_size[2]) | |
| for index, block in enumerate(self.blocks): | |
| x = block.forward_frame(x, e0, grid, self.rope, text_cache[index], kv_cache[index], frame) | |
| mx.eval(x) | |
| hidden = x | |
| tokens = self.head(x, e_head) | |
| flow = unpatchify(tokens, grid, self.patch_size, self.out_dim) | |
| flow = mx.transpose(flow, (0, 2, 1, 3, 4)) | |
| return flow, hidden | |