Instructions to use pratikgx/FastFrames-S-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/FastFrames-S-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/FastFrames-S-MLX --local-dir FastFrames-S-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download fastframes_mlx/wan_vae.py from pratikgx/FastFrames-S-MLX: direct link, hf CLI and curl.
- Browser
- Download file 17.1 kB
-
https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/wan_vae.py
- Command line
-
hf download hf://pratikgx/FastFrames-S-MLX/fastframes_mlx/wan_vae.py
-
curl -L -o wan_vae.py https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/wan_vae.py
17.1 kB
| # Adapted from ml-explore/mlx-examples video/wan2.1 (MIT). | |
| # Upstream copyright: Copyright © 2026 Apple Inc. | |
| # Copyright © 2026 Apple Inc. | |
| """ | |
| Wan2.1 VAE encoder and decoder. | |
| Encodes video frames to latents and decodes latents to video frames | |
| using chunked processing with causal temporal caching. | |
| """ | |
| import re | |
| from typing import Dict, List, Optional | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| from .vae_layers import ( | |
| AttentionBlock, | |
| CausalConv3d, | |
| Resample, | |
| ResidualBlock, | |
| create_cache_entry, | |
| ) | |
| class Decoder3d(nn.Module): | |
| """ | |
| VAE Decoder for video generation. | |
| Input: [B, T, H/8, W/8, z_dim] (channels-last) | |
| Output: [B, T*4, H, W, 3] | |
| """ | |
| def __init__( | |
| self, | |
| dim: int = 96, | |
| z_dim: int = 16, | |
| dim_mult: Optional[List[int]] = None, | |
| num_res_blocks: int = 2, | |
| attn_scales: Optional[List[float]] = None, | |
| temporal_upsample: Optional[List[bool]] = None, | |
| ): | |
| super().__init__() | |
| if dim_mult is None: | |
| dim_mult = [1, 2, 4, 4] | |
| if attn_scales is None: | |
| attn_scales = [] | |
| if temporal_upsample is None: | |
| temporal_upsample = [True, True, False] | |
| self.dim = dim | |
| self.z_dim = z_dim | |
| self.dim_mult = dim_mult | |
| self.num_res_blocks = num_res_blocks | |
| self.temporal_upsample = temporal_upsample | |
| dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] | |
| scale = 1.0 / 2 ** (len(dim_mult) - 2) | |
| self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) | |
| self.middle_res1 = ResidualBlock(dims[0], dims[0]) | |
| self.middle_attn = AttentionBlock(dims[0]) | |
| self.middle_res2 = ResidualBlock(dims[0], dims[0]) | |
| # Build upsample stages as nested lists | |
| self.upsamples = [] | |
| for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): | |
| stage = [] | |
| if i == 1 or i == 2 or i == 3: | |
| in_dim = in_dim // 2 | |
| for j in range(num_res_blocks + 1): | |
| stage.append(ResidualBlock(in_dim, out_dim)) | |
| if scale in attn_scales: | |
| stage.append(AttentionBlock(out_dim)) | |
| in_dim = out_dim | |
| if i != len(dim_mult) - 1: | |
| mode = "upsample3d" if temporal_upsample[i] else "upsample2d" | |
| stage.append(Resample(out_dim, mode=mode)) | |
| scale *= 2.0 | |
| self.upsamples.append(stage) | |
| self.head_norm = nn.RMSNorm(dims[-1], eps=1e-12) | |
| self.head_conv = CausalConv3d(dims[-1], 3, 3, padding=1) | |
| # Count temporal cache slots from architecture | |
| n = 1 + 2 + 2 # conv1, middle_res1, middle_res2 | |
| for stage in self.upsamples: | |
| for layer in stage: | |
| if isinstance(layer, ResidualBlock): | |
| n += 2 | |
| elif isinstance(layer, Resample) and hasattr(layer, "time_conv"): | |
| n += 1 | |
| n += 1 # head_conv | |
| self.num_cache_slots = n | |
| def __call__(self, x, feat_cache): | |
| cache_idx = 0 | |
| new_cache = [] | |
| cache_input = x | |
| x = self.conv1(x, feat_cache[cache_idx]) | |
| new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx])) | |
| cache_idx += 1 | |
| x, c1, c2 = self.middle_res1( | |
| x, feat_cache[cache_idx], feat_cache[cache_idx + 1] | |
| ) | |
| new_cache.append(c1) | |
| new_cache.append(c2) | |
| cache_idx += 2 | |
| x = self.middle_attn(x) | |
| x, c1, c2 = self.middle_res2( | |
| x, feat_cache[cache_idx], feat_cache[cache_idx + 1] | |
| ) | |
| new_cache.append(c1) | |
| new_cache.append(c2) | |
| cache_idx += 2 | |
| for stage in self.upsamples: | |
| for layer in stage: | |
| if isinstance(layer, ResidualBlock): | |
| x, c1, c2 = layer( | |
| x, feat_cache[cache_idx], feat_cache[cache_idx + 1] | |
| ) | |
| new_cache.append(c1) | |
| new_cache.append(c2) | |
| cache_idx += 2 | |
| elif isinstance(layer, AttentionBlock): | |
| x = layer(x) | |
| elif isinstance(layer, Resample): | |
| x, c = layer(x, feat_cache[cache_idx]) | |
| if c is not None: | |
| new_cache.append(c) | |
| cache_idx += 1 | |
| x = self.head_norm(x) | |
| x = nn.silu(x) | |
| cache_input = x | |
| x = self.head_conv(x, feat_cache[cache_idx]) | |
| new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx])) | |
| cache_idx += 1 | |
| return x, new_cache | |
| class Encoder3d(nn.Module): | |
| """ | |
| VAE Encoder for video generation. | |
| Input: [B, T, H, W, 3] (channels-last) | |
| Output: [B, T', H/8, W/8, z_dim*2] | |
| """ | |
| def __init__( | |
| self, | |
| dim: int = 96, | |
| z_dim: int = 16, | |
| dim_mult: Optional[List[int]] = None, | |
| num_res_blocks: int = 2, | |
| attn_scales: Optional[List[float]] = None, | |
| temporal_downsample: Optional[List[bool]] = None, | |
| ): | |
| super().__init__() | |
| if dim_mult is None: | |
| dim_mult = [1, 2, 4, 4] | |
| if attn_scales is None: | |
| attn_scales = [] | |
| if temporal_downsample is None: | |
| temporal_downsample = [False, True, True] | |
| self.dim = dim | |
| self.z_dim = z_dim | |
| self.dim_mult = dim_mult | |
| self.num_res_blocks = num_res_blocks | |
| self.temporal_downsample = temporal_downsample | |
| dims = [dim * u for u in [1] + dim_mult] | |
| scale = 1.0 | |
| self.conv1 = CausalConv3d(3, dims[0], 3, padding=1) | |
| # Build downsample stages as nested lists | |
| self.downsamples = [] | |
| for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): | |
| stage = [] | |
| for j in range(num_res_blocks): | |
| stage.append(ResidualBlock(in_dim, out_dim)) | |
| if scale in attn_scales: | |
| stage.append(AttentionBlock(out_dim)) | |
| in_dim = out_dim | |
| if i != len(dim_mult) - 1: | |
| mode = "downsample3d" if temporal_downsample[i] else "downsample2d" | |
| stage.append(Resample(out_dim, mode=mode)) | |
| scale /= 2.0 | |
| self.downsamples.append(stage) | |
| self.middle_res1 = ResidualBlock(dims[-1], dims[-1]) | |
| self.middle_attn = AttentionBlock(dims[-1]) | |
| self.middle_res2 = ResidualBlock(dims[-1], dims[-1]) | |
| self.head_norm = nn.RMSNorm(dims[-1], eps=1e-12) | |
| self.head_conv = CausalConv3d(dims[-1], z_dim * 2, 3, padding=1) | |
| # Count temporal cache slots from architecture | |
| n = 1 # conv1 | |
| for stage in self.downsamples: | |
| for layer in stage: | |
| if isinstance(layer, ResidualBlock): | |
| n += 2 | |
| elif isinstance(layer, Resample) and hasattr(layer, "time_conv"): | |
| n += 1 | |
| n += 2 + 2 + 1 # middle_res1, middle_res2, head_conv | |
| self.num_cache_slots = n | |
| def __call__(self, x, feat_cache): | |
| cache_idx = 0 | |
| new_cache = [] | |
| cache_input = x | |
| x = self.conv1(x, feat_cache[cache_idx]) | |
| new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx])) | |
| cache_idx += 1 | |
| for stage in self.downsamples: | |
| for layer in stage: | |
| if isinstance(layer, ResidualBlock): | |
| x, c1, c2 = layer( | |
| x, feat_cache[cache_idx], feat_cache[cache_idx + 1] | |
| ) | |
| new_cache.append(c1) | |
| new_cache.append(c2) | |
| cache_idx += 2 | |
| elif isinstance(layer, AttentionBlock): | |
| x = layer(x) | |
| elif isinstance(layer, Resample): | |
| x, c = layer(x, feat_cache[cache_idx]) | |
| if c is not None: | |
| new_cache.append(c) | |
| cache_idx += 1 | |
| x, c1, c2 = self.middle_res1( | |
| x, feat_cache[cache_idx], feat_cache[cache_idx + 1] | |
| ) | |
| new_cache.append(c1) | |
| new_cache.append(c2) | |
| cache_idx += 2 | |
| x = self.middle_attn(x) | |
| x, c1, c2 = self.middle_res2( | |
| x, feat_cache[cache_idx], feat_cache[cache_idx + 1] | |
| ) | |
| new_cache.append(c1) | |
| new_cache.append(c2) | |
| cache_idx += 2 | |
| x = self.head_norm(x) | |
| x = nn.silu(x) | |
| cache_input = x | |
| x = self.head_conv(x, feat_cache[cache_idx]) | |
| new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx])) | |
| cache_idx += 1 | |
| return x, new_cache | |
| class WanVAE(nn.Module): | |
| """ | |
| High-level VAE wrapper for Wan2.1. | |
| Encode: [F, H, W, C] video -> [F', H/8, W/8, z_dim] latent | |
| Decode: [F, H, W, C] latent -> [F*4, H*8, W*8, 3] video clamped to [-1, 1] | |
| """ | |
| def __init__(self): | |
| super().__init__() | |
| self.encoder = Encoder3d() | |
| self.conv1 = CausalConv3d(32, 32, 1) | |
| self.decoder = Decoder3d() | |
| self.conv2 = CausalConv3d(16, 16, 1) | |
| self.mean = mx.array( | |
| [ | |
| -0.7571, | |
| -0.7089, | |
| -0.9113, | |
| 0.1075, | |
| -0.1745, | |
| 0.9653, | |
| -0.1517, | |
| 1.5508, | |
| 0.4134, | |
| -0.0715, | |
| 0.5517, | |
| -0.3632, | |
| -0.1922, | |
| -0.9497, | |
| 0.2503, | |
| -0.2921, | |
| ] | |
| ) | |
| self.std = mx.array( | |
| [ | |
| 2.8184, | |
| 1.4541, | |
| 2.3275, | |
| 2.6558, | |
| 1.2196, | |
| 1.7708, | |
| 2.6052, | |
| 2.0743, | |
| 3.2687, | |
| 2.1526, | |
| 2.8652, | |
| 1.5579, | |
| 1.6382, | |
| 1.1253, | |
| 2.8251, | |
| 1.9160, | |
| ] | |
| ) | |
| self.z_dim = 16 | |
| # Pre-compile for the frame-by-frame loop: avoids recompiling each frame. | |
| self._compiled_decode = mx.compile(self.decoder.__call__) | |
| self._compiled_encode = mx.compile(self.encoder.__call__) | |
| def decode(self, z: mx.array) -> mx.array: | |
| """ | |
| Decode latent to video. | |
| Args: | |
| z: Latent tensor [F, H, W, C] (channels-last) | |
| Returns: | |
| Video tensor [F, H, W, C] clamped to [-1, 1] (channels-last) | |
| """ | |
| # Add batch dim: [F, H, W, C] -> [1, F, H, W, C] | |
| z = z[None] | |
| # Unscale latents | |
| scale = 1.0 / self.std | |
| z = z / scale.reshape(1, 1, 1, 1, self.z_dim) + self.mean.reshape( | |
| 1, 1, 1, 1, self.z_dim | |
| ) | |
| # Pre-decoder conv | |
| x = self.conv2(z) | |
| # Decode one frame at a time. mx.eval per frame releases intermediates, keeping memory bounded. | |
| num_frames = x.shape[1] | |
| feat_cache = [None] * self.decoder.num_cache_slots | |
| outputs = [] | |
| for i in range(num_frames): | |
| frame = x[:, i : i + 1, :, :, :] | |
| out_frame, feat_cache = self._compiled_decode(frame, feat_cache) | |
| mx.eval(out_frame) | |
| outputs.append(out_frame) | |
| out = mx.concatenate(outputs, axis=1) | |
| out = mx.clip(out, -1.0, 1.0) | |
| # Remove batch dim: [1, F, H, W, C] -> [F, H, W, C] | |
| return out[0] | |
| def encode(self, x: mx.array) -> mx.array: | |
| """ | |
| Encode video to latent. | |
| Args: | |
| x: Video tensor [F, H, W, C] (channels-last) | |
| Returns: | |
| Latent tensor [F', H/8, W/8, C] (channels-last) | |
| """ | |
| # Add batch dim: [F, H, W, C] -> [1, F, H, W, C] | |
| x = x[None] | |
| num_frames = x.shape[1] | |
| feat_cache = [None] * self.encoder.num_cache_slots | |
| outputs = [] | |
| # First chunk is 1 frame (causal init), subsequent chunks are 4 frames (matching VAE temporal stride). | |
| i = 0 | |
| chunk_idx = 0 | |
| while i < num_frames: | |
| if chunk_idx == 0: | |
| chunk = x[:, i : i + 1, :, :, :] | |
| i += 1 | |
| else: | |
| chunk = x[:, i : i + 4, :, :, :] | |
| i += 4 | |
| out_chunk, feat_cache = self._compiled_encode(chunk, feat_cache) | |
| mx.eval(out_chunk) | |
| outputs.append(out_chunk) | |
| chunk_idx += 1 | |
| out = mx.concatenate(outputs, axis=1) | |
| # Post-encoder conv and extract mu | |
| out = self.conv1(out) | |
| mu = out[:, :, :, :, : self.z_dim] | |
| # Scale: (mu - mean) * (1/std) | |
| scale = 1.0 / self.std | |
| mu = (mu - self.mean.reshape(1, 1, 1, 1, self.z_dim)) * scale.reshape( | |
| 1, 1, 1, 1, self.z_dim | |
| ) | |
| # Remove batch dim: [1, F', H', W', C] -> [F', H', W', C] | |
| return mu[0] | |
| def sanitize(weights: Dict[str, mx.array]) -> Dict[str, mx.array]: | |
| """Remap PyTorch VAE keys to MLX format.""" | |
| remapped = {} | |
| for key, value in weights.items(): | |
| new_key = key | |
| # Transpose convolution weights | |
| if "weight" in new_key: | |
| if len(value.shape) == 5: | |
| value = mx.transpose(value, (0, 2, 3, 4, 1)) | |
| elif len(value.shape) == 4: | |
| value = mx.transpose(value, (0, 2, 3, 1)) | |
| new_key = new_key.replace(".gamma", ".weight") | |
| new_key = new_key.replace("decoder.middle.0.", "decoder.middle_res1.") | |
| new_key = new_key.replace("decoder.middle.1.", "decoder.middle_attn.") | |
| new_key = new_key.replace("decoder.middle.2.", "decoder.middle_res2.") | |
| new_key = new_key.replace("decoder.head.0.", "decoder.head_norm.") | |
| new_key = new_key.replace("decoder.head.2.", "decoder.head_conv.") | |
| new_key = new_key.replace("encoder.middle.0.", "encoder.middle_res1.") | |
| new_key = new_key.replace("encoder.middle.1.", "encoder.middle_attn.") | |
| new_key = new_key.replace("encoder.middle.2.", "encoder.middle_res2.") | |
| new_key = new_key.replace("encoder.head.0.", "encoder.head_norm.") | |
| new_key = new_key.replace("encoder.head.2.", "encoder.head_conv.") | |
| if "decoder.upsamples." in new_key: | |
| new_key = _map_vae_upsample_key(new_key) | |
| if "encoder.downsamples." in new_key: | |
| new_key = _map_vae_downsample_key(new_key) | |
| new_key = re.sub(r"\.residual\.0\.", ".norm1.", new_key) | |
| new_key = re.sub(r"\.residual\.2\.", ".conv1.", new_key) | |
| new_key = re.sub(r"\.residual\.3\.", ".norm2.", new_key) | |
| new_key = re.sub(r"\.residual\.6\.", ".conv2.", new_key) | |
| # Resample conv: .resample.1. -> .conv. | |
| new_key = re.sub(r"\.resample\.1\.", ".conv.", new_key) | |
| # Squeeze 1x1 conv weights to 2D for nn.Linear (to_qkv, proj) | |
| if ("to_qkv" in new_key or "proj" in new_key) and "weight" in new_key: | |
| if ( | |
| len(value.shape) == 4 | |
| and value.shape[1] == 1 | |
| and value.shape[2] == 1 | |
| ): | |
| value = value.reshape(value.shape[0], value.shape[3]) | |
| # Squeeze norm weights to 1D (required — nn.RMSNorm expects 1D) | |
| if "norm" in new_key and "weight" in new_key: | |
| if len(value.shape) > 1: | |
| value = mx.squeeze(value) | |
| remapped[new_key] = value | |
| return remapped | |
| def _map_vae_upsample_key(key: str) -> str: | |
| match = re.match(r"decoder\.upsamples\.(\d+)\.(.*)", key) | |
| if not match: | |
| return key | |
| layer_idx = int(match.group(1)) | |
| rest = match.group(2) | |
| # Decoder stages: (num_res_blocks+1) ResBlocks + 1 Resample each, except last (no Resample). | |
| # Assumes attn_scales=[] (Wan2.1 default — no AttentionBlocks in stages). | |
| num_res_blocks, num_stages = 2, 4 | |
| stage_sizes = [num_res_blocks + 2] * (num_stages - 1) + [num_res_blocks + 1] | |
| stage = 0 | |
| local_idx = layer_idx | |
| for s, size in enumerate(stage_sizes): | |
| if local_idx < size: | |
| stage = s | |
| break | |
| local_idx -= size | |
| return f"decoder.upsamples.{stage}.{local_idx}.{rest}" | |
| def _map_vae_downsample_key(key: str) -> str: | |
| match = re.match(r"encoder\.downsamples\.(\d+)\.(.*)", key) | |
| if not match: | |
| return key | |
| layer_idx = int(match.group(1)) | |
| rest = match.group(2) | |
| # Encoder stages: num_res_blocks ResBlocks + 1 Resample each, except last (no Resample). | |
| # Assumes attn_scales=[] (Wan2.1 default — no AttentionBlocks in stages). | |
| num_res_blocks, num_stages = 2, 4 | |
| stage_sizes = [num_res_blocks + 1] * (num_stages - 1) + [num_res_blocks] | |
| stage = 0 | |
| local_idx = layer_idx | |
| for s, size in enumerate(stage_sizes): | |
| if local_idx < size: | |
| stage = s | |
| break | |
| local_idx -= size | |
| return f"encoder.downsamples.{stage}.{local_idx}.{rest}" | |