Instructions to use pratikgx/FastFrames-S-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/FastFrames-S-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/FastFrames-S-MLX --local-dir FastFrames-S-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download fastframes_mlx/weights.py from pratikgx/FastFrames-S-MLX: direct link, hf CLI and curl.
- Browser
- Download file 7.35 kB
-
https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/weights.py
- Command line
-
hf download hf://pratikgx/FastFrames-S-MLX/fastframes_mlx/weights.py
-
curl -L -o weights.py https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/weights.py
7.35 kB
| """Load official checkpoints into MLX modules. | |
| Weights stay in the Hugging Face cache. This package does not re-host them. | |
| * FastFrames-S refiner and pixel decoder: ``apple/FastFrames-S`` (apple-amlr). | |
| * Generator: ``zhuhz22/Causal-Forcing`` ``causal-forcing++/framewise-2step.pt`` | |
| (Apache-2.0), EMA key ``generator_ema``. | |
| * umT5 encoder, tokenizer, and Wan VAE: ``Wan-AI/Wan2.1-T2V-1.3B`` (Apache-2.0). | |
| """ | |
| from __future__ import annotations | |
| from pathlib import Path | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| from mlx.utils import tree_flatten | |
| from .config import WAN_1_3B, FastFramesConfig | |
| from .dit import CausalWanModel | |
| from .kv_refiner import KVRefinementNetwork | |
| from .pixel_decoder import PixelDecoder | |
| from .t5 import umt5_xxl | |
| _PREFIXES = ("model._fsdp_wrapped_module.", "_fsdp_wrapped_module.", "model.") | |
| FILES = { | |
| "refiner": ("apple/FastFrames-S", "fastframes-s/kv_refiner.safetensors"), | |
| "decoder": ("apple/FastFrames-S", "fastframes-s/pixel_decoder.safetensors"), | |
| "generator": ("zhuhz22/Causal-Forcing", "causal-forcing++/framewise-2step.pt"), | |
| "t5": ("Wan-AI/Wan2.1-T2V-1.3B", "models_t5_umt5-xxl-enc-bf16.pth"), | |
| "vae": ("Wan-AI/Wan2.1-T2V-1.3B", "Wan2.1_VAE.pth"), | |
| "tokenizer": ("Wan-AI/Wan2.1-T2V-1.3B", "google/umt5-xxl/tokenizer.json"), | |
| } | |
| def download(name: str, cache_dir: str | Path | None = None) -> Path: | |
| from huggingface_hub import hf_hub_download | |
| repo, filename = FILES[name] | |
| path = hf_hub_download(repo_id=repo, filename=filename, cache_dir=None if cache_dir is None else str(cache_dir)) | |
| return Path(path) | |
| def _strip(name: str) -> str: | |
| for prefix in _PREFIXES: | |
| if name.startswith(prefix): | |
| return name[len(prefix) :] | |
| return name | |
| def torch_to_mx(tensor): | |
| import torch | |
| tensor = tensor.detach().cpu().contiguous() | |
| if tensor.dtype == torch.bfloat16: | |
| return mx.array(tensor.view(torch.uint16).numpy()).view(mx.bfloat16) | |
| return mx.array(tensor.numpy()) | |
| def load_torch_state(path: Path, key: str | None = None) -> dict: | |
| import torch | |
| state = torch.load(path, map_location="cpu", weights_only=True) | |
| if key is not None: | |
| if key not in state: | |
| raise KeyError(f"{path} has no {key!r}; top-level keys: {list(state)[:12]}") | |
| state = state[key] | |
| return {_strip(k): v for k, v in state.items()} | |
| def load_safetensors(path: Path) -> dict: | |
| return dict(mx.load(str(path))) | |
| def _replace(key: str, pairs) -> str: | |
| for src, dst in pairs: | |
| key = key.replace(src, dst) | |
| return key | |
| DIT_KEYS = ( | |
| ("text_embedding.0.", "text_embed_fc1."), | |
| ("text_embedding.2.", "text_embed_fc2."), | |
| ("time_embedding.0.", "time_embed_fc1."), | |
| ("time_embedding.2.", "time_embed_fc2."), | |
| ("time_projection.1.", "time_proj_fc."), | |
| (".ffn.0.", ".ffn.fc1."), | |
| (".ffn.2.", ".ffn.fc2."), | |
| ) | |
| REFINER_KEYS = ( | |
| ("t_embed.mlp.0.", "t_embed.fc1."), | |
| ("t_embed.mlp.2.", "t_embed.fc2."), | |
| (".mlp.0.", ".mlp.fc1."), | |
| (".mlp.2.", ".mlp.fc2."), | |
| ) | |
| T5_DROP = ("ffn.gate.1.", "dropout") | |
| T5_KEYS = (("ffn.gate.0.", "ffn.gate."),) | |
| def _prepare(raw, renames=(), transpose_conv=True, drop_substrings=()): | |
| prepared = {} | |
| for key, value in raw.items(): | |
| if "freqs" in key or any(part in key for part in drop_substrings): | |
| continue | |
| key = _strip(key) | |
| key = _replace(key, renames) | |
| if not isinstance(value, mx.array): | |
| value = torch_to_mx(value) | |
| if transpose_conv and value.ndim == 5 and key.endswith("weight"): | |
| value = mx.transpose(value, (0, 2, 3, 4, 1)) | |
| prepared[key] = value | |
| return prepared | |
| def _set_nested(module, key, value): | |
| """Assign one parameter. | |
| ``load_weights`` turns all-digit path components into list indexes, which | |
| breaks ``phi_proj.768`` (a dict keyed by width). Walk lists and dicts | |
| explicitly instead. | |
| """ | |
| obj = module | |
| parts = key.split(".") | |
| for part in parts[:-1]: | |
| if isinstance(obj, list): | |
| obj = obj[int(part)] | |
| elif isinstance(obj, dict): | |
| obj = obj[part] | |
| else: | |
| obj = getattr(obj, part) | |
| leaf = parts[-1] | |
| if isinstance(obj, dict): | |
| obj[leaf] = value | |
| else: | |
| setattr(obj, leaf, value) | |
| def assign(module, arrays, drop_substrings=()): | |
| filtered = {} | |
| for key, value in arrays.items(): | |
| if any(part in key for part in drop_substrings): | |
| continue | |
| filtered[key] = value | |
| have = dict(tree_flatten(module.parameters())) | |
| missing = sorted(set(have) - set(filtered)) | |
| extra = sorted(set(filtered) - set(have)) | |
| if missing or extra: | |
| raise KeyError( | |
| f"checkpoint does not match the module: " | |
| f"{len(missing)} missing (e.g. {missing[:8]}), " | |
| f"{len(extra)} unexpected (e.g. {extra[:8]})" | |
| ) | |
| for key, value in filtered.items(): | |
| _set_nested(module, key, value) | |
| mx.eval(module.parameters()) | |
| def cast_floating(module, dtype): | |
| items = [(key, value.astype(dtype)) for key, value in tree_flatten(module.parameters())] | |
| for key, value in items: | |
| _set_nested(module, key, value) | |
| mx.eval(module.parameters()) | |
| def quantize_linears(module, bits: int, group_size: int = 64): | |
| """Post-training quantization of ``nn.Linear`` only. Not a calibrated quant.""" | |
| def predicate(path, mod): | |
| if not isinstance(mod, nn.Linear): | |
| return False | |
| # Group size must divide the input features. Skip the rare linear | |
| # (none in FastFrames-S at group size 64) rather than failing the load. | |
| return mod.weight.shape[-1] % group_size == 0 | |
| nn.quantize(module, bits=bits, group_size=group_size, class_predicate=predicate) | |
| return module | |
| def load_fastframes_s(cache_dir=None, bits: int | None = None, group_size: int = 64, load_vae: bool = False): | |
| """Download (if needed) and build the FastFrames-S stack in bf16, decoder in fp32.""" | |
| cfg = FastFramesConfig() | |
| dit_path = download("generator", cache_dir) | |
| refiner_path = download("refiner", cache_dir) | |
| decoder_path = download("decoder", cache_dir) | |
| t5_path = download("t5", cache_dir) | |
| tok_path = download("tokenizer", cache_dir) | |
| dit = CausalWanModel(WAN_1_3B) | |
| assign(dit, _prepare(load_torch_state(dit_path, key="generator_ema"), DIT_KEYS)) | |
| cast_floating(dit, mx.bfloat16) | |
| if bits: | |
| quantize_linears(dit, bits, group_size) | |
| refiner = KVRefinementNetwork() | |
| assign(refiner, _prepare(load_safetensors(refiner_path), REFINER_KEYS)) | |
| cast_floating(refiner, mx.bfloat16) | |
| decoder = PixelDecoder() | |
| assign(decoder, _prepare(load_safetensors(decoder_path))) | |
| text = umt5_xxl() | |
| assign(text, _prepare(load_torch_state(t5_path), T5_KEYS, drop_substrings=T5_DROP)) | |
| cast_floating(text, mx.bfloat16) | |
| if bits: | |
| quantize_linears(text, bits, group_size) | |
| vae = None | |
| if load_vae: | |
| vae = load_vae_model(cache_dir) | |
| return cfg, dit, refiner, decoder, text, tok_path, vae | |
| def load_vae_model(cache_dir=None): | |
| from .wan_vae import WanVAE | |
| path = download("vae", cache_dir) | |
| state = load_torch_state(path) | |
| arrays = {k: torch_to_mx(v) for k, v in state.items()} | |
| model = WanVAE() | |
| prepared = WanVAE.sanitize(arrays) | |
| assign(model, prepared) | |
| return model | |