"""Load official checkpoints into MLX modules. Weights stay in the Hugging Face cache. This package does not re-host them. * FastFrames-S refiner and pixel decoder: ``apple/FastFrames-S`` (apple-amlr). * Generator: ``zhuhz22/Causal-Forcing`` ``causal-forcing++/framewise-2step.pt`` (Apache-2.0), EMA key ``generator_ema``. * umT5 encoder, tokenizer, and Wan VAE: ``Wan-AI/Wan2.1-T2V-1.3B`` (Apache-2.0). """ from __future__ import annotations from pathlib import Path import mlx.core as mx import mlx.nn as nn from mlx.utils import tree_flatten from .config import WAN_1_3B, FastFramesConfig from .dit import CausalWanModel from .kv_refiner import KVRefinementNetwork from .pixel_decoder import PixelDecoder from .t5 import umt5_xxl _PREFIXES = ("model._fsdp_wrapped_module.", "_fsdp_wrapped_module.", "model.") FILES = { "refiner": ("apple/FastFrames-S", "fastframes-s/kv_refiner.safetensors"), "decoder": ("apple/FastFrames-S", "fastframes-s/pixel_decoder.safetensors"), "generator": ("zhuhz22/Causal-Forcing", "causal-forcing++/framewise-2step.pt"), "t5": ("Wan-AI/Wan2.1-T2V-1.3B", "models_t5_umt5-xxl-enc-bf16.pth"), "vae": ("Wan-AI/Wan2.1-T2V-1.3B", "Wan2.1_VAE.pth"), "tokenizer": ("Wan-AI/Wan2.1-T2V-1.3B", "google/umt5-xxl/tokenizer.json"), } def download(name: str, cache_dir: str | Path | None = None) -> Path: from huggingface_hub import hf_hub_download repo, filename = FILES[name] path = hf_hub_download(repo_id=repo, filename=filename, cache_dir=None if cache_dir is None else str(cache_dir)) return Path(path) def _strip(name: str) -> str: for prefix in _PREFIXES: if name.startswith(prefix): return name[len(prefix) :] return name def torch_to_mx(tensor): import torch tensor = tensor.detach().cpu().contiguous() if tensor.dtype == torch.bfloat16: return mx.array(tensor.view(torch.uint16).numpy()).view(mx.bfloat16) return mx.array(tensor.numpy()) def load_torch_state(path: Path, key: str | None = None) -> dict: import torch state = torch.load(path, map_location="cpu", weights_only=True) if key is not None: if key not in state: raise KeyError(f"{path} has no {key!r}; top-level keys: {list(state)[:12]}") state = state[key] return {_strip(k): v for k, v in state.items()} def load_safetensors(path: Path) -> dict: return dict(mx.load(str(path))) def _replace(key: str, pairs) -> str: for src, dst in pairs: key = key.replace(src, dst) return key DIT_KEYS = ( ("text_embedding.0.", "text_embed_fc1."), ("text_embedding.2.", "text_embed_fc2."), ("time_embedding.0.", "time_embed_fc1."), ("time_embedding.2.", "time_embed_fc2."), ("time_projection.1.", "time_proj_fc."), (".ffn.0.", ".ffn.fc1."), (".ffn.2.", ".ffn.fc2."), ) REFINER_KEYS = ( ("t_embed.mlp.0.", "t_embed.fc1."), ("t_embed.mlp.2.", "t_embed.fc2."), (".mlp.0.", ".mlp.fc1."), (".mlp.2.", ".mlp.fc2."), ) T5_DROP = ("ffn.gate.1.", "dropout") T5_KEYS = (("ffn.gate.0.", "ffn.gate."),) def _prepare(raw, renames=(), transpose_conv=True, drop_substrings=()): prepared = {} for key, value in raw.items(): if "freqs" in key or any(part in key for part in drop_substrings): continue key = _strip(key) key = _replace(key, renames) if not isinstance(value, mx.array): value = torch_to_mx(value) if transpose_conv and value.ndim == 5 and key.endswith("weight"): value = mx.transpose(value, (0, 2, 3, 4, 1)) prepared[key] = value return prepared def _set_nested(module, key, value): """Assign one parameter. ``load_weights`` turns all-digit path components into list indexes, which breaks ``phi_proj.768`` (a dict keyed by width). Walk lists and dicts explicitly instead. """ obj = module parts = key.split(".") for part in parts[:-1]: if isinstance(obj, list): obj = obj[int(part)] elif isinstance(obj, dict): obj = obj[part] else: obj = getattr(obj, part) leaf = parts[-1] if isinstance(obj, dict): obj[leaf] = value else: setattr(obj, leaf, value) def assign(module, arrays, drop_substrings=()): filtered = {} for key, value in arrays.items(): if any(part in key for part in drop_substrings): continue filtered[key] = value have = dict(tree_flatten(module.parameters())) missing = sorted(set(have) - set(filtered)) extra = sorted(set(filtered) - set(have)) if missing or extra: raise KeyError( f"checkpoint does not match the module: " f"{len(missing)} missing (e.g. {missing[:8]}), " f"{len(extra)} unexpected (e.g. {extra[:8]})" ) for key, value in filtered.items(): _set_nested(module, key, value) mx.eval(module.parameters()) def cast_floating(module, dtype): items = [(key, value.astype(dtype)) for key, value in tree_flatten(module.parameters())] for key, value in items: _set_nested(module, key, value) mx.eval(module.parameters()) def quantize_linears(module, bits: int, group_size: int = 64): """Post-training quantization of ``nn.Linear`` only. Not a calibrated quant.""" def predicate(path, mod): if not isinstance(mod, nn.Linear): return False # Group size must divide the input features. Skip the rare linear # (none in FastFrames-S at group size 64) rather than failing the load. return mod.weight.shape[-1] % group_size == 0 nn.quantize(module, bits=bits, group_size=group_size, class_predicate=predicate) return module def load_fastframes_s(cache_dir=None, bits: int | None = None, group_size: int = 64, load_vae: bool = False): """Download (if needed) and build the FastFrames-S stack in bf16, decoder in fp32.""" cfg = FastFramesConfig() dit_path = download("generator", cache_dir) refiner_path = download("refiner", cache_dir) decoder_path = download("decoder", cache_dir) t5_path = download("t5", cache_dir) tok_path = download("tokenizer", cache_dir) dit = CausalWanModel(WAN_1_3B) assign(dit, _prepare(load_torch_state(dit_path, key="generator_ema"), DIT_KEYS)) cast_floating(dit, mx.bfloat16) if bits: quantize_linears(dit, bits, group_size) refiner = KVRefinementNetwork() assign(refiner, _prepare(load_safetensors(refiner_path), REFINER_KEYS)) cast_floating(refiner, mx.bfloat16) decoder = PixelDecoder() assign(decoder, _prepare(load_safetensors(decoder_path))) text = umt5_xxl() assign(text, _prepare(load_torch_state(t5_path), T5_KEYS, drop_substrings=T5_DROP)) cast_floating(text, mx.bfloat16) if bits: quantize_linears(text, bits, group_size) vae = None if load_vae: vae = load_vae_model(cache_dir) return cfg, dit, refiner, decoder, text, tok_path, vae def load_vae_model(cache_dir=None): from .wan_vae import WanVAE path = download("vae", cache_dir) state = load_torch_state(path) arrays = {k: torch_to_mx(v) for k, v in state.items()} model = WanVAE() prepared = WanVAE.sanitize(arrays) assign(model, prepared) return model