Text-to-Video
MLX
apple-silicon
pratikgx's picture
Add unofficial MLX port of Apple FastFrames-S (code only; official weights loaded at runtime)
e87ef2c verified
Raw History Blame Contribute Delete
7.35 kB
"""Load official checkpoints into MLX modules.
Weights stay in the Hugging Face cache. This package does not re-host them.
* FastFrames-S refiner and pixel decoder: ``apple/FastFrames-S`` (apple-amlr).
* Generator: ``zhuhz22/Causal-Forcing`` ``causal-forcing++/framewise-2step.pt``
(Apache-2.0), EMA key ``generator_ema``.
* umT5 encoder, tokenizer, and Wan VAE: ``Wan-AI/Wan2.1-T2V-1.3B`` (Apache-2.0).
"""
from __future__ import annotations
from pathlib import Path
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from .config import WAN_1_3B, FastFramesConfig
from .dit import CausalWanModel
from .kv_refiner import KVRefinementNetwork
from .pixel_decoder import PixelDecoder
from .t5 import umt5_xxl
_PREFIXES = ("model._fsdp_wrapped_module.", "_fsdp_wrapped_module.", "model.")
FILES = {
"refiner": ("apple/FastFrames-S", "fastframes-s/kv_refiner.safetensors"),
"decoder": ("apple/FastFrames-S", "fastframes-s/pixel_decoder.safetensors"),
"generator": ("zhuhz22/Causal-Forcing", "causal-forcing++/framewise-2step.pt"),
"t5": ("Wan-AI/Wan2.1-T2V-1.3B", "models_t5_umt5-xxl-enc-bf16.pth"),
"vae": ("Wan-AI/Wan2.1-T2V-1.3B", "Wan2.1_VAE.pth"),
"tokenizer": ("Wan-AI/Wan2.1-T2V-1.3B", "google/umt5-xxl/tokenizer.json"),
}
def download(name: str, cache_dir: str | Path | None = None) -> Path:
from huggingface_hub import hf_hub_download
repo, filename = FILES[name]
path = hf_hub_download(repo_id=repo, filename=filename, cache_dir=None if cache_dir is None else str(cache_dir))
return Path(path)
def _strip(name: str) -> str:
for prefix in _PREFIXES:
if name.startswith(prefix):
return name[len(prefix) :]
return name
def torch_to_mx(tensor):
import torch
tensor = tensor.detach().cpu().contiguous()
if tensor.dtype == torch.bfloat16:
return mx.array(tensor.view(torch.uint16).numpy()).view(mx.bfloat16)
return mx.array(tensor.numpy())
def load_torch_state(path: Path, key: str | None = None) -> dict:
import torch
state = torch.load(path, map_location="cpu", weights_only=True)
if key is not None:
if key not in state:
raise KeyError(f"{path} has no {key!r}; top-level keys: {list(state)[:12]}")
state = state[key]
return {_strip(k): v for k, v in state.items()}
def load_safetensors(path: Path) -> dict:
return dict(mx.load(str(path)))
def _replace(key: str, pairs) -> str:
for src, dst in pairs:
key = key.replace(src, dst)
return key
DIT_KEYS = (
("text_embedding.0.", "text_embed_fc1."),
("text_embedding.2.", "text_embed_fc2."),
("time_embedding.0.", "time_embed_fc1."),
("time_embedding.2.", "time_embed_fc2."),
("time_projection.1.", "time_proj_fc."),
(".ffn.0.", ".ffn.fc1."),
(".ffn.2.", ".ffn.fc2."),
)
REFINER_KEYS = (
("t_embed.mlp.0.", "t_embed.fc1."),
("t_embed.mlp.2.", "t_embed.fc2."),
(".mlp.0.", ".mlp.fc1."),
(".mlp.2.", ".mlp.fc2."),
)
T5_DROP = ("ffn.gate.1.", "dropout")
T5_KEYS = (("ffn.gate.0.", "ffn.gate."),)
def _prepare(raw, renames=(), transpose_conv=True, drop_substrings=()):
prepared = {}
for key, value in raw.items():
if "freqs" in key or any(part in key for part in drop_substrings):
continue
key = _strip(key)
key = _replace(key, renames)
if not isinstance(value, mx.array):
value = torch_to_mx(value)
if transpose_conv and value.ndim == 5 and key.endswith("weight"):
value = mx.transpose(value, (0, 2, 3, 4, 1))
prepared[key] = value
return prepared
def _set_nested(module, key, value):
"""Assign one parameter.
``load_weights`` turns all-digit path components into list indexes, which
breaks ``phi_proj.768`` (a dict keyed by width). Walk lists and dicts
explicitly instead.
"""
obj = module
parts = key.split(".")
for part in parts[:-1]:
if isinstance(obj, list):
obj = obj[int(part)]
elif isinstance(obj, dict):
obj = obj[part]
else:
obj = getattr(obj, part)
leaf = parts[-1]
if isinstance(obj, dict):
obj[leaf] = value
else:
setattr(obj, leaf, value)
def assign(module, arrays, drop_substrings=()):
filtered = {}
for key, value in arrays.items():
if any(part in key for part in drop_substrings):
continue
filtered[key] = value
have = dict(tree_flatten(module.parameters()))
missing = sorted(set(have) - set(filtered))
extra = sorted(set(filtered) - set(have))
if missing or extra:
raise KeyError(
f"checkpoint does not match the module: "
f"{len(missing)} missing (e.g. {missing[:8]}), "
f"{len(extra)} unexpected (e.g. {extra[:8]})"
)
for key, value in filtered.items():
_set_nested(module, key, value)
mx.eval(module.parameters())
def cast_floating(module, dtype):
items = [(key, value.astype(dtype)) for key, value in tree_flatten(module.parameters())]
for key, value in items:
_set_nested(module, key, value)
mx.eval(module.parameters())
def quantize_linears(module, bits: int, group_size: int = 64):
"""Post-training quantization of ``nn.Linear`` only. Not a calibrated quant."""
def predicate(path, mod):
if not isinstance(mod, nn.Linear):
return False
# Group size must divide the input features. Skip the rare linear
# (none in FastFrames-S at group size 64) rather than failing the load.
return mod.weight.shape[-1] % group_size == 0
nn.quantize(module, bits=bits, group_size=group_size, class_predicate=predicate)
return module
def load_fastframes_s(cache_dir=None, bits: int | None = None, group_size: int = 64, load_vae: bool = False):
"""Download (if needed) and build the FastFrames-S stack in bf16, decoder in fp32."""
cfg = FastFramesConfig()
dit_path = download("generator", cache_dir)
refiner_path = download("refiner", cache_dir)
decoder_path = download("decoder", cache_dir)
t5_path = download("t5", cache_dir)
tok_path = download("tokenizer", cache_dir)
dit = CausalWanModel(WAN_1_3B)
assign(dit, _prepare(load_torch_state(dit_path, key="generator_ema"), DIT_KEYS))
cast_floating(dit, mx.bfloat16)
if bits:
quantize_linears(dit, bits, group_size)
refiner = KVRefinementNetwork()
assign(refiner, _prepare(load_safetensors(refiner_path), REFINER_KEYS))
cast_floating(refiner, mx.bfloat16)
decoder = PixelDecoder()
assign(decoder, _prepare(load_safetensors(decoder_path)))
text = umt5_xxl()
assign(text, _prepare(load_torch_state(t5_path), T5_KEYS, drop_substrings=T5_DROP))
cast_floating(text, mx.bfloat16)
if bits:
quantize_linears(text, bits, group_size)
vae = None
if load_vae:
vae = load_vae_model(cache_dir)
return cfg, dit, refiner, decoder, text, tok_path, vae
def load_vae_model(cache_dir=None):
from .wan_vae import WanVAE
path = download("vae", cache_dir)
state = load_torch_state(path)
arrays = {k: torch_to_mx(v) for k, v in state.items()}
model = WanVAE()
prepared = WanVAE.sanitize(arrays)
assign(model, prepared)
return model