Text-to-Video
MLX
apple-silicon
pratikgx's picture
Add unofficial MLX port of Apple FastFrames-S (code only; official weights loaded at runtime)
e87ef2c verified
Raw History Blame Contribute Delete
4.85 kB
"""Generate a clip with FastFrames-S on MLX.
python -m fastframes_mlx.generate --prompt "A corgi surfing a wave at sunset" --out clip.mp4
"""
from __future__ import annotations
import argparse
from dataclasses import replace
from pathlib import Path
import mlx.core as mx
from .config import FastFramesConfig, WanConfig
from .dit import CausalWanModel
from .kv_refiner import KVRefinementNetwork
from .pipeline import FastFrames, format_report
from .pixel_decoder import PixelDecoder
from .tokenizer import UMT5Tokenizer
from .video import save_mp4
def _smoke_stack():
"""Tiny random weights so the CLI path can be exercised without a download."""
wan = WanConfig(dim=32, ffn_dim=64, num_heads=4, num_layers=2, text_dim=32, freq_dim=16)
cfg = FastFramesConfig(
name="smoke",
wan=wan,
num_frames=2,
latent_height=4,
latent_width=4,
num_pixel_frames=5,
pixel_height=8,
pixel_width=8,
refiner_width=16,
refiner_stem_blocks=1,
refiner_stem_heads=4,
refiner_stem_mlp=2,
pixel_patch=(4, 4, 4),
decoder_low_blocks=1,
decoder_high_blocks=1,
decoder_ffn=2,
)
dit = CausalWanModel(wan)
refiner = KVRefinementNetwork(
dim=wan.dim,
num_heads=wan.num_heads,
num_blocks=wan.num_layers,
width=cfg.refiner_width,
n_stem=cfg.refiner_stem_blocks,
stem_heads=cfg.refiner_stem_heads,
stem_mlp=cfg.refiner_stem_mlp,
)
decoder = PixelDecoder(
dim=wan.dim,
pixel_patch_size=cfg.pixel_patch,
n_low_blocks=cfg.decoder_low_blocks,
n_high_blocks=cfg.decoder_high_blocks,
ffn_mult=cfg.decoder_ffn,
)
# Fixed prompt embedding: the smoke path does not load umT5.
prompt = mx.zeros((1, wan.text_len, wan.text_dim))
model = FastFrames(cfg, dit, refiner, decoder)
return model, prompt
def build_argparser():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--prompt", help="text prompt (required unless --smoke)")
parser.add_argument("--out", default="clip.mp4", help="output mp4 path")
parser.add_argument("--seed", type=int, default=0)
parser.add_argument(
"--latent-frames",
type=int,
default=21,
help="latent frames (21 -> 81 RGB frames). The released model was trained at 21",
)
parser.add_argument("--cache-dir", type=Path, default=None, help="Hugging Face cache directory")
parser.add_argument("--bits", type=int, choices=(4, 8), default=None, help="quantize DiT and umT5 linears")
parser.add_argument("--group-size", type=int, default=64)
parser.add_argument(
"--decoder",
choices=("pixel", "vae"),
default="pixel",
help="pixel is the FastFrames decoder; vae decodes the DiT latents with Wan2.1-VAE",
)
parser.add_argument(
"--keep-text-encoder",
action="store_true",
help="keep umT5 allocated during the rollout (higher memory)",
)
parser.add_argument(
"--smoke",
action="store_true",
help="run a tiny random network (not FastFrames weights) to check the pipeline",
)
return parser
def main(argv=None):
args = build_argparser().parse_args(argv)
if not args.smoke and not args.prompt:
build_argparser().error("--prompt is required unless --smoke")
if args.smoke:
model, prompt_embeds = _smoke_stack()
prompt = args.prompt or "smoke"
frames, report = model.generate(prompt, args.seed, prompt_embeds=prompt_embeds)
else:
from .weights import load_fastframes_s
cfg, dit, refiner, decoder, text, tok_path, vae = load_fastframes_s(
cache_dir=args.cache_dir,
bits=args.bits,
group_size=args.group_size,
load_vae=args.decoder == "vae",
)
if args.latent_frames != cfg.num_frames:
if args.latent_frames < 1:
build_argparser().error("--latent-frames must be >= 1")
cfg = replace(
cfg,
num_frames=args.latent_frames,
num_pixel_frames=4 * args.latent_frames - 3,
)
tokenizer = UMT5Tokenizer(tok_path, seq_len=cfg.wan.text_len)
model = FastFrames(cfg, dit, refiner, decoder, text, tokenizer, vae)
frames, report = model.generate(
args.prompt,
args.seed,
use_vae=args.decoder == "vae",
release_text_encoder=not args.keep_text_encoder,
)
path = save_mp4(args.out, frames, fps=model.config.fps)
print(format_report(report))
print(f"wrote {path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())