"""Generate a clip with FastFrames-S on MLX. python -m fastframes_mlx.generate --prompt "A corgi surfing a wave at sunset" --out clip.mp4 """ from __future__ import annotations import argparse from dataclasses import replace from pathlib import Path import mlx.core as mx from .config import FastFramesConfig, WanConfig from .dit import CausalWanModel from .kv_refiner import KVRefinementNetwork from .pipeline import FastFrames, format_report from .pixel_decoder import PixelDecoder from .tokenizer import UMT5Tokenizer from .video import save_mp4 def _smoke_stack(): """Tiny random weights so the CLI path can be exercised without a download.""" wan = WanConfig(dim=32, ffn_dim=64, num_heads=4, num_layers=2, text_dim=32, freq_dim=16) cfg = FastFramesConfig( name="smoke", wan=wan, num_frames=2, latent_height=4, latent_width=4, num_pixel_frames=5, pixel_height=8, pixel_width=8, refiner_width=16, refiner_stem_blocks=1, refiner_stem_heads=4, refiner_stem_mlp=2, pixel_patch=(4, 4, 4), decoder_low_blocks=1, decoder_high_blocks=1, decoder_ffn=2, ) dit = CausalWanModel(wan) refiner = KVRefinementNetwork( dim=wan.dim, num_heads=wan.num_heads, num_blocks=wan.num_layers, width=cfg.refiner_width, n_stem=cfg.refiner_stem_blocks, stem_heads=cfg.refiner_stem_heads, stem_mlp=cfg.refiner_stem_mlp, ) decoder = PixelDecoder( dim=wan.dim, pixel_patch_size=cfg.pixel_patch, n_low_blocks=cfg.decoder_low_blocks, n_high_blocks=cfg.decoder_high_blocks, ffn_mult=cfg.decoder_ffn, ) # Fixed prompt embedding: the smoke path does not load umT5. prompt = mx.zeros((1, wan.text_len, wan.text_dim)) model = FastFrames(cfg, dit, refiner, decoder) return model, prompt def build_argparser(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--prompt", help="text prompt (required unless --smoke)") parser.add_argument("--out", default="clip.mp4", help="output mp4 path") parser.add_argument("--seed", type=int, default=0) parser.add_argument( "--latent-frames", type=int, default=21, help="latent frames (21 -> 81 RGB frames). The released model was trained at 21", ) parser.add_argument("--cache-dir", type=Path, default=None, help="Hugging Face cache directory") parser.add_argument("--bits", type=int, choices=(4, 8), default=None, help="quantize DiT and umT5 linears") parser.add_argument("--group-size", type=int, default=64) parser.add_argument( "--decoder", choices=("pixel", "vae"), default="pixel", help="pixel is the FastFrames decoder; vae decodes the DiT latents with Wan2.1-VAE", ) parser.add_argument( "--keep-text-encoder", action="store_true", help="keep umT5 allocated during the rollout (higher memory)", ) parser.add_argument( "--smoke", action="store_true", help="run a tiny random network (not FastFrames weights) to check the pipeline", ) return parser def main(argv=None): args = build_argparser().parse_args(argv) if not args.smoke and not args.prompt: build_argparser().error("--prompt is required unless --smoke") if args.smoke: model, prompt_embeds = _smoke_stack() prompt = args.prompt or "smoke" frames, report = model.generate(prompt, args.seed, prompt_embeds=prompt_embeds) else: from .weights import load_fastframes_s cfg, dit, refiner, decoder, text, tok_path, vae = load_fastframes_s( cache_dir=args.cache_dir, bits=args.bits, group_size=args.group_size, load_vae=args.decoder == "vae", ) if args.latent_frames != cfg.num_frames: if args.latent_frames < 1: build_argparser().error("--latent-frames must be >= 1") cfg = replace( cfg, num_frames=args.latent_frames, num_pixel_frames=4 * args.latent_frames - 3, ) tokenizer = UMT5Tokenizer(tok_path, seq_len=cfg.wan.text_len) model = FastFrames(cfg, dit, refiner, decoder, text, tokenizer, vae) frames, report = model.generate( args.prompt, args.seed, use_vae=args.decoder == "vae", release_text_encoder=not args.keep_text_encoder, ) path = save_mp4(args.out, frames, fps=model.config.fps) print(format_report(report)) print(f"wrote {path}") return 0 if __name__ == "__main__": raise SystemExit(main())