Instructions to use pratikgx/FastFrames-S-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/FastFrames-S-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/FastFrames-S-MLX --local-dir FastFrames-S-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Add unofficial MLX port of Apple FastFrames-S (code only; official weights loaded at runtime)
Browse files- .gitattributes +1 -0
- .gitignore +11 -0
- LICENSE +34 -0
- LICENSE_APPLE_AMLR +88 -0
- README.md +121 -0
- SOURCE.txt +1 -0
- docs/port-notes.md +172 -0
- examples/corgi.mp4 +3 -0
- fastframes_mlx/__init__.py +3 -0
- fastframes_mlx/config.py +67 -0
- fastframes_mlx/dit.py +279 -0
- fastframes_mlx/generate.py +140 -0
- fastframes_mlx/kv_refiner.py +192 -0
- fastframes_mlx/ops.py +117 -0
- fastframes_mlx/pipeline.py +237 -0
- fastframes_mlx/pixel_decoder.py +123 -0
- fastframes_mlx/rope.py +89 -0
- fastframes_mlx/scheduler.py +39 -0
- fastframes_mlx/t5.py +152 -0
- fastframes_mlx/tokenizer.py +61 -0
- fastframes_mlx/vae_layers.py +231 -0
- fastframes_mlx/video.py +46 -0
- fastframes_mlx/wan_vae.py +520 -0
- fastframes_mlx/weights.py +227 -0
- pyproject.toml +33 -0
- tests/data/fastframes_s_keys.json +103 -0
- tests/test_parity.py +583 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
examples/corgi.mp4 filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.py[cod]
|
| 3 |
+
.venv/
|
| 4 |
+
third_party/
|
| 5 |
+
outputs/
|
| 6 |
+
*.mp4
|
| 7 |
+
!examples/*.mp4
|
| 8 |
+
.pytest_cache/
|
| 9 |
+
*.egg-info/
|
| 10 |
+
dist/
|
| 11 |
+
build/
|
LICENSE
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 fastframes-mlx contributors
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
| 22 |
+
|
| 23 |
+
-------------------------------------------------------------------------------
|
| 24 |
+
Third-party code
|
| 25 |
+
|
| 26 |
+
fastframes_mlx/vae_layers.py and fastframes_mlx/wan_vae.py are adapted from
|
| 27 |
+
ml-explore/mlx-examples (video/wan2.1), MIT License, Copyright © 2026 Apple Inc.
|
| 28 |
+
That notice is kept at the top of those files.
|
| 29 |
+
|
| 30 |
+
The model math follows the public PyTorch release
|
| 31 |
+
apple-aiml-research/ml-fastframes (Apple Sample Code License for FastFrames
|
| 32 |
+
code; Apache-2.0 for the Wan2.1, Self-Forcing, and Causal Forcing portions).
|
| 33 |
+
This repository reimplements that inference path in MLX. It does not vendor
|
| 34 |
+
the FastFrames PyTorch sources and it does not redistribute any model weights.
|
LICENSE_APPLE_AMLR
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Disclaimer: IMPORTANT: This Apple Machine Learning Research Model is
|
| 2 |
+
specifically developed and released by Apple Inc. ("Apple") for the sole purpose
|
| 3 |
+
of scientific research of artificial intelligence and machine-learning
|
| 4 |
+
technology. “Apple Machine Learning Research Model” means the model, including
|
| 5 |
+
but not limited to algorithms, formulas, trained model weights, parameters,
|
| 6 |
+
configurations, checkpoints, and any related materials (including
|
| 7 |
+
documentation).
|
| 8 |
+
|
| 9 |
+
This Apple Machine Learning Research Model is provided to You by
|
| 10 |
+
Apple in consideration of your agreement to the following terms, and your use,
|
| 11 |
+
modification, creation of Model Derivatives, and or redistribution of the Apple
|
| 12 |
+
Machine Learning Research Model constitutes acceptance of this Agreement. If You
|
| 13 |
+
do not agree with these terms, please do not use, modify, create Model
|
| 14 |
+
Derivatives of, or distribute this Apple Machine Learning Research Model or
|
| 15 |
+
Model Derivatives.
|
| 16 |
+
|
| 17 |
+
* License Scope: In consideration of your agreement to abide by the following
|
| 18 |
+
terms, and subject to these terms, Apple hereby grants you a personal,
|
| 19 |
+
non-exclusive, worldwide, non-transferable, royalty-free, revocable, and
|
| 20 |
+
limited license, to use, copy, modify, distribute, and create Model
|
| 21 |
+
Derivatives (defined below) of the Apple Machine Learning Research Model
|
| 22 |
+
exclusively for Research Purposes. You agree that any Model Derivatives You
|
| 23 |
+
may create or that may be created for You will be limited to Research Purposes
|
| 24 |
+
as well. “Research Purposes” means non-commercial scientific research and
|
| 25 |
+
academic development activities, such as experimentation, analysis, testing
|
| 26 |
+
conducted by You with the sole intent to advance scientific knowledge and
|
| 27 |
+
research. “Research Purposes” does not include any commercial exploitation,
|
| 28 |
+
product development or use in any commercial product or service.
|
| 29 |
+
|
| 30 |
+
* Distribution of Apple Machine Learning Research Model and Model Derivatives:
|
| 31 |
+
If you choose to redistribute Apple Machine Learning Research Model or its
|
| 32 |
+
Model Derivatives, you must provide a copy of this Agreement to such third
|
| 33 |
+
party, and ensure that the following attribution notice be provided: “Apple
|
| 34 |
+
Machine Learning Research Model is licensed under the Apple Machine Learning
|
| 35 |
+
Research Model License Agreement.” Additionally, all Model Derivatives must
|
| 36 |
+
clearly be identified as such, including disclosure of modifications and
|
| 37 |
+
changes made to the Apple Machine Learning Research Model. The name,
|
| 38 |
+
trademarks, service marks or logos of Apple may not be used to endorse or
|
| 39 |
+
promote Model Derivatives or the relationship between You and Apple. “Model
|
| 40 |
+
Derivatives” means any models or any other artifacts created by modifications,
|
| 41 |
+
improvements, adaptations, alterations to the architecture, algorithm or
|
| 42 |
+
training processes of the Apple Machine Learning Research Model, or by any
|
| 43 |
+
retraining, fine-tuning of the Apple Machine Learning Research Model.
|
| 44 |
+
|
| 45 |
+
* No Other License: Except as expressly stated in this notice, no other rights
|
| 46 |
+
or licenses, express or implied, are granted by Apple herein, including but
|
| 47 |
+
not limited to any patent, trademark, and similar intellectual property rights
|
| 48 |
+
worldwide that may be infringed by the Apple Machine Learning Research Model,
|
| 49 |
+
the Model Derivatives or by other works in which the Apple Machine Learning
|
| 50 |
+
Research Model may be incorporated.
|
| 51 |
+
|
| 52 |
+
* Compliance with Laws: Your use of Apple Machine Learning Research Model must
|
| 53 |
+
be in compliance with all applicable laws and regulations.
|
| 54 |
+
|
| 55 |
+
* Term and Termination: The term of this Agreement will begin upon your
|
| 56 |
+
acceptance of this Agreement or use of the Apple Machine Learning Research
|
| 57 |
+
Model and will continue until terminated in accordance with the following
|
| 58 |
+
terms. Apple may terminate this Agreement at any time if You are in breach of
|
| 59 |
+
any term or condition of this Agreement. Upon termination of this Agreement,
|
| 60 |
+
You must cease to use all Apple Machine Learning Research Models and Model
|
| 61 |
+
Derivatives and permanently delete any copy thereof. Sections 3, 6 and 7 will
|
| 62 |
+
survive termination.
|
| 63 |
+
|
| 64 |
+
* Disclaimer and Limitation of Liability: This Apple Machine Learning Research
|
| 65 |
+
Model and any outputs generated by the Apple Machine Learning Research Model
|
| 66 |
+
are provided on an “AS IS” basis. APPLE MAKES NO WARRANTIES, EXPRESS OR
|
| 67 |
+
IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF
|
| 68 |
+
NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE,
|
| 69 |
+
REGARDING THE APPLE MACHINE LEARNING RESEARCH MODEL OR OUTPUTS GENERATED BY
|
| 70 |
+
THE APPLE MACHINE LEARNING RESEARCH MODEL. You are solely responsible for
|
| 71 |
+
determining the appropriateness of using or redistributing the Apple Machine
|
| 72 |
+
Learning Research Model and any outputs of the Apple Machine Learning Research
|
| 73 |
+
Model and assume any risks associated with Your use of the Apple Machine
|
| 74 |
+
Learning Research Model and any output and results. IN NO EVENT SHALL APPLE BE
|
| 75 |
+
LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
| 76 |
+
IN ANY WAY OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION OF
|
| 77 |
+
THE APPLE MACHINE LEARNING RESEARCH MODEL AND ANY OUTPUTS OF THE APPLE MACHINE
|
| 78 |
+
LEARNING RESEARCH MODEL, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
|
| 79 |
+
TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS
|
| 80 |
+
BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
| 81 |
+
|
| 82 |
+
* Governing Law: This Agreement will be governed by and construed under the laws
|
| 83 |
+
of the State of California without regard to its choice of law principles. The
|
| 84 |
+
Convention on Contracts for the International Sale of Goods shall not apply to
|
| 85 |
+
the Agreement except that the arbitration clause and any arbitration hereunder
|
| 86 |
+
shall be governed by the Federal Arbitration Act, Chapters 1 and 2.
|
| 87 |
+
|
| 88 |
+
Copyright (C) 2026 Apple Inc. All Rights Reserved.
|
README.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
library_name: mlx
|
| 4 |
+
pipeline_tag: text-to-video
|
| 5 |
+
tags:
|
| 6 |
+
- mlx
|
| 7 |
+
- apple-silicon
|
| 8 |
+
- text-to-video
|
| 9 |
+
base_model:
|
| 10 |
+
- apple/FastFrames-S
|
| 11 |
+
- Wan-AI/Wan2.1-T2V-1.3B
|
| 12 |
+
- zhuhz22/Causal-Forcing
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# FastFrames-S-MLX
|
| 16 |
+
|
| 17 |
+
Unofficial [MLX](https://github.com/ml-explore/mlx) port of Apple's [FastFrames-S](https://huggingface.co/apple/FastFrames-S) (1.3B), a two-step autoregressive text-to-video model. It generates **832×480** video, **81 frames** at **16 fps** (about 5 seconds).
|
| 18 |
+
|
| 19 |
+
This repository contains the `fastframes_mlx` inference package and an example clip. **It does not re-host model weights.** At runtime the generate script downloads the official checkpoints (see below), converts them in memory to MLX, and runs on Apple Silicon.
|
| 20 |
+
|
| 21 |
+
A full converted MLX weight dump is not included: the stack is large (the umT5 encoder alone is about 11 GB), conversion is in-memory at load time rather than a saved MLX checkpoint format, and producing that dump was not practical on the machine used to publish this repo. The code loads the official weights directly.
|
| 22 |
+
|
| 23 |
+
Example clip, prompt `"A corgi surfing a wave at sunset"`, seed 0, generated on a 64GB M5 Pro Mac mini:
|
| 24 |
+
|
| 25 |
+
<video src="examples/corgi.mp4" controls></video>
|
| 26 |
+
|
| 27 |
+
If the player does not show, download [examples/corgi.mp4](examples/corgi.mp4).
|
| 28 |
+
|
| 29 |
+
## Credit
|
| 30 |
+
|
| 31 |
+
This is an unofficial port. All credit for FastFrames goes to Apple and the authors: Gene Chou, Guandao Yang, Fujun Luan, Lars Mescheder, Wei Dong, Stephan R. Richter, and Vladlen Koltun.
|
| 32 |
+
|
| 33 |
+
- Weights: [apple/FastFrames-S](https://huggingface.co/apple/FastFrames-S)
|
| 34 |
+
- Official code (CUDA): [apple-aiml-research/ml-fastframes](https://github.com/apple-aiml-research/ml-fastframes)
|
| 35 |
+
- Paper: [FastFrames: Video Generation at Interactive Latency](https://arxiv.org/abs/2611.23226) (arXiv:2611.23226)
|
| 36 |
+
- Project page: https://apple-aiml-research.github.io/ml-fastframes/
|
| 37 |
+
|
| 38 |
+
Apple Machine Learning Research Model is licensed under the Apple Machine Learning Research Model License Agreement.
|
| 39 |
+
|
| 40 |
+
The name, trademarks, service marks, or logos of Apple may not be used to endorse or promote this port. "Apple" here is attribution only.
|
| 41 |
+
|
| 42 |
+
The port also depends on:
|
| 43 |
+
|
| 44 |
+
- [Wan2.1](https://github.com/Wan-Video/Wan2.1) / [Wan-AI/Wan2.1-T2V-1.3B](https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B) (Apache-2.0) for the DiT, umT5 encoder, tokenizer, and optional VAE
|
| 45 |
+
- [Causal Forcing++](https://github.com/thu-ml/Causal-Forcing) / [zhuhz22/Causal-Forcing](https://huggingface.co/zhuhz22/Causal-Forcing) (Apache-2.0) for the frozen frame-wise 2-step generator
|
| 46 |
+
- [Self-Forcing](https://github.com/guandeh17/Self-Forcing) (Apache-2.0), which the official FastFrames code also builds on
|
| 47 |
+
- [ml-explore/mlx-examples](https://github.com/ml-explore/mlx-examples) `video/wan2.1` (MIT, Copyright © 2026 Apple Inc., [PR #1409](https://github.com/ml-explore/mlx-examples/pull/1409)) for the VAE layers in `vae_layers.py` and `wan_vae.py`
|
| 48 |
+
|
| 49 |
+
## License
|
| 50 |
+
|
| 51 |
+
Two licenses apply, and they are not the same.
|
| 52 |
+
|
| 53 |
+
| Piece | License | Where |
|
| 54 |
+
| --- | --- | --- |
|
| 55 |
+
| This Python package (`fastframes_mlx`) | MIT | [`LICENSE`](LICENSE) |
|
| 56 |
+
| FastFrames-S KV refiner and pixel decoder weights, loaded from [`apple/FastFrames-S`](https://huggingface.co/apple/FastFrames-S) | [Apple Machine Learning Research Model License](LICENSE_APPLE_AMLR) (`apple-amlr`) | [`LICENSE_APPLE_AMLR`](LICENSE_APPLE_AMLR) |
|
| 57 |
+
| Official FastFrames Python/CUDA code | [Apple Sample Code License](https://github.com/apple-aiml-research/ml-fastframes/blob/main/LICENSE) | not vendored here; this is a reimplementation |
|
| 58 |
+
| Generator (`generator_ema` in `zhuhz22/Causal-Forcing`) | Apache-2.0 | upstream repo |
|
| 59 |
+
| umT5-XXL encoder, tokenizer, optional Wan VAE (`Wan-AI/Wan2.1-T2V-1.3B`) | Apache-2.0 | upstream repo |
|
| 60 |
+
|
| 61 |
+
The Apple Machine Learning Research Model License allows use, modification, creation of model derivatives, and redistribution **only for research purposes** (non-commercial scientific research and academic development). Redistribution of those weights or derivatives must include a copy of that agreement, the attribution notice above, and a clear statement of modifications. This port does not redistribute those weights; if you use them, you must accept that license.
|
| 62 |
+
|
| 63 |
+
This repository identifies the port as an unofficial MLX reimplementation of the FastFrames-S inference path. Changes relative to the official CUDA release include: MLX kernels instead of PyTorch/CUDA, in-memory layout remapping of checkpoint keys and 3D conv weights, float64 tables computed in NumPy and stored as float32 so they can run on Metal, and optional post-training quantization of `nn.Linear` layers (`--bits`, untested).
|
| 64 |
+
|
| 65 |
+
## Install and run
|
| 66 |
+
|
| 67 |
+
Needs an Apple Silicon Mac, Python 3.12, and ffmpeg.
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
brew install ffmpeg
|
| 71 |
+
uv venv --python 3.12
|
| 72 |
+
source .venv/bin/activate
|
| 73 |
+
uv pip install -e .
|
| 74 |
+
python -m fastframes_mlx.generate --prompt "A corgi surfing a wave at sunset" --out clip.mp4 --seed 0
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
The first run downloads official checkpoints into the Hugging Face cache, roughly:
|
| 78 |
+
|
| 79 |
+
- `apple/FastFrames-S` `fastframes-s/*` (refiner + decoder, about 1 GB)
|
| 80 |
+
- `zhuhz22/Causal-Forcing` `causal-forcing++/framewise-2step.pt` (the 1.3B generator)
|
| 81 |
+
- `Wan-AI/Wan2.1-T2V-1.3B` `models_t5_umt5-xxl-enc-bf16.pth` (about 11 GB) and `google/umt5-xxl/tokenizer.json`
|
| 82 |
+
|
| 83 |
+
If a source repo requires login, run `hf auth login` first.
|
| 84 |
+
|
| 85 |
+
Useful flags: `--latent-frames` (default 21 → 81 RGB frames), `--bits 4` or `--bits 8` (untested), `--decoder vae` to decode with Wan2.1-VAE instead of the FastFrames pixel decoder, `--smoke` for a tiny random network that does not download weights.
|
| 86 |
+
|
| 87 |
+
## Measured result (64GB M5 Pro Mac mini)
|
| 88 |
+
|
| 89 |
+
One run, seed 0, official prompt `"A corgi surfing a wave at sunset"`, FastFrames-S, 832×480, 81 frames at 16 fps. `--bits` was not used. A background inference app was paused for this run.
|
| 90 |
+
|
| 91 |
+
| | |
|
| 92 |
+
| --- | --- |
|
| 93 |
+
| Generation | 30.4 s (30.43 s measured) |
|
| 94 |
+
| Per output frame | 375.7 ms (~2.7 fps, about 6× off real time) |
|
| 95 |
+
| MLX peak memory | 22.93 GB |
|
| 96 |
+
| RSS peak | 30.37 GB |
|
| 97 |
+
| Wall clock | 35.9 s (includes load) |
|
| 98 |
+
| Text encoder | 871 ms |
|
| 99 |
+
| Decoder | 1,116 ms |
|
| 100 |
+
|
| 101 |
+
Full logs, the speed table, and the clip are in the benchmark dataset: [pratikgx/fastframes-s-m5-pro-mac-mini-bench](https://huggingface.co/datasets/pratikgx/fastframes-s-m5-pro-mac-mini-bench).
|
| 102 |
+
|
| 103 |
+
## Caveats
|
| 104 |
+
|
| 105 |
+
- Verified by 13 CPU parity tests against the official PyTorch reference, plus one Mac mini run.
|
| 106 |
+
- No side-by-side output comparison versus the official CUDA implementation yet.
|
| 107 |
+
- `--bits` quantization is untested.
|
| 108 |
+
- Single run, single seed.
|
| 109 |
+
- FastFrames-L (14B), the LingBot camera demo, and training are not implemented.
|
| 110 |
+
|
| 111 |
+
More detail on what the CPU parity tests did and did not cover is in [`docs/port-notes.md`](docs/port-notes.md). The package in this repo is commit `99755c01bf645fce1ae95ef564104ce3fea666cf` of the port (the revision used for the Mac mini run).
|
| 112 |
+
|
| 113 |
+
```bibtex
|
| 114 |
+
@article{chou2026fastframes,
|
| 115 |
+
title = {FastFrames: Video Generation at Interactive Latency},
|
| 116 |
+
author = {Chou, Gene and Yang, Guandao and Luan, Fujun and Mescheder, Lars and
|
| 117 |
+
Dong, Wei and Richter, Stephan R. and Koltun, Vladlen},
|
| 118 |
+
journal = {arXiv preprint arXiv:2611.23226},
|
| 119 |
+
year = {2026}
|
| 120 |
+
}
|
| 121 |
+
```
|
SOURCE.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
fastframes_mlx package taken from commit 99755c01bf645fce1ae95ef564104ce3fea666cf
|
docs/port-notes.md
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> Original package README from the MLX port, written while developing and testing on a Linux CPU box, before the 64GB M5 Pro Mac mini run documented on the [model card](../README.md). Kept here for the detailed CPU parity notes.
|
| 2 |
+
|
| 3 |
+
# fastframes-mlx
|
| 4 |
+
|
| 5 |
+
MLX inference for [Apple FastFrames-S](https://huggingface.co/apple/FastFrames-S), the 1.3B two-step autoregressive video model from *FastFrames: Video Generation at Interactive Latency* (Chou et al., 2026). The target machine is a 64 GB Apple Silicon Mac. This tree was developed and tested on Linux with the MLX CPU build, not on an M5.
|
| 6 |
+
|
| 7 |
+
FastFrames-S keeps the frozen [Causal Forcing++](https://github.com/thu-ml/Causal-Forcing) frame-wise 2-step Wan2.1-1.3B generator, then replaces the clean-cache DiT pass with a KV refiner and replaces the Wan VAE decoder with a causal ConvNeXt pixel decoder. A clip is 21 latent frames, decoded to **81 RGB frames at 832×480, 16 fps**.
|
| 8 |
+
|
| 9 |
+
Weights are downloaded from Hugging Face at runtime and converted to MLX. They are not stored in this repo.
|
| 10 |
+
|
| 11 |
+
## Licenses
|
| 12 |
+
|
| 13 |
+
Read these before you run generation. Using the weights means accepting the weight licenses.
|
| 14 |
+
|
| 15 |
+
| Piece | Where it loads from | License |
|
| 16 |
+
|---|---|---|
|
| 17 |
+
| KV refiner, pixel decoder | [`apple/FastFrames-S`](https://huggingface.co/apple/FastFrames-S) | [apple-amlr](https://huggingface.co/apple/FastFrames-S) (Apple Machine Learning Research) |
|
| 18 |
+
| Generator (`generator_ema`) | [`zhuhz22/Causal-Forcing`](https://huggingface.co/zhuhz22/Causal-Forcing) `causal-forcing++/framewise-2step.pt` | Apache-2.0 |
|
| 19 |
+
| umT5-XXL encoder, tokenizer, VAE | [`Wan-AI/Wan2.1-T2V-1.3B`](https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B) | Apache-2.0 |
|
| 20 |
+
|
| 21 |
+
This Python package is MIT. See `LICENSE`. Do not re-upload the checkpoints.
|
| 22 |
+
|
| 23 |
+
## Credits
|
| 24 |
+
|
| 25 |
+
- Apple FastFrames: [code](https://github.com/apple-aiml-research/ml-fastframes), [FastFrames-S weights](https://huggingface.co/apple/FastFrames-S), [FastFrames-L weights](https://huggingface.co/apple/FastFrames-L). Authors: Gene Chou, Guandao Yang, Fujun Luan, Lars Mescheder, Wei Dong, Stephan R. Richter, Vladlen Koltun.
|
| 26 |
+
- Wan2.1 (Apache-2.0): [Wan-Video/Wan2.1](https://github.com/Wan-Video/Wan2.1). The DiT, umT5 encoder, and VAE follow that architecture.
|
| 27 |
+
- Causal Forcing and Causal Forcing++ (Apache-2.0): [thu-ml/Causal-Forcing](https://github.com/thu-ml/Causal-Forcing). FastFrames-S's generator is their frame-wise 2-step checkpoint.
|
| 28 |
+
- Self-Forcing (Apache-2.0): [guandeh17/Self-Forcing](https://github.com/guandeh17/Self-Forcing), which the official FastFrames code also builds on.
|
| 29 |
+
- MLX Wan2.1 VAE: `vae_layers.py` and `wan_vae.py` are adapted from [ml-explore/mlx-examples](https://github.com/ml-explore/mlx-examples) `video/wan2.1` (MIT, Copyright © 2026 Apple Inc., PR [#1409](https://github.com/ml-explore/mlx-examples/pull/1409)). The DiT, refiner, pixel decoder, and umT5 encoder here are a separate MLX reimplementation checked against the official PyTorch modules.
|
| 30 |
+
|
| 31 |
+
## What was checked, and what was not
|
| 32 |
+
|
| 33 |
+
Checked on this Linux CPU box (MLX 0.32.4, PyTorch 2.15.0+cpu), with tiny tensors unless noted:
|
| 34 |
+
|
| 35 |
+
- RoPE matches the official complex128 op (max abs error under `1e-6`).
|
| 36 |
+
- Flow-matching sigmas match. The snapped timestep index matches for 1000, 937.5, 833.3333129882812, 625, and 0.
|
| 37 |
+
- A 2-layer causal DiT `forward_frame`, including the second frame reading the first frame's cache, matches PyTorch math SDPA within `2e-2` / `3e-2` (both sides cast attention to bf16, which is what the release does).
|
| 38 |
+
- KV refiner stem and repair match within `2e-4` in fp32.
|
| 39 |
+
- Depthwise causal conv matches within `1e-5`. The pixel decoder, with a non-zero output projection so the zero-init head cannot hide the conv stack, matches within `2e-4`.
|
| 40 |
+
- A 2-layer umT5 encoder matches within `2e-4`.
|
| 41 |
+
- Prompt whitespace cleaning matches `fastframes.wan.tokenizers`.
|
| 42 |
+
- A 2-frame bf16 rollout through the same recipe as `FastFrames.rollout` matches within **0.015625** (latents), **0.03125** (pre-head features), and **3e-5** (time modulation). Those are the numbers measured here.
|
| 43 |
+
- The VAE's causal conv3d matches the official Wan op within `1e-5` on one random tensor.
|
| 44 |
+
- The 66 KV-refiner keys and 31 pixel-decoder keys in the released safetensors headers match this package's parameter names.
|
| 45 |
+
- `python -m fastframes_mlx.generate --smoke` writes an mp4. That network is random and tiny. It is not FastFrames.
|
| 46 |
+
|
| 47 |
+
Not checked:
|
| 48 |
+
|
| 49 |
+
- No Apple Silicon and no Metal run. Smoke tests on CPU only checked that the CLI prints a latency line and an allocator peak, not that those numbers mean anything for a 1.3B model.
|
| 50 |
+
- The real 1.3B generator, the 11 GB text encoder, and an 81-frame 832×480 clip were **not** executed. There is no end-to-end video from the released weights.
|
| 51 |
+
- The generator `.pt` key set was not diffed against the module. Only the tiny `CausalWanModel` state dict and the refiner/decoder headers were.
|
| 52 |
+
- `transformers.AutoTokenizer` was not installed, so token ids were not compared to it. Cleaning was. `tokenizer.json` from the Wan repo appends `</s>` via its post-processor, and this code pads to 512 with `<pad>`.
|
| 53 |
+
|
| 54 |
+
## Setup on macOS
|
| 55 |
+
|
| 56 |
+
Needs an Apple Silicon Mac, Python 3.10+, and `ffmpeg` (`brew install ffmpeg`). `uv` or `pip` both work.
|
| 57 |
+
|
| 58 |
+
```bash
|
| 59 |
+
brew install ffmpeg
|
| 60 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
| 61 |
+
cd /path/to/this/repo
|
| 62 |
+
uv venv --python 3.12
|
| 63 |
+
source .venv/bin/activate
|
| 64 |
+
uv pip install -e .
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
pip equivalent:
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
python3 -m venv .venv
|
| 71 |
+
source .venv/bin/activate
|
| 72 |
+
pip install -e .
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
On Linux the same install pulls `mlx[cpu]`. Parity tests also need `einops`:
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
pip install -e ".[parity]"
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
Hugging Face will download, into the HF cache, roughly:
|
| 82 |
+
|
| 83 |
+
- `apple/FastFrames-S` `fastframes-s/*` (refiner + decoder, about 1 GB)
|
| 84 |
+
- `zhuhz22/Causal-Forcing` `causal-forcing++/framewise-2step.pt` (the 1.3B generator)
|
| 85 |
+
- `Wan-AI/Wan2.1-T2V-1.3B` `models_t5_umt5-xxl-enc-bf16.pth` (about 11 GB) and `google/umt5-xxl/tokenizer.json`
|
| 86 |
+
- `Wan2.1_VAE.pth` only if you pass `--decoder vae`
|
| 87 |
+
|
| 88 |
+
Log in first if a repo asks for a token: `hf auth login` (or `huggingface-cli login`).
|
| 89 |
+
|
| 90 |
+
## Generate on a 64 GB Apple Silicon Mac
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
python -m fastframes_mlx.generate \
|
| 94 |
+
--prompt "A corgi surfing a wave at sunset" \
|
| 95 |
+
--out clip.mp4 \
|
| 96 |
+
--seed 0
|
| 97 |
+
```
|
| 98 |
+
|
| 99 |
+
That is the released recipe: 21 latent frames, pixel decoder, bf16 DiT and refiner, fp32 pixel decoder, umT5 freed after the prompt. The process prints, from **this** run only:
|
| 100 |
+
|
| 101 |
+
- text-encoder time
|
| 102 |
+
- one line per latent frame (4 DiT steps plus a clean `t = 0` pass on frame 0; 2 DiT steps plus the KV refiner after that)
|
| 103 |
+
- pixel-decoder time
|
| 104 |
+
- total wall clock and the total divided by 81 output frames
|
| 105 |
+
- MLX allocator peak after the text encoder is released
|
| 106 |
+
- process RSS high-water mark for the whole command (this still includes the text encoder, because RSS does not go backwards)
|
| 107 |
+
|
| 108 |
+
Nothing in this README is a speed number. Those lines are whatever the machine measures.
|
| 109 |
+
|
| 110 |
+
Useful variants:
|
| 111 |
+
|
| 112 |
+
```bash
|
| 113 |
+
# one latent frame, to see whether the weights load, before a full clip
|
| 114 |
+
python -m fastframes_mlx.generate \
|
| 115 |
+
--prompt "A corgi surfing a wave at sunset" \
|
| 116 |
+
--latent-frames 1 --out one.mp4
|
| 117 |
+
|
| 118 |
+
# uncalibrated 4-bit or 8-bit linears in the DiT and umT5 only
|
| 119 |
+
python -m fastframes_mlx.generate \
|
| 120 |
+
--prompt "A corgi surfing a wave at sunset" \
|
| 121 |
+
--bits 4 --out clip_q4.mp4
|
| 122 |
+
|
| 123 |
+
# decode the DiT latents with the Wan2.1 VAE instead of the FastFrames pixel decoder
|
| 124 |
+
python -m fastframes_mlx.generate \
|
| 125 |
+
--prompt "A corgi surfing a wave at sunset" \
|
| 126 |
+
--decoder vae --out clip_vae.mp4
|
| 127 |
+
|
| 128 |
+
# environment check with random tiny weights, no download
|
| 129 |
+
python -m fastframes_mlx.generate --smoke --out outputs/smoke.mp4
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
`--seed` is a NumPy PCG64 stream in this process. It will not reproduce the CUDA Philox video from `scripts/generate.py` in the official repo.
|
| 133 |
+
|
| 134 |
+
## Memory estimate (not measured)
|
| 135 |
+
|
| 136 |
+
Counted from the architectures, not from an allocator on a Mac.
|
| 137 |
+
|
| 138 |
+
| Resident piece | Parameters | Dtype | Size |
|
| 139 |
+
|---|---:|---|---:|
|
| 140 |
+
| umT5-XXL encoder | 5,680,910,336 | bf16 | 11.36 GB |
|
| 141 |
+
| Causal DiT (Wan2.1-1.3B) | 1,418,996,800 | bf16 | 2.84 GB |
|
| 142 |
+
| KV refiner | 346,205,952 | bf16 | 0.69 GB |
|
| 143 |
+
| Pixel decoder | 63,873,024 | fp32 | 0.26 GB |
|
| 144 |
+
| KV cache, 30 layers × K and V × 21 × 1560 tokens × 12 × 128 | | bf16 | 6.04 GB |
|
| 145 |
+
| Wan VAE, only with `--decoder vae` | 126,892,531 | bf16 or fp32 | 0.25 GB or 0.51 GB |
|
| 146 |
+
|
| 147 |
+
Weights plus cache, text encoder included: **21.2 GB**, plus the VAE if you asked for it. After the text encoder is released (the default): **9.8 GB** of those tensors, plus activations.
|
| 148 |
+
|
| 149 |
+
A single attention score matrix for the last frame, if it were fully materialized, would be `12 × 1560 × 32760 × 2` bytes ≈ **1.23 GB**. MLX SDPA may not form that matrix. FFN activations at 1560 × 8960 are tens of MB. The computation graph is eval'd every block so it does not keep all 30 layers live.
|
| 150 |
+
|
| 151 |
+
**Estimate for a 64 GB Mac:** on the order of **15–25 GB** during the rollout with the text encoder released, and a process high-water mark around **25–40 GB** while umT5 is alive, including allocator cache and the download/convert spike (the `.pt` and `.pth` loaders briefly hold a Torch copy). 4-bit linears shrink the DiT and umT5 weights by roughly 4×. They do not shrink the 6.04 GB KV cache, the refiner's grouped GEMMs, or the pixel decoder. This estimate was not checked with `mx.get_peak_memory()` on Metal.
|
| 152 |
+
|
| 153 |
+
## Parity tests
|
| 154 |
+
|
| 155 |
+
```bash
|
| 156 |
+
git clone --depth 1 https://github.com/apple-aiml-research/ml-fastframes.git third_party/ml-fastframes
|
| 157 |
+
FASTFRAMES_SRC=$PWD/third_party/ml-fastframes python -m unittest tests.test_parity -v
|
| 158 |
+
```
|
| 159 |
+
|
| 160 |
+
If `FASTFRAMES_SRC` is unset, the tests use `/tmp/ml-fastframes` when that checkout exists, otherwise they clone into `third_party/ml-fastframes`.
|
| 161 |
+
|
| 162 |
+
The official `attention()` requests cuDNN SDPA. That kernel is absent on CPU, so the tests patch it to PyTorch's math SDPA (`backend="auto"`) and compare with MLX SDPA. Both still cast Q/K/V to bf16. RoPE frequencies and the timestep sinusoid are computed in NumPy float64 and stored as float32; Metal has no float64. The DiT casts the sinusoid to the activation dtype immediately.
|
| 163 |
+
|
| 164 |
+
## Known gaps
|
| 165 |
+
|
| 166 |
+
- FastFrames-L (14B), the LingBot camera demo, and training are not implemented. `WanConfig` has the 14B widths, and that is all.
|
| 167 |
+
- FastFrames-S sets `fast_ops=False`. The fused-QKV / fp32-RoPE path used by FastFrames-L is not implemented.
|
| 168 |
+
- The pixel decoder runs in fp32, which is how the weights are stored. The official CUDA script also enters a bf16 autocast around `decode`. On CPU that autocast turns itself off. This port does not try to match CUDA autocast op by op.
|
| 169 |
+
- Quantization is MLX post-training quantization of `nn.Linear` only (`--bits 4` or `8`, group size 64). Layers whose input size is not divisible by the group size are left alone. The refiner's grouped GEMM parameters, convolutions, the token embedding, and the pixel decoder stay full precision. No image-quality check was done.
|
| 170 |
+
- The VAE follows the mlx-examples Wan2.1 port (encoder head emits `2 * z_dim` channels, then a 1×1 conv splits mean and log-variance). A full `WanVAE_.decode` was not numerically compared to the official module on real weights. Only causal conv3d was.
|
| 171 |
+
- KV cache updates allocate new arrays. They are not CUDA-style in-place copies.
|
| 172 |
+
- Generation length defaults to 21 latent frames. `--latent-frames` changes that and keeps the last `4N - 3` RGB frames. The model was trained at N = 21.
|
examples/corgi.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:97505c3954786ac284a62a780e9383486dfd42adb7cd24f9085076abd30ae4c9
|
| 3 |
+
size 693212
|
fastframes_mlx/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MLX port of Apple FastFrames-S (1.3B autoregressive video diffusion)."""
|
| 2 |
+
|
| 3 |
+
__version__ = "0.1.0"
|
fastframes_mlx/config.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Architectures and the FastFrames-S sampling recipe.
|
| 2 |
+
|
| 3 |
+
Numbers are taken from the official ``FF_S`` config and the FastFrames-S
|
| 4 |
+
training configs (shift-5 timesteps, 21 latent frames, 832x480, 16 fps).
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass(frozen=True)
|
| 13 |
+
class WanConfig:
|
| 14 |
+
dim: int = 1536
|
| 15 |
+
ffn_dim: int = 8960
|
| 16 |
+
num_heads: int = 12
|
| 17 |
+
num_layers: int = 30
|
| 18 |
+
patch_size: tuple = (1, 2, 2)
|
| 19 |
+
text_len: int = 512
|
| 20 |
+
in_dim: int = 16
|
| 21 |
+
freq_dim: int = 256
|
| 22 |
+
text_dim: int = 4096
|
| 23 |
+
out_dim: int = 16
|
| 24 |
+
eps: float = 1e-6
|
| 25 |
+
|
| 26 |
+
@property
|
| 27 |
+
def head_dim(self) -> int:
|
| 28 |
+
return self.dim // self.num_heads
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
WAN_1_3B = WanConfig()
|
| 32 |
+
WAN_14B = WanConfig(dim=5120, ffn_dim=13824, num_heads=40, num_layers=40)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@dataclass(frozen=True)
|
| 36 |
+
class FastFramesConfig:
|
| 37 |
+
"""FastFrames-S inference recipe (``fast_ops`` off, fp32 pixel decoder)."""
|
| 38 |
+
|
| 39 |
+
name: str = "ff-s"
|
| 40 |
+
wan: WanConfig = WAN_1_3B
|
| 41 |
+
num_frames: int = 21
|
| 42 |
+
latent_height: int = 60
|
| 43 |
+
latent_width: int = 104
|
| 44 |
+
num_pixel_frames: int = 81
|
| 45 |
+
pixel_height: int = 480
|
| 46 |
+
pixel_width: int = 832
|
| 47 |
+
fps: int = 16
|
| 48 |
+
first_frame_timesteps: tuple = (1000.0, 937.5, 833.3333129882812, 625.0)
|
| 49 |
+
timesteps: tuple = (1000.0, 833.3333129882812)
|
| 50 |
+
refine_timestep: float = 833.3333129882812
|
| 51 |
+
shift: float = 5.0
|
| 52 |
+
# Refiner (released FastFrames-S checkpoint).
|
| 53 |
+
refiner_width: int = 768
|
| 54 |
+
refiner_stem_blocks: int = 4
|
| 55 |
+
refiner_stem_heads: int = 8
|
| 56 |
+
refiner_stem_mlp: int = 4
|
| 57 |
+
refiner_head_mlp: int = 2
|
| 58 |
+
# Pixel decoder.
|
| 59 |
+
pixel_patch: tuple = (4, 16, 16)
|
| 60 |
+
decoder_low_blocks: int = 2
|
| 61 |
+
decoder_high_blocks: int = 1
|
| 62 |
+
decoder_ffn: int = 4
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def frame_tokens(cfg: FastFramesConfig) -> int:
|
| 66 |
+
ph, pw = cfg.wan.patch_size[1], cfg.wan.patch_size[2]
|
| 67 |
+
return (cfg.latent_height // ph) * (cfg.latent_width // pw)
|
fastframes_mlx/dit.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frame-wise causal Wan2.1 DiT used as the FastFrames generator.
|
| 2 |
+
|
| 3 |
+
This is the inference path in ``CausalWanModel.forward_frame``: one latent
|
| 4 |
+
frame attends to its own tokens and to the KV cache of earlier frames. The
|
| 5 |
+
pre-head hidden state is what the pixel decoder reads. RoPE follows the
|
| 6 |
+
complex reference (FastFrames-S sets ``fast_ops=False``) but the multiply is
|
| 7 |
+
float32 so it can run on the GPU.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import math
|
| 13 |
+
|
| 14 |
+
import mlx.core as mx
|
| 15 |
+
import mlx.nn as nn
|
| 16 |
+
|
| 17 |
+
from .config import WanConfig
|
| 18 |
+
from .ops import gelu_tanh, layer_norm, rms_norm, sdpa, silu, sinusoidal_embedding_1d
|
| 19 |
+
from .rope import RopeTables
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _zeros(shape):
|
| 23 |
+
return mx.zeros(shape)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class _RMS(nn.Module):
|
| 27 |
+
def __init__(self, dim, eps):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.eps = eps
|
| 30 |
+
self.weight = mx.ones((dim,))
|
| 31 |
+
|
| 32 |
+
def __call__(self, x):
|
| 33 |
+
return rms_norm(x, self.weight, self.eps)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class _AffineLN(nn.Module):
|
| 37 |
+
def __init__(self, dim, eps):
|
| 38 |
+
super().__init__()
|
| 39 |
+
self.eps = eps
|
| 40 |
+
self.weight = mx.ones((dim,))
|
| 41 |
+
self.bias = mx.zeros((dim,))
|
| 42 |
+
|
| 43 |
+
def __call__(self, x):
|
| 44 |
+
return layer_norm(x, self.eps, self.weight, self.bias)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class SelfAttention(nn.Module):
|
| 48 |
+
def __init__(self, dim, num_heads, eps):
|
| 49 |
+
super().__init__()
|
| 50 |
+
self.num_heads = num_heads
|
| 51 |
+
self.head_dim = dim // num_heads
|
| 52 |
+
self.q = nn.Linear(dim, dim)
|
| 53 |
+
self.k = nn.Linear(dim, dim)
|
| 54 |
+
self.v = nn.Linear(dim, dim)
|
| 55 |
+
self.o = nn.Linear(dim, dim)
|
| 56 |
+
self.norm_q = _RMS(dim, eps)
|
| 57 |
+
self.norm_k = _RMS(dim, eps)
|
| 58 |
+
|
| 59 |
+
def qkv(self, x):
|
| 60 |
+
b, s, _ = x.shape
|
| 61 |
+
q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim)
|
| 62 |
+
k = self.norm_k(self.k(x)).reshape(b, s, self.num_heads, self.head_dim)
|
| 63 |
+
v = self.v(x).reshape(b, s, self.num_heads, self.head_dim)
|
| 64 |
+
return q, k, v
|
| 65 |
+
|
| 66 |
+
def text_kv(self, context):
|
| 67 |
+
b = context.shape[0]
|
| 68 |
+
k = self.norm_k(self.k(context)).reshape(b, -1, self.num_heads, self.head_dim)
|
| 69 |
+
v = self.v(context).reshape(b, -1, self.num_heads, self.head_dim)
|
| 70 |
+
return k, v
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class CrossAttention(SelfAttention):
|
| 74 |
+
def __call__(self, x, cache):
|
| 75 |
+
b, s, _ = x.shape
|
| 76 |
+
q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim)
|
| 77 |
+
k, v = cache
|
| 78 |
+
out = sdpa(q, k, v).reshape(b, s, -1)
|
| 79 |
+
return self.o(out)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class FeedForward(nn.Module):
|
| 83 |
+
def __init__(self, dim, ffn_dim):
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.fc1 = nn.Linear(dim, ffn_dim)
|
| 86 |
+
self.fc2 = nn.Linear(ffn_dim, dim)
|
| 87 |
+
|
| 88 |
+
def __call__(self, x):
|
| 89 |
+
return self.fc2(gelu_tanh(self.fc1(x)))
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
class CausalBlock(nn.Module):
|
| 93 |
+
def __init__(self, dim, ffn_dim, num_heads, eps):
|
| 94 |
+
super().__init__()
|
| 95 |
+
self.eps = eps
|
| 96 |
+
self.self_attn = SelfAttention(dim, num_heads, eps)
|
| 97 |
+
self.cross_attn = CrossAttention(dim, num_heads, eps)
|
| 98 |
+
self.norm3 = _AffineLN(dim, eps)
|
| 99 |
+
self.ffn = FeedForward(dim, ffn_dim)
|
| 100 |
+
self.modulation = _zeros((1, 6, dim))
|
| 101 |
+
|
| 102 |
+
def forward_frame(self, x, e, grid, rope: RopeTables, cross_kv, kv_cache, frame: int):
|
| 103 |
+
"""``x`` is ``[B, tokens, dim]``. ``e`` is ``[B, 1, 6, dim]``."""
|
| 104 |
+
tokens = x.shape[1]
|
| 105 |
+
frames, height, width = grid
|
| 106 |
+
mod = self.modulation.reshape(1, 1, 6, -1) + e
|
| 107 |
+
parts = mx.split(mod, 6, axis=2)
|
| 108 |
+
e0, e1, e2, e3, e4, e5 = [p.reshape(p.shape[0], p.shape[1], 1, p.shape[-1]) for p in parts]
|
| 109 |
+
h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1)
|
| 110 |
+
h = (h * (1.0 + e1) + e0).reshape(x.shape[0], tokens, -1)
|
| 111 |
+
q, k, v = self.self_attn.qkv(h)
|
| 112 |
+
q = rope.apply(q, frames, height, width, start_frame=frame)
|
| 113 |
+
k = rope.apply(k, frames, height, width, start_frame=frame)
|
| 114 |
+
kv_cache.write(frame, k, v)
|
| 115 |
+
k_pre, v_pre = kv_cache.prefix(frame)
|
| 116 |
+
y = self.self_attn.o(sdpa(q, k_pre, v_pre).reshape(x.shape[0], tokens, -1))
|
| 117 |
+
x = x + (y.reshape(x.shape[0], 1, tokens, -1) * e2).reshape(x.shape[0], tokens, -1)
|
| 118 |
+
x = x + self.cross_attn(self.norm3(x), cross_kv)
|
| 119 |
+
y = self.ffn((layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) * (1.0 + e4) + e3).reshape(x.shape[0], tokens, -1))
|
| 120 |
+
y = (y.reshape(x.shape[0], 1, tokens, -1) * e5).reshape(x.shape[0], tokens, -1)
|
| 121 |
+
return x + y
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
class CausalHead(nn.Module):
|
| 125 |
+
def __init__(self, dim, out_dim, patch_size, eps):
|
| 126 |
+
super().__init__()
|
| 127 |
+
self.eps = eps
|
| 128 |
+
self.patch_size = patch_size
|
| 129 |
+
self.out_dim = out_dim
|
| 130 |
+
self.head = nn.Linear(dim, math.prod(patch_size) * out_dim)
|
| 131 |
+
self.modulation = _zeros((1, 2, dim))
|
| 132 |
+
|
| 133 |
+
def __call__(self, x, e):
|
| 134 |
+
"""``x`` ``[B, tokens, dim]``, ``e`` ``[B, 1, 1, dim]``."""
|
| 135 |
+
tokens = x.shape[1]
|
| 136 |
+
mod = self.modulation.reshape(1, 1, 2, -1) + e
|
| 137 |
+
shift, scale = mx.split(mod, 2, axis=2)
|
| 138 |
+
h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1)
|
| 139 |
+
h = (h * (1.0 + scale) + shift).reshape(x.shape[0], tokens, -1)
|
| 140 |
+
return self.head(h)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def unpatchify(tokens, grid, patch_size, out_dim):
|
| 144 |
+
"""``[B, tokens, prod(patch)*C]`` -> ``[B, C, F, H, W]`` in Wan layout."""
|
| 145 |
+
b = tokens.shape[0]
|
| 146 |
+
f, h, w = grid
|
| 147 |
+
pt, ph, pw = patch_size
|
| 148 |
+
u = tokens.reshape(b, f, h, w, pt, ph, pw, out_dim)
|
| 149 |
+
u = mx.transpose(u, (0, 7, 1, 4, 2, 5, 3, 6))
|
| 150 |
+
return u.reshape(b, out_dim, f * pt, h * ph, w * pw)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class PatchEmbed(nn.Module):
|
| 154 |
+
def __init__(self, in_dim, dim, patch_size):
|
| 155 |
+
super().__init__()
|
| 156 |
+
self.patch_size = tuple(patch_size)
|
| 157 |
+
pt, ph, pw = self.patch_size
|
| 158 |
+
# MLX conv weight is [out, kd, kh, kw, in].
|
| 159 |
+
self.weight = _zeros((dim, pt, ph, pw, in_dim))
|
| 160 |
+
self.bias = _zeros((dim,))
|
| 161 |
+
|
| 162 |
+
def __call__(self, latent):
|
| 163 |
+
"""``latent`` is ``[B, 1, C, H, W]`` -> ``[B, tokens, dim]``."""
|
| 164 |
+
from .ops import conv3d_ncdhw
|
| 165 |
+
|
| 166 |
+
x = mx.transpose(latent, (0, 2, 1, 3, 4))
|
| 167 |
+
y = conv3d_ncdhw(x, self.weight, self.bias, stride=self.patch_size)
|
| 168 |
+
y = mx.transpose(y, (0, 2, 3, 4, 1))
|
| 169 |
+
return y.reshape(y.shape[0], -1, y.shape[-1])
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
class KVCache:
|
| 173 |
+
"""Per-layer list of per-frame ``(k, v)`` with shape ``[B, tokens, heads, dim]``."""
|
| 174 |
+
|
| 175 |
+
def __init__(self, num_frames: int):
|
| 176 |
+
self.num_frames = num_frames
|
| 177 |
+
self.slots = [None] * num_frames
|
| 178 |
+
|
| 179 |
+
def write(self, frame, k, v):
|
| 180 |
+
mx.eval(k, v)
|
| 181 |
+
self.slots[frame] = (k, v)
|
| 182 |
+
|
| 183 |
+
def prefix(self, frame):
|
| 184 |
+
ks = [self.slots[i][0] for i in range(frame + 1)]
|
| 185 |
+
vs = [self.slots[i][1] for i in range(frame + 1)]
|
| 186 |
+
if len(ks) == 1:
|
| 187 |
+
return ks[0], vs[0]
|
| 188 |
+
return mx.concatenate(ks, axis=1), mx.concatenate(vs, axis=1)
|
| 189 |
+
|
| 190 |
+
def slot(self, frame):
|
| 191 |
+
return self.slots[frame]
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
class LayerCache:
|
| 195 |
+
def __init__(self, num_layers, num_frames):
|
| 196 |
+
self.layers = [KVCache(num_frames) for _ in range(num_layers)]
|
| 197 |
+
|
| 198 |
+
def __getitem__(self, index):
|
| 199 |
+
return self.layers[index]
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class CausalWanModel(nn.Module):
|
| 203 |
+
def __init__(self, config: WanConfig = WanConfig()):
|
| 204 |
+
super().__init__()
|
| 205 |
+
self.config = config
|
| 206 |
+
dim = config.dim
|
| 207 |
+
self.dim = dim
|
| 208 |
+
self.num_heads = config.num_heads
|
| 209 |
+
self.num_layers = config.num_layers
|
| 210 |
+
self.patch_size = config.patch_size
|
| 211 |
+
self.text_len = config.text_len
|
| 212 |
+
self.freq_dim = config.freq_dim
|
| 213 |
+
self.out_dim = config.out_dim
|
| 214 |
+
self.eps = config.eps
|
| 215 |
+
self.patch_embedding = PatchEmbed(config.in_dim, dim, config.patch_size)
|
| 216 |
+
self.text_embed_fc1 = nn.Linear(config.text_dim, dim)
|
| 217 |
+
self.text_embed_fc2 = nn.Linear(dim, dim)
|
| 218 |
+
self.time_embed_fc1 = nn.Linear(config.freq_dim, dim)
|
| 219 |
+
self.time_embed_fc2 = nn.Linear(dim, dim)
|
| 220 |
+
self.time_proj_fc = nn.Linear(dim, dim * 6)
|
| 221 |
+
self.blocks = [
|
| 222 |
+
CausalBlock(dim, config.ffn_dim, config.num_heads, config.eps) for _ in range(config.num_layers)
|
| 223 |
+
]
|
| 224 |
+
self.head = CausalHead(dim, config.out_dim, config.patch_size, config.eps)
|
| 225 |
+
# Plain object so the cos/sin tables are not registered as parameters.
|
| 226 |
+
self.rope = RopeTables(config.head_dim)
|
| 227 |
+
|
| 228 |
+
def embed_text(self, prompt_embeds):
|
| 229 |
+
if isinstance(prompt_embeds, (list, tuple)):
|
| 230 |
+
rows = []
|
| 231 |
+
for row in prompt_embeds:
|
| 232 |
+
if row.shape[0] < self.text_len:
|
| 233 |
+
pad = mx.zeros((self.text_len - row.shape[0], row.shape[-1]), dtype=row.dtype)
|
| 234 |
+
row = mx.concatenate([row, pad], axis=0)
|
| 235 |
+
rows.append(row[: self.text_len])
|
| 236 |
+
prompt_embeds = mx.stack(rows, axis=0)
|
| 237 |
+
elif prompt_embeds.shape[1] < self.text_len:
|
| 238 |
+
pad = mx.zeros(
|
| 239 |
+
(prompt_embeds.shape[0], self.text_len - prompt_embeds.shape[1], prompt_embeds.shape[2]),
|
| 240 |
+
dtype=prompt_embeds.dtype,
|
| 241 |
+
)
|
| 242 |
+
prompt_embeds = mx.concatenate([prompt_embeds, pad], axis=1)
|
| 243 |
+
elif prompt_embeds.shape[1] > self.text_len:
|
| 244 |
+
prompt_embeds = prompt_embeds[:, : self.text_len]
|
| 245 |
+
return self.text_embed_fc2(gelu_tanh(self.text_embed_fc1(prompt_embeds)))
|
| 246 |
+
|
| 247 |
+
def time_conditioning(self, t, dtype):
|
| 248 |
+
"""``t`` is ``[B, F]`` -> ``(e0 [B,F,6,C], e_head [B,F,1,C])``."""
|
| 249 |
+
emb = sinusoidal_embedding_1d(self.freq_dim, t.reshape(-1)).astype(dtype)
|
| 250 |
+
e = self.time_embed_fc2(silu(self.time_embed_fc1(emb)))
|
| 251 |
+
e0 = self.time_proj_fc(silu(e)).reshape(*t.shape, 6, self.dim)
|
| 252 |
+
e_head = e.reshape(*t.shape, 1, self.dim)
|
| 253 |
+
return e0, e_head
|
| 254 |
+
|
| 255 |
+
def text_cache(self, prompt_embeds):
|
| 256 |
+
context = self.embed_text(prompt_embeds)
|
| 257 |
+
caches = []
|
| 258 |
+
for block in self.blocks:
|
| 259 |
+
k, v = block.cross_attn.text_kv(context)
|
| 260 |
+
mx.eval(k, v)
|
| 261 |
+
caches.append((k, v))
|
| 262 |
+
return caches
|
| 263 |
+
|
| 264 |
+
def forward_frame(self, latent, e0, e_head, text_cache, kv_cache: LayerCache, frame: int):
|
| 265 |
+
"""Denoise ``latent`` ``[B, 1, 16, H, W]``.
|
| 266 |
+
|
| 267 |
+
Returns flow ``[B, 1, 16, H, W]`` and pre-head states ``[B, tokens, dim]``.
|
| 268 |
+
"""
|
| 269 |
+
_, _, _, height, width = latent.shape
|
| 270 |
+
x = self.patch_embedding(latent)
|
| 271 |
+
grid = (1, height // self.patch_size[1], width // self.patch_size[2])
|
| 272 |
+
for index, block in enumerate(self.blocks):
|
| 273 |
+
x = block.forward_frame(x, e0, grid, self.rope, text_cache[index], kv_cache[index], frame)
|
| 274 |
+
mx.eval(x)
|
| 275 |
+
hidden = x
|
| 276 |
+
tokens = self.head(x, e_head)
|
| 277 |
+
flow = unpatchify(tokens, grid, self.patch_size, self.out_dim)
|
| 278 |
+
flow = mx.transpose(flow, (0, 2, 1, 3, 4))
|
| 279 |
+
return flow, hidden
|
fastframes_mlx/generate.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generate a clip with FastFrames-S on MLX.
|
| 2 |
+
|
| 3 |
+
python -m fastframes_mlx.generate --prompt "A corgi surfing a wave at sunset" --out clip.mp4
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
from dataclasses import replace
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import mlx.core as mx
|
| 13 |
+
|
| 14 |
+
from .config import FastFramesConfig, WanConfig
|
| 15 |
+
from .dit import CausalWanModel
|
| 16 |
+
from .kv_refiner import KVRefinementNetwork
|
| 17 |
+
from .pipeline import FastFrames, format_report
|
| 18 |
+
from .pixel_decoder import PixelDecoder
|
| 19 |
+
from .tokenizer import UMT5Tokenizer
|
| 20 |
+
from .video import save_mp4
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _smoke_stack():
|
| 24 |
+
"""Tiny random weights so the CLI path can be exercised without a download."""
|
| 25 |
+
wan = WanConfig(dim=32, ffn_dim=64, num_heads=4, num_layers=2, text_dim=32, freq_dim=16)
|
| 26 |
+
cfg = FastFramesConfig(
|
| 27 |
+
name="smoke",
|
| 28 |
+
wan=wan,
|
| 29 |
+
num_frames=2,
|
| 30 |
+
latent_height=4,
|
| 31 |
+
latent_width=4,
|
| 32 |
+
num_pixel_frames=5,
|
| 33 |
+
pixel_height=8,
|
| 34 |
+
pixel_width=8,
|
| 35 |
+
refiner_width=16,
|
| 36 |
+
refiner_stem_blocks=1,
|
| 37 |
+
refiner_stem_heads=4,
|
| 38 |
+
refiner_stem_mlp=2,
|
| 39 |
+
pixel_patch=(4, 4, 4),
|
| 40 |
+
decoder_low_blocks=1,
|
| 41 |
+
decoder_high_blocks=1,
|
| 42 |
+
decoder_ffn=2,
|
| 43 |
+
)
|
| 44 |
+
dit = CausalWanModel(wan)
|
| 45 |
+
refiner = KVRefinementNetwork(
|
| 46 |
+
dim=wan.dim,
|
| 47 |
+
num_heads=wan.num_heads,
|
| 48 |
+
num_blocks=wan.num_layers,
|
| 49 |
+
width=cfg.refiner_width,
|
| 50 |
+
n_stem=cfg.refiner_stem_blocks,
|
| 51 |
+
stem_heads=cfg.refiner_stem_heads,
|
| 52 |
+
stem_mlp=cfg.refiner_stem_mlp,
|
| 53 |
+
)
|
| 54 |
+
decoder = PixelDecoder(
|
| 55 |
+
dim=wan.dim,
|
| 56 |
+
pixel_patch_size=cfg.pixel_patch,
|
| 57 |
+
n_low_blocks=cfg.decoder_low_blocks,
|
| 58 |
+
n_high_blocks=cfg.decoder_high_blocks,
|
| 59 |
+
ffn_mult=cfg.decoder_ffn,
|
| 60 |
+
)
|
| 61 |
+
# Fixed prompt embedding: the smoke path does not load umT5.
|
| 62 |
+
prompt = mx.zeros((1, wan.text_len, wan.text_dim))
|
| 63 |
+
model = FastFrames(cfg, dit, refiner, decoder)
|
| 64 |
+
return model, prompt
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def build_argparser():
|
| 68 |
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
| 69 |
+
parser.add_argument("--prompt", help="text prompt (required unless --smoke)")
|
| 70 |
+
parser.add_argument("--out", default="clip.mp4", help="output mp4 path")
|
| 71 |
+
parser.add_argument("--seed", type=int, default=0)
|
| 72 |
+
parser.add_argument(
|
| 73 |
+
"--latent-frames",
|
| 74 |
+
type=int,
|
| 75 |
+
default=21,
|
| 76 |
+
help="latent frames (21 -> 81 RGB frames). The released model was trained at 21",
|
| 77 |
+
)
|
| 78 |
+
parser.add_argument("--cache-dir", type=Path, default=None, help="Hugging Face cache directory")
|
| 79 |
+
parser.add_argument("--bits", type=int, choices=(4, 8), default=None, help="quantize DiT and umT5 linears")
|
| 80 |
+
parser.add_argument("--group-size", type=int, default=64)
|
| 81 |
+
parser.add_argument(
|
| 82 |
+
"--decoder",
|
| 83 |
+
choices=("pixel", "vae"),
|
| 84 |
+
default="pixel",
|
| 85 |
+
help="pixel is the FastFrames decoder; vae decodes the DiT latents with Wan2.1-VAE",
|
| 86 |
+
)
|
| 87 |
+
parser.add_argument(
|
| 88 |
+
"--keep-text-encoder",
|
| 89 |
+
action="store_true",
|
| 90 |
+
help="keep umT5 allocated during the rollout (higher memory)",
|
| 91 |
+
)
|
| 92 |
+
parser.add_argument(
|
| 93 |
+
"--smoke",
|
| 94 |
+
action="store_true",
|
| 95 |
+
help="run a tiny random network (not FastFrames weights) to check the pipeline",
|
| 96 |
+
)
|
| 97 |
+
return parser
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def main(argv=None):
|
| 101 |
+
args = build_argparser().parse_args(argv)
|
| 102 |
+
if not args.smoke and not args.prompt:
|
| 103 |
+
build_argparser().error("--prompt is required unless --smoke")
|
| 104 |
+
if args.smoke:
|
| 105 |
+
model, prompt_embeds = _smoke_stack()
|
| 106 |
+
prompt = args.prompt or "smoke"
|
| 107 |
+
frames, report = model.generate(prompt, args.seed, prompt_embeds=prompt_embeds)
|
| 108 |
+
else:
|
| 109 |
+
from .weights import load_fastframes_s
|
| 110 |
+
|
| 111 |
+
cfg, dit, refiner, decoder, text, tok_path, vae = load_fastframes_s(
|
| 112 |
+
cache_dir=args.cache_dir,
|
| 113 |
+
bits=args.bits,
|
| 114 |
+
group_size=args.group_size,
|
| 115 |
+
load_vae=args.decoder == "vae",
|
| 116 |
+
)
|
| 117 |
+
if args.latent_frames != cfg.num_frames:
|
| 118 |
+
if args.latent_frames < 1:
|
| 119 |
+
build_argparser().error("--latent-frames must be >= 1")
|
| 120 |
+
cfg = replace(
|
| 121 |
+
cfg,
|
| 122 |
+
num_frames=args.latent_frames,
|
| 123 |
+
num_pixel_frames=4 * args.latent_frames - 3,
|
| 124 |
+
)
|
| 125 |
+
tokenizer = UMT5Tokenizer(tok_path, seq_len=cfg.wan.text_len)
|
| 126 |
+
model = FastFrames(cfg, dit, refiner, decoder, text, tokenizer, vae)
|
| 127 |
+
frames, report = model.generate(
|
| 128 |
+
args.prompt,
|
| 129 |
+
args.seed,
|
| 130 |
+
use_vae=args.decoder == "vae",
|
| 131 |
+
release_text_encoder=not args.keep_text_encoder,
|
| 132 |
+
)
|
| 133 |
+
path = save_mp4(args.out, frames, fps=model.config.fps)
|
| 134 |
+
print(format_report(report))
|
| 135 |
+
print(f"wrote {path}")
|
| 136 |
+
return 0
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
if __name__ == "__main__":
|
| 140 |
+
raise SystemExit(main())
|
fastframes_mlx/kv_refiner.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""KV refinement network (FastFrames section 3.3).
|
| 2 |
+
|
| 3 |
+
After a frame's last denoising step the cache holds noisy keys and values.
|
| 4 |
+
The refiner maps those, plus a stem over the clean latent, to a residual
|
| 5 |
+
update. Released weights use one grouped GEMM over all 30 DiT layers
|
| 6 |
+
(``heads_grouped.gemms.0``) and a bias-free projection named
|
| 7 |
+
``heads_grouped.phi_proj.<width>``.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import mlx.core as mx
|
| 13 |
+
import mlx.nn as nn
|
| 14 |
+
|
| 15 |
+
from .ops import gelu_tanh, layer_norm, sdpa_heads_first, silu, sinusoidal_embedding_1d
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class TimestepEmbedder(nn.Module):
|
| 19 |
+
def __init__(self, width, freq_dim=256):
|
| 20 |
+
super().__init__()
|
| 21 |
+
self.freq_dim = freq_dim
|
| 22 |
+
self.fc1 = nn.Linear(freq_dim, width)
|
| 23 |
+
self.fc2 = nn.Linear(width, width)
|
| 24 |
+
|
| 25 |
+
def __call__(self, t):
|
| 26 |
+
emb = sinusoidal_embedding_1d(self.freq_dim, t.reshape(-1)).astype(mx.float32)
|
| 27 |
+
emb = emb.astype(self.fc1.weight.dtype)
|
| 28 |
+
return self.fc2(silu(self.fc1(emb)))
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class Modulation(nn.Module):
|
| 32 |
+
def __init__(self, width, n):
|
| 33 |
+
super().__init__()
|
| 34 |
+
self.n = n
|
| 35 |
+
self.lin = nn.Linear(width, n * width)
|
| 36 |
+
|
| 37 |
+
def __call__(self, e):
|
| 38 |
+
return mx.split(self.lin(silu(e))[:, None, :], self.n, axis=-1)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class _MLP(nn.Module):
|
| 42 |
+
def __init__(self, width, hidden):
|
| 43 |
+
super().__init__()
|
| 44 |
+
self.fc1 = nn.Linear(width, hidden)
|
| 45 |
+
self.fc2 = nn.Linear(hidden, width)
|
| 46 |
+
|
| 47 |
+
def __call__(self, x):
|
| 48 |
+
return self.fc2(gelu_tanh(self.fc1(x)))
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class _Affine(nn.Module):
|
| 52 |
+
def __init__(self, width):
|
| 53 |
+
super().__init__()
|
| 54 |
+
self.weight = mx.ones((width,))
|
| 55 |
+
self.bias = mx.zeros((width,))
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class StemBlock(nn.Module):
|
| 59 |
+
def __init__(self, width, n_heads, mlp_mult):
|
| 60 |
+
super().__init__()
|
| 61 |
+
self.n_heads = n_heads
|
| 62 |
+
self.head_dim = width // n_heads
|
| 63 |
+
self.qkv = nn.Linear(width, 3 * width)
|
| 64 |
+
self.proj = nn.Linear(width, width)
|
| 65 |
+
self.mlp = _MLP(width, width * mlp_mult)
|
| 66 |
+
self.mod = Modulation(width, 6)
|
| 67 |
+
self.eps = 1e-5
|
| 68 |
+
|
| 69 |
+
def __call__(self, x, e):
|
| 70 |
+
b, s, _ = x.shape
|
| 71 |
+
sh1, sc1, g1, sh2, sc2, g2 = self.mod(e)
|
| 72 |
+
h = layer_norm(x, self.eps) * (1.0 + sc1) + sh1
|
| 73 |
+
qkv = self.qkv(h).reshape(b, s, 3, self.n_heads, self.head_dim)
|
| 74 |
+
q, k, v = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2]
|
| 75 |
+
q = q.transpose(0, 2, 1, 3)
|
| 76 |
+
k = k.transpose(0, 2, 1, 3)
|
| 77 |
+
v = v.transpose(0, 2, 1, 3)
|
| 78 |
+
attn = sdpa_heads_first(q, k, v).transpose(0, 2, 1, 3).reshape(b, s, -1)
|
| 79 |
+
x = x + g1 * self.proj(attn)
|
| 80 |
+
h = layer_norm(x, self.eps) * (1.0 + sc2) + sh2
|
| 81 |
+
return x + g2 * self.mlp(h)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class GroupGEMM(nn.Module):
|
| 85 |
+
def __init__(self, num_members, dim, width, num_heads, head_dim, mlp_mult=2):
|
| 86 |
+
super().__init__()
|
| 87 |
+
self.dim = dim
|
| 88 |
+
self.width = width
|
| 89 |
+
self.num_heads = num_heads
|
| 90 |
+
self.head_dim = head_dim
|
| 91 |
+
n = num_members
|
| 92 |
+
d_v = width
|
| 93 |
+
d_k = width
|
| 94 |
+
|
| 95 |
+
def weight(i, o):
|
| 96 |
+
return mx.zeros((n, i, o))
|
| 97 |
+
|
| 98 |
+
def bias(o):
|
| 99 |
+
return mx.zeros((n, 1, o))
|
| 100 |
+
|
| 101 |
+
self.proj_k_w, self.proj_k_b = weight(dim, d_k), bias(d_k)
|
| 102 |
+
self.proj_v_w, self.proj_v_b = weight(dim, d_v), bias(d_v)
|
| 103 |
+
self.fuse_w, self.fuse_b = weight(d_k + 2 * d_v, d_v), bias(d_v)
|
| 104 |
+
self.m0_w, self.m0_b = weight(d_v, d_v * mlp_mult), bias(d_v * mlp_mult)
|
| 105 |
+
self.m2_w, self.m2_b = weight(d_v * mlp_mult, d_v), bias(d_v)
|
| 106 |
+
self.mod_w, self.mod_b = weight(width, 2 * d_v), bias(2 * d_v)
|
| 107 |
+
self.dK_w, self.dK_b = weight(d_v, dim), bias(dim)
|
| 108 |
+
self.dV_w, self.dV_b = weight(d_v, dim), bias(dim)
|
| 109 |
+
self.v_logscale = mx.zeros((n, 1, dim))
|
| 110 |
+
|
| 111 |
+
def _renorm(self, x, ref):
|
| 112 |
+
b, s, _ = x.shape
|
| 113 |
+
xh = x.reshape(b, s, self.num_heads, self.head_dim)
|
| 114 |
+
rh = ref.reshape(b, s, self.num_heads, self.head_dim)
|
| 115 |
+
scale = mx.linalg.norm(rh, axis=-1, keepdims=True) / (mx.linalg.norm(xh, axis=-1, keepdims=True) + 1e-6)
|
| 116 |
+
return (xh * scale).reshape(b, s, -1)
|
| 117 |
+
|
| 118 |
+
def __call__(self, n_k, n_v, phi_v, e):
|
| 119 |
+
b = n_k.shape[0]
|
| 120 |
+
pk = mx.matmul(n_k, self.proj_k_w) + self.proj_k_b
|
| 121 |
+
pv = mx.matmul(n_v, self.proj_v_w) + self.proj_v_b
|
| 122 |
+
h = mx.matmul(mx.concatenate([pk, pv, phi_v], axis=-1), self.fuse_w) + self.fuse_b
|
| 123 |
+
e1 = silu(e).reshape(1, 1, self.width)
|
| 124 |
+
e1 = mx.broadcast_to(e1, (b, 1, self.width))
|
| 125 |
+
shift, scale = mx.split(mx.matmul(e1, self.mod_w) + self.mod_b, 2, axis=-1)
|
| 126 |
+
h = layer_norm(h, 1e-5) * (1.0 + scale) + shift
|
| 127 |
+
h = mx.matmul(h, self.m0_w) + self.m0_b
|
| 128 |
+
h = mx.matmul(gelu_tanh(h), self.m2_w) + self.m2_b
|
| 129 |
+
k = self._renorm(n_k + mx.matmul(h, self.dK_w) + self.dK_b, n_k)
|
| 130 |
+
v = (n_v + mx.matmul(h, self.dV_w) + self.dV_b) * mx.exp(self.v_logscale)
|
| 131 |
+
return k, v
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
class GroupedHeads(nn.Module):
|
| 135 |
+
def __init__(self, num_blocks, dim, width, num_heads, head_dim, mlp_mult):
|
| 136 |
+
super().__init__()
|
| 137 |
+
self.num_blocks = num_blocks
|
| 138 |
+
self.gemms = [GroupGEMM(num_blocks, dim, width, num_heads, head_dim, mlp_mult)]
|
| 139 |
+
self.phi_proj = {str(width): nn.Linear(width, width, bias=False)}
|
| 140 |
+
|
| 141 |
+
def __call__(self, n_k, n_v, phi, e):
|
| 142 |
+
phi2 = phi[0] if phi.ndim == 3 else phi
|
| 143 |
+
gemm = self.gemms[0]
|
| 144 |
+
projected = self.phi_proj[str(gemm.width)](phi2)
|
| 145 |
+
phi_v = mx.broadcast_to(projected[None], (self.num_blocks, projected.shape[0], projected.shape[1]))
|
| 146 |
+
return gemm(n_k, n_v, phi_v, e)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
class KVRefinementNetwork(nn.Module):
|
| 150 |
+
def __init__(
|
| 151 |
+
self,
|
| 152 |
+
dim=1536,
|
| 153 |
+
num_heads=12,
|
| 154 |
+
num_blocks=30,
|
| 155 |
+
width=768,
|
| 156 |
+
n_stem=4,
|
| 157 |
+
stem_heads=8,
|
| 158 |
+
stem_mlp=4,
|
| 159 |
+
head_mlp=2,
|
| 160 |
+
):
|
| 161 |
+
super().__init__()
|
| 162 |
+
if dim % num_heads != 0:
|
| 163 |
+
raise ValueError(f"dim {dim} is not divisible by heads {num_heads}")
|
| 164 |
+
self.dim = dim
|
| 165 |
+
self.num_heads = num_heads
|
| 166 |
+
self.head_dim = dim // num_heads
|
| 167 |
+
self.num_blocks = num_blocks
|
| 168 |
+
self.width = width
|
| 169 |
+
self.t_embed = TimestepEmbedder(width)
|
| 170 |
+
self.stem_in = nn.Linear(dim, width)
|
| 171 |
+
self.stem_blocks = [StemBlock(width, stem_heads, stem_mlp) for _ in range(n_stem)]
|
| 172 |
+
self.stem_norm = _Affine(width)
|
| 173 |
+
self.heads_grouped = GroupedHeads(num_blocks, dim, width, num_heads, self.head_dim, head_mlp)
|
| 174 |
+
|
| 175 |
+
def embed_timestep(self, timestep, batch, dtype):
|
| 176 |
+
if not isinstance(timestep, mx.array):
|
| 177 |
+
timestep = mx.array([float(timestep)], dtype=mx.float32)
|
| 178 |
+
timestep = timestep.astype(mx.float32).reshape(-1)
|
| 179 |
+
if timestep.size == 1 and batch != 1:
|
| 180 |
+
timestep = mx.broadcast_to(timestep, (batch,))
|
| 181 |
+
elif timestep.size == 1 and batch == 1:
|
| 182 |
+
pass
|
| 183 |
+
return self.t_embed(timestep).astype(dtype)
|
| 184 |
+
|
| 185 |
+
def compute_phi(self, stem_tokens, e):
|
| 186 |
+
x = self.stem_in(stem_tokens)
|
| 187 |
+
for block in self.stem_blocks:
|
| 188 |
+
x = block(x, e)
|
| 189 |
+
return layer_norm(x, 1e-5, self.stem_norm.weight, self.stem_norm.bias)
|
| 190 |
+
|
| 191 |
+
def repair_all(self, n_k, n_v, phi, e):
|
| 192 |
+
return self.heads_grouped(n_k, n_v, phi, e)
|
fastframes_mlx/ops.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Small numeric helpers shared by the MLX modules.
|
| 2 |
+
|
| 3 |
+
LayerNorm and RMSNorm follow the Wan / FastFrames definitions: the reduction is
|
| 4 |
+
in float32, the result is cast back to the activation dtype, and RMSNorm
|
| 5 |
+
multiplies the weight after that cast.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
|
| 14 |
+
import mlx.core as mx
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def gelu_tanh(x: mx.array) -> mx.array:
|
| 18 |
+
"""PyTorch ``GELU(approximate='tanh')`` and the umT5 encoder GELU."""
|
| 19 |
+
c = math.sqrt(2.0 / math.pi)
|
| 20 |
+
return 0.5 * x * (1.0 + mx.tanh(c * (x + 0.044715 * (x * x * x))))
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def gelu_erf(x: mx.array) -> mx.array:
|
| 24 |
+
"""Exact PyTorch ``GELU()`` (erf), used by the pixel-decoder ConvNeXt blocks."""
|
| 25 |
+
return 0.5 * x * (1.0 + mx.erf(x / math.sqrt(2.0)))
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def silu(x: mx.array) -> mx.array:
|
| 29 |
+
return x * mx.sigmoid(x)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def layer_norm(x, eps, weight=None, bias=None):
|
| 33 |
+
xf = x.astype(mx.float32)
|
| 34 |
+
mean = mx.mean(xf, axis=-1, keepdims=True)
|
| 35 |
+
var = mx.mean(mx.square(xf - mean), axis=-1, keepdims=True)
|
| 36 |
+
y = (xf - mean) * mx.rsqrt(var + eps)
|
| 37 |
+
if weight is not None:
|
| 38 |
+
y = y * weight.astype(mx.float32)
|
| 39 |
+
if bias is not None:
|
| 40 |
+
y = y + bias.astype(mx.float32)
|
| 41 |
+
return y.astype(x.dtype)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def rms_norm(x, weight, eps):
|
| 45 |
+
xf = x.astype(mx.float32)
|
| 46 |
+
y = xf * mx.rsqrt(mx.mean(mx.square(xf), axis=-1, keepdims=True) + eps)
|
| 47 |
+
return y.astype(x.dtype) * weight
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def sinusoidal_embedding_1d(dim: int, position: mx.array) -> mx.array:
|
| 51 |
+
"""Wan / DiT timestep embedding.
|
| 52 |
+
|
| 53 |
+
Angles are NumPy float64, matching PyTorch's float64 sinusoid. The array
|
| 54 |
+
handed back to MLX is float32 so it can live on the GPU.
|
| 55 |
+
"""
|
| 56 |
+
if dim % 2 != 0:
|
| 57 |
+
raise ValueError(f"sinusoidal dim must be even, got {dim}")
|
| 58 |
+
half = dim // 2
|
| 59 |
+
pos = np.asarray(position.astype(mx.float32)).astype(np.float64).reshape(-1)
|
| 60 |
+
div = np.arange(half, dtype=np.float64) / np.float64(half)
|
| 61 |
+
sinusoid = pos[:, None] * np.power(np.float64(10000.0), -div)[None, :]
|
| 62 |
+
emb = np.concatenate([np.cos(sinusoid), np.sin(sinusoid)], axis=1).astype(np.float32)
|
| 63 |
+
return mx.array(emb)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def sdpa(q, k, v):
|
| 67 |
+
"""Scaled dot-product attention.
|
| 68 |
+
|
| 69 |
+
``q, k, v`` are ``[B, L, H, D]``. Matches FastFrames' ``attention``: cast to
|
| 70 |
+
bfloat16, scale by ``1/sqrt(D)``, cast back to ``q.dtype``. The official
|
| 71 |
+
release asks for the cuDNN SDPA kernel; this is the MLX SDPA kernel (math
|
| 72 |
+
on CPU).
|
| 73 |
+
"""
|
| 74 |
+
out_dtype = q.dtype
|
| 75 |
+
q = q.transpose(0, 2, 1, 3).astype(mx.bfloat16)
|
| 76 |
+
k = k.transpose(0, 2, 1, 3).astype(mx.bfloat16)
|
| 77 |
+
v = v.transpose(0, 2, 1, 3).astype(mx.bfloat16)
|
| 78 |
+
scale = q.shape[-1] ** -0.5
|
| 79 |
+
out = mx.fast.scaled_dot_product_attention(q, k, v, scale=scale)
|
| 80 |
+
return out.transpose(0, 2, 1, 3).astype(out_dtype)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def sdpa_heads_first(q, k, v):
|
| 84 |
+
"""SDPA for ``[B, H, L, D]`` queries. No forced bfloat16 cast (refiner stem)."""
|
| 85 |
+
scale = q.shape[-1] ** -0.5
|
| 86 |
+
return mx.fast.scaled_dot_product_attention(q, k, v, scale=scale)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def conv3d_ncdhw(x, weight, bias=None, stride=(1, 1, 1)):
|
| 90 |
+
"""Conv3d on ``[N, C, D, H, W]``. ``weight`` is MLX layout ``[O, Kd, Kh, Kw, I]``."""
|
| 91 |
+
xt = mx.transpose(x, (0, 2, 3, 4, 1))
|
| 92 |
+
y = mx.conv3d(xt, weight.astype(xt.dtype), stride=stride, padding=0)
|
| 93 |
+
if bias is not None:
|
| 94 |
+
y = y + bias.astype(y.dtype)
|
| 95 |
+
return mx.transpose(y, (0, 4, 1, 2, 3))
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def depthwise_conv3d_ncdhw(x, weight, bias=None):
|
| 99 |
+
"""Depthwise conv3d, stride 1, padding 0. ``weight`` is ``[C, Kd, Kh, Kw, 1]``."""
|
| 100 |
+
xt = mx.transpose(x, (0, 2, 3, 4, 1))
|
| 101 |
+
_, d, h, w, _ = xt.shape
|
| 102 |
+
kd, kh, kw = int(weight.shape[1]), int(weight.shape[2]), int(weight.shape[3])
|
| 103 |
+
od, oh, ow = d - kd + 1, h - kh + 1, w - kw + 1
|
| 104 |
+
if od < 1 or oh < 1 or ow < 1:
|
| 105 |
+
raise ValueError(f"depthwise conv output is empty for input {(d, h, w)} kernel {(kd, kh, kw)}")
|
| 106 |
+
acc = None
|
| 107 |
+
w = weight.astype(xt.dtype)
|
| 108 |
+
for kt in range(kd):
|
| 109 |
+
for ih in range(kh):
|
| 110 |
+
for iw in range(kw):
|
| 111 |
+
sl = xt[:, kt : kt + od, ih : ih + oh, iw : iw + ow, :]
|
| 112 |
+
coeff = w[:, kt, ih, iw, 0]
|
| 113 |
+
term = sl * coeff
|
| 114 |
+
acc = term if acc is None else acc + term
|
| 115 |
+
if bias is not None:
|
| 116 |
+
acc = acc + bias.astype(acc.dtype)
|
| 117 |
+
return mx.transpose(acc, (0, 4, 1, 2, 3))
|
fastframes_mlx/pipeline.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Autoregressive FastFrames-S rollout.
|
| 2 |
+
|
| 3 |
+
Each latent frame is denoised with the frozen two-step schedule (four steps on
|
| 4 |
+
frame 0). Frame 0 then gets a clean DiT pass at t = 0, which writes the cache.
|
| 5 |
+
Later frames overwrite that slot with the KV refiner instead of another DiT
|
| 6 |
+
pass. The pixel decoder turns the stacked pre-head states into RGB.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
import mlx.core as mx
|
| 14 |
+
import numpy as np
|
| 15 |
+
|
| 16 |
+
from .config import FastFramesConfig, frame_tokens
|
| 17 |
+
from .dit import LayerCache
|
| 18 |
+
from .pixel_decoder import align_frames
|
| 19 |
+
from .scheduler import FlowMatchScheduler
|
| 20 |
+
from .tokenizer import UMT5Tokenizer
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _rss_bytes():
|
| 24 |
+
import resource
|
| 25 |
+
import sys
|
| 26 |
+
|
| 27 |
+
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
| 28 |
+
if sys.platform == "darwin":
|
| 29 |
+
return int(rss)
|
| 30 |
+
return int(rss) * 1024
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class FastFrames:
|
| 34 |
+
def __init__(self, config: FastFramesConfig, dit, refiner, decoder, text_encoder=None, tokenizer=None, vae=None):
|
| 35 |
+
self.config = config
|
| 36 |
+
self.dit = dit
|
| 37 |
+
self.refiner = refiner
|
| 38 |
+
self.decoder = decoder
|
| 39 |
+
self.text_encoder = text_encoder
|
| 40 |
+
self.tokenizer = tokenizer
|
| 41 |
+
self.vae = vae
|
| 42 |
+
self.scheduler = FlowMatchScheduler(shift=config.shift)
|
| 43 |
+
self.frame_tokens = frame_tokens(config)
|
| 44 |
+
self._time = {}
|
| 45 |
+
self._refine_embedding = None
|
| 46 |
+
|
| 47 |
+
def encode_prompt(self, prompt: str):
|
| 48 |
+
if self.text_encoder is None or self.tokenizer is None:
|
| 49 |
+
raise RuntimeError("text encoder is not loaded")
|
| 50 |
+
ids, mask = self.tokenizer.encode(prompt)
|
| 51 |
+
context = self.text_encoder(ids, mask)
|
| 52 |
+
context = context * mask.astype(context.dtype)[:, :, None]
|
| 53 |
+
mx.eval(context)
|
| 54 |
+
return context
|
| 55 |
+
|
| 56 |
+
def release_text_encoder(self):
|
| 57 |
+
self.text_encoder = None
|
| 58 |
+
self.tokenizer = None
|
| 59 |
+
|
| 60 |
+
def _conditioning(self, timestep, dtype):
|
| 61 |
+
key = (float(timestep), str(dtype))
|
| 62 |
+
if key not in self._time:
|
| 63 |
+
t = mx.array([[float(timestep)]], dtype=mx.float32)
|
| 64 |
+
self._time[key] = self.dit.time_conditioning(t, dtype)
|
| 65 |
+
mx.eval(self._time[key])
|
| 66 |
+
return self._time[key]
|
| 67 |
+
|
| 68 |
+
def _predict(self, noisy, timestep, frame, text_cache, kv_cache):
|
| 69 |
+
dtype = noisy.dtype
|
| 70 |
+
e0, e_head = self._conditioning(timestep, dtype)
|
| 71 |
+
flow, features = self.dit.forward_frame(noisy, e0, e_head, text_cache, kv_cache, frame)
|
| 72 |
+
flat_flow = flow.reshape(-1, *flow.shape[2:])
|
| 73 |
+
flat_noisy = noisy.reshape(-1, *noisy.shape[2:])
|
| 74 |
+
t = mx.array([float(timestep)], dtype=mx.float32)
|
| 75 |
+
x0 = self.scheduler.flow_to_x0(flat_flow, flat_noisy, t).reshape(flow.shape)
|
| 76 |
+
mx.eval(x0, features, e_head)
|
| 77 |
+
return x0, features, e_head
|
| 78 |
+
|
| 79 |
+
def _refine(self, x0, frame, kv_cache):
|
| 80 |
+
dtype = x0.dtype
|
| 81 |
+
if self._refine_embedding is None:
|
| 82 |
+
self._refine_embedding = self.refiner.embed_timestep(self.config.refine_timestep, 1, dtype)
|
| 83 |
+
mx.eval(self._refine_embedding)
|
| 84 |
+
stem = self.dit.patch_embedding(x0)
|
| 85 |
+
phi = self.refiner.compute_phi(stem.astype(self.refiner.stem_in.weight.dtype), self._refine_embedding)
|
| 86 |
+
tokens = self.frame_tokens
|
| 87 |
+
keys = []
|
| 88 |
+
values = []
|
| 89 |
+
for layer in kv_cache.layers:
|
| 90 |
+
k, v = layer.slot(frame)
|
| 91 |
+
keys.append(k.reshape(tokens, -1))
|
| 92 |
+
values.append(v.reshape(tokens, -1))
|
| 93 |
+
keys = mx.stack(keys, axis=0)
|
| 94 |
+
values = mx.stack(values, axis=0)
|
| 95 |
+
keys, values = self.refiner.repair_all(keys, values, phi, self._refine_embedding)
|
| 96 |
+
heads = self.dit.num_heads
|
| 97 |
+
head_dim = self.dit.dim // heads
|
| 98 |
+
for index, layer in enumerate(kv_cache.layers):
|
| 99 |
+
k = keys[index].reshape(1, tokens, heads, head_dim).astype(dtype)
|
| 100 |
+
v = values[index].reshape(1, tokens, heads, head_dim).astype(dtype)
|
| 101 |
+
layer.write(frame, k, v)
|
| 102 |
+
mx.eval(phi)
|
| 103 |
+
|
| 104 |
+
def _initial_noise(self, seed, dtype):
|
| 105 |
+
cfg = self.config
|
| 106 |
+
rng = np.random.Generator(np.random.PCG64(int(seed)))
|
| 107 |
+
noise = rng.standard_normal((1, cfg.num_frames, 16, cfg.latent_height, cfg.latent_width), dtype=np.float32)
|
| 108 |
+
frame_shape = (1, 16, cfg.latent_height, cfg.latent_width)
|
| 109 |
+
|
| 110 |
+
def transition(_step, _frame):
|
| 111 |
+
return mx.array(rng.standard_normal(frame_shape, dtype=np.float32)).astype(dtype)
|
| 112 |
+
|
| 113 |
+
return mx.array(noise).astype(dtype), transition
|
| 114 |
+
|
| 115 |
+
def rollout(self, prompt_embeds, seed: int):
|
| 116 |
+
cfg = self.config
|
| 117 |
+
dtype = prompt_embeds.dtype
|
| 118 |
+
noise, transition = self._initial_noise(seed, dtype)
|
| 119 |
+
text_cache = self.dit.text_cache(prompt_embeds)
|
| 120 |
+
kv_cache = LayerCache(self.dit.num_layers, cfg.num_frames)
|
| 121 |
+
latents = []
|
| 122 |
+
features = []
|
| 123 |
+
modulation = []
|
| 124 |
+
frame_times = []
|
| 125 |
+
for frame in range(cfg.num_frames):
|
| 126 |
+
started = time.perf_counter()
|
| 127 |
+
schedule = cfg.first_frame_timesteps if frame == 0 else cfg.timesteps
|
| 128 |
+
x = noise[:, frame : frame + 1]
|
| 129 |
+
x0 = features_f = mod = None
|
| 130 |
+
for step, timestep in enumerate(schedule):
|
| 131 |
+
x0, features_f, mod = self._predict(x, timestep, frame, text_cache, kv_cache)
|
| 132 |
+
if step + 1 < len(schedule):
|
| 133 |
+
nxt = mx.array([float(schedule[step + 1])], dtype=mx.float32)
|
| 134 |
+
eps = transition(step, frame)
|
| 135 |
+
flat = self.scheduler.add_noise(x0.reshape(-1, *x0.shape[2:]), eps, nxt)
|
| 136 |
+
x = flat.reshape(x0.shape)
|
| 137 |
+
mx.eval(x)
|
| 138 |
+
latents.append(x0)
|
| 139 |
+
features.append(features_f)
|
| 140 |
+
modulation.append(mod)
|
| 141 |
+
if frame == 0:
|
| 142 |
+
self._predict(x0, 0.0, frame, text_cache, kv_cache)
|
| 143 |
+
else:
|
| 144 |
+
self._refine(x0, frame, kv_cache)
|
| 145 |
+
mx.eval(kv_cache.layers[0].slot(frame)[0])
|
| 146 |
+
frame_times.append(time.perf_counter() - started)
|
| 147 |
+
latent = mx.concatenate(latents, axis=1)
|
| 148 |
+
feats = mx.concatenate(features, axis=1)
|
| 149 |
+
mods = mx.concatenate(modulation, axis=1)
|
| 150 |
+
mx.eval(latent, feats, mods)
|
| 151 |
+
return latent, feats, mods, frame_times
|
| 152 |
+
|
| 153 |
+
def decode_pixels(self, features, modulation):
|
| 154 |
+
cfg = self.config
|
| 155 |
+
grid = (
|
| 156 |
+
modulation.shape[1],
|
| 157 |
+
cfg.latent_height // cfg.wan.patch_size[1],
|
| 158 |
+
cfg.latent_width // cfg.wan.patch_size[2],
|
| 159 |
+
)
|
| 160 |
+
# The released decoder is stored in fp32. Official CUDA inference also
|
| 161 |
+
# wraps this call in bf16 autocast; this port runs the fp32 weights.
|
| 162 |
+
feats = features.astype(self.decoder.fc_out.weight.dtype)
|
| 163 |
+
mods = modulation.astype(self.decoder.modulation.dtype)
|
| 164 |
+
pixels = self.decoder(feats, mods, grid)
|
| 165 |
+
pixels = align_frames(pixels, cfg.num_pixel_frames)
|
| 166 |
+
mx.eval(pixels)
|
| 167 |
+
return pixels
|
| 168 |
+
|
| 169 |
+
def decode_vae(self, latent):
|
| 170 |
+
if self.vae is None:
|
| 171 |
+
raise RuntimeError("VAE is not loaded; pass --decoder vae")
|
| 172 |
+
# [1, F, C, H, W] -> [F, H, W, C]
|
| 173 |
+
z = mx.transpose(latent[0], (0, 2, 3, 1)).astype(mx.float32)
|
| 174 |
+
pixels = self.vae.decode(z)
|
| 175 |
+
mx.eval(pixels)
|
| 176 |
+
# [T, H, W, 3] -> [1, T, 3, H, W]
|
| 177 |
+
return mx.transpose(pixels, (0, 3, 1, 2))[None]
|
| 178 |
+
|
| 179 |
+
def to_uint8(self, pixels):
|
| 180 |
+
"""``pixels`` ``[1, T, 3, H, W]`` or ``[T, 3, H, W]`` in ``[-1, 1]``."""
|
| 181 |
+
if pixels.ndim == 5:
|
| 182 |
+
pixels = pixels[0]
|
| 183 |
+
values = mx.round((mx.clip(pixels.astype(mx.float32), -1, 1) + 1.0) * 127.5)
|
| 184 |
+
values = mx.clip(values, 0, 255).astype(mx.uint8)
|
| 185 |
+
values = mx.transpose(values, (0, 2, 3, 1))
|
| 186 |
+
mx.eval(values)
|
| 187 |
+
return np.array(values)
|
| 188 |
+
|
| 189 |
+
def generate(self, prompt: str, seed: int, prompt_embeds=None, use_vae: bool = False, release_text_encoder: bool = False):
|
| 190 |
+
report = {"prompt": prompt, "seed": int(seed)}
|
| 191 |
+
started = time.perf_counter()
|
| 192 |
+
if prompt_embeds is None:
|
| 193 |
+
t0 = time.perf_counter()
|
| 194 |
+
prompt_embeds = self.encode_prompt(prompt)
|
| 195 |
+
report["text_encoder_s"] = time.perf_counter() - t0
|
| 196 |
+
if release_text_encoder:
|
| 197 |
+
self.release_text_encoder()
|
| 198 |
+
mx.clear_cache()
|
| 199 |
+
# Peak after the encoder is released, so this is the rollout allocation.
|
| 200 |
+
mx.reset_peak_memory()
|
| 201 |
+
latent, features, modulation, frame_times = self.rollout(prompt_embeds, seed)
|
| 202 |
+
report["latent_frame_s"] = frame_times
|
| 203 |
+
t1 = time.perf_counter()
|
| 204 |
+
if use_vae:
|
| 205 |
+
pixels = self.decode_vae(latent)
|
| 206 |
+
else:
|
| 207 |
+
pixels = self.decode_pixels(features, modulation)
|
| 208 |
+
report["decoder_s"] = time.perf_counter() - t1
|
| 209 |
+
frames = self.to_uint8(pixels)
|
| 210 |
+
report["total_s"] = time.perf_counter() - started
|
| 211 |
+
report["output_frames"] = int(frames.shape[0])
|
| 212 |
+
report["height"] = int(frames.shape[1])
|
| 213 |
+
report["width"] = int(frames.shape[2])
|
| 214 |
+
report["mlx_peak_bytes"] = int(mx.get_peak_memory())
|
| 215 |
+
report["rss_peak_bytes"] = _rss_bytes()
|
| 216 |
+
return frames, report
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def format_report(report) -> str:
|
| 220 |
+
lines = []
|
| 221 |
+
n_out = report["output_frames"]
|
| 222 |
+
lines.append(
|
| 223 |
+
f"output {report['width']}x{report['height']} {n_out} frames seed={report['seed']}"
|
| 224 |
+
)
|
| 225 |
+
if "text_encoder_s" in report:
|
| 226 |
+
lines.append(f"text encoder {report['text_encoder_s'] * 1000:.1f} ms")
|
| 227 |
+
for index, seconds in enumerate(report["latent_frame_s"]):
|
| 228 |
+
lines.append(f"latent frame {index:02d} {seconds * 1000:.1f} ms")
|
| 229 |
+
lines.append(f"decoder {report['decoder_s'] * 1000:.1f} ms")
|
| 230 |
+
lines.append(f"total {report['total_s']:.3f} s")
|
| 231 |
+
if n_out:
|
| 232 |
+
lines.append(
|
| 233 |
+
f"amortized wall clock per output frame {report['total_s'] / n_out * 1000:.1f} ms"
|
| 234 |
+
)
|
| 235 |
+
lines.append(f"MLX allocator peak {report['mlx_peak_bytes'] / 1e9:.3f} GB")
|
| 236 |
+
lines.append(f"process RSS peak {report['rss_peak_bytes'] / 1e9:.3f} GB")
|
| 237 |
+
return "\n".join(lines)
|
fastframes_mlx/pixel_decoder.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Causal ConvNeXt pixel decoder (FastFrames section 3.2).
|
| 2 |
+
|
| 3 |
+
Reads the DiT's pre-head states and the head's time modulation and emits RGB.
|
| 4 |
+
The released FastFrames-S weights are float32. A latent clip of ``F`` frames on
|
| 5 |
+
the ``(F, 30, 52)`` grid becomes ``4F`` RGB frames at 480x832; the pipeline
|
| 6 |
+
keeps the last 81 (drops the three frames of causal warmup).
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import math
|
| 12 |
+
|
| 13 |
+
import mlx.core as mx
|
| 14 |
+
import mlx.nn as nn
|
| 15 |
+
|
| 16 |
+
from .dit import unpatchify
|
| 17 |
+
from .ops import conv3d_ncdhw, depthwise_conv3d_ncdhw, gelu_erf, layer_norm
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class CausalConv3d(nn.Module):
|
| 21 |
+
"""Left-only temporal pad, symmetric spatial pad, then conv."""
|
| 22 |
+
|
| 23 |
+
def __init__(self, in_channels, out_channels, kernel_size, groups=1):
|
| 24 |
+
super().__init__()
|
| 25 |
+
if isinstance(kernel_size, int):
|
| 26 |
+
kernel_size = (kernel_size, kernel_size, kernel_size)
|
| 27 |
+
self.kernel = tuple(kernel_size)
|
| 28 |
+
self.groups = groups
|
| 29 |
+
self.in_channels = in_channels
|
| 30 |
+
ic = in_channels // groups
|
| 31 |
+
self.weight = mx.zeros((out_channels, *self.kernel, ic))
|
| 32 |
+
self.bias = mx.zeros((out_channels,))
|
| 33 |
+
|
| 34 |
+
def __call__(self, x):
|
| 35 |
+
pt, ph, pw = (k // 2 for k in self.kernel)
|
| 36 |
+
if pt or ph or pw:
|
| 37 |
+
x = mx.pad(x, [(0, 0), (0, 0), (2 * pt, 0), (ph, ph), (pw, pw)])
|
| 38 |
+
if self.groups == self.in_channels and self.groups > 1:
|
| 39 |
+
return depthwise_conv3d_ncdhw(x, self.weight, self.bias)
|
| 40 |
+
return conv3d_ncdhw(x, self.weight, self.bias)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class ChannelLayerNorm(nn.Module):
|
| 44 |
+
def __init__(self, channels, eps=1e-6):
|
| 45 |
+
super().__init__()
|
| 46 |
+
self.eps = eps
|
| 47 |
+
self.weight = mx.ones((channels,))
|
| 48 |
+
self.bias = mx.zeros((channels,))
|
| 49 |
+
|
| 50 |
+
def __call__(self, x):
|
| 51 |
+
y = mx.transpose(x, (0, 2, 3, 4, 1))
|
| 52 |
+
y = layer_norm(y, self.eps, self.weight, self.bias)
|
| 53 |
+
return mx.transpose(y, (0, 4, 1, 2, 3))
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class CausalResBlock(nn.Module):
|
| 57 |
+
def __init__(self, dim, ffn_mult=4):
|
| 58 |
+
super().__init__()
|
| 59 |
+
self.dw = CausalConv3d(dim, dim, 3, groups=dim)
|
| 60 |
+
self.norm = ChannelLayerNorm(dim)
|
| 61 |
+
self.pw1 = CausalConv3d(dim, dim * ffn_mult, 1)
|
| 62 |
+
self.pw2 = CausalConv3d(dim * ffn_mult, dim, 1)
|
| 63 |
+
|
| 64 |
+
def __call__(self, x):
|
| 65 |
+
h = self.dw(x)
|
| 66 |
+
h = self.norm(h)
|
| 67 |
+
h = self.pw1(h)
|
| 68 |
+
h = gelu_erf(h)
|
| 69 |
+
h = self.pw2(h)
|
| 70 |
+
return x + h
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class PixelDecoder(nn.Module):
|
| 74 |
+
def __init__(
|
| 75 |
+
self,
|
| 76 |
+
dim=1536,
|
| 77 |
+
pixel_patch_size=(4, 16, 16),
|
| 78 |
+
out_dim=3,
|
| 79 |
+
n_low_blocks=2,
|
| 80 |
+
n_high_blocks=1,
|
| 81 |
+
ffn_mult=4,
|
| 82 |
+
eps=1e-6,
|
| 83 |
+
):
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.dim = dim
|
| 86 |
+
self.out_dim = out_dim
|
| 87 |
+
self.patch_size = tuple(pixel_patch_size)
|
| 88 |
+
self.eps = eps
|
| 89 |
+
self.modulation = mx.zeros((1, 2, dim))
|
| 90 |
+
self.entry_norm = ChannelLayerNorm(dim, eps)
|
| 91 |
+
self.entry_pw = CausalConv3d(dim, dim, 1)
|
| 92 |
+
self.conv_low = [CausalResBlock(dim, ffn_mult) for _ in range(n_low_blocks)]
|
| 93 |
+
self.conv_high = [CausalResBlock(dim, ffn_mult) for _ in range(n_high_blocks)]
|
| 94 |
+
self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * out_dim)
|
| 95 |
+
|
| 96 |
+
def __call__(self, x, e_mod, grid):
|
| 97 |
+
"""``x`` ``[B, F*L, C]``, ``e_mod`` ``[B, F, 1, C]``, ``grid`` ``(F, H, W)``."""
|
| 98 |
+
num_frames = e_mod.shape[1]
|
| 99 |
+
frame_tokens = x.shape[1] // num_frames
|
| 100 |
+
mod = self.modulation.reshape(1, 1, 2, -1) + e_mod
|
| 101 |
+
shift, scale = mx.split(mod, 2, axis=2)
|
| 102 |
+
h = layer_norm(x, self.eps).reshape(x.shape[0], num_frames, frame_tokens, self.dim)
|
| 103 |
+
h = h * (1.0 + scale) + shift
|
| 104 |
+
f, gh, gw = grid
|
| 105 |
+
value = h.reshape(x.shape[0], f, gh, gw, self.dim)
|
| 106 |
+
value = mx.transpose(value, (0, 4, 1, 2, 3))
|
| 107 |
+
value = self.entry_pw(self.entry_norm(value))
|
| 108 |
+
for block in self.conv_low:
|
| 109 |
+
value = block(value)
|
| 110 |
+
for block in self.conv_high:
|
| 111 |
+
value = block(value)
|
| 112 |
+
b, _, f, gh, gw = value.shape
|
| 113 |
+
tokens = mx.transpose(value, (0, 2, 3, 4, 1)).reshape(b, f, gh * gw, self.dim)
|
| 114 |
+
tokens = self.fc_out(tokens).reshape(b, f * gh * gw, -1)
|
| 115 |
+
pixels = unpatchify(tokens, (f, gh, gw), self.patch_size, self.out_dim)
|
| 116 |
+
return mx.transpose(pixels, (0, 2, 1, 3, 4))
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def align_frames(pixels, num_frames: int):
|
| 120 |
+
"""Keep the last ``num_frames`` frames (84 -> 81 for the released recipe)."""
|
| 121 |
+
if pixels.shape[1] < num_frames:
|
| 122 |
+
raise ValueError(f"decoder produced {pixels.shape[1]} frames, need at least {num_frames}")
|
| 123 |
+
return pixels[:, pixels.shape[1] - num_frames :]
|
fastframes_mlx/rope.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""3-D rotary embeddings for the Wan DiT.
|
| 2 |
+
|
| 3 |
+
Frequencies match ``rope_params`` / ``causal_rope_apply``: the angles are
|
| 4 |
+
computed in NumPy float64 (the same libm PyTorch uses for the complex128
|
| 5 |
+
reference), then stored as float32. Metal has no float64, so the rotation
|
| 6 |
+
itself runs in float32 on whatever device holds the activations.
|
| 7 |
+
``start_frame`` is the latent-frame index of the first token.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
|
| 14 |
+
import mlx.core as mx
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _rope_component(max_seq_len: int, dim: int, theta: float = 10000.0):
|
| 18 |
+
"""Cos/sin of shape ``[max_seq_len, dim // 2]`` for one RoPE axis.
|
| 19 |
+
|
| 20 |
+
``dim`` is the argument of Wan's ``rope_params`` (twice the number of
|
| 21 |
+
complex frequencies). Exponents are ``arange(0, dim, 2) / dim``.
|
| 22 |
+
"""
|
| 23 |
+
idx = np.arange(0, dim, 2, dtype=np.float64)
|
| 24 |
+
inv = 1.0 / np.power(np.float64(theta), idx / np.float64(dim))
|
| 25 |
+
freqs = np.arange(max_seq_len, dtype=np.float64)[:, None] * inv[None, :]
|
| 26 |
+
cos = np.cos(freqs).astype(np.float32)
|
| 27 |
+
sin = np.sin(freqs).astype(np.float32)
|
| 28 |
+
return mx.array(cos), mx.array(sin)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class RopeTables:
|
| 32 |
+
def __init__(self, head_dim: int, max_seq_len: int = 1024, theta: float = 10000.0):
|
| 33 |
+
# Same axis widths as WanModel.rope_table: (d - 4*(d//6), 2*(d//6), 2*(d//6)).
|
| 34 |
+
d = head_dim
|
| 35 |
+
axis_dims = (d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6))
|
| 36 |
+
self.parts = tuple(_rope_component(max_seq_len, dim, theta) for dim in axis_dims)
|
| 37 |
+
self.head_dim = head_dim
|
| 38 |
+
|
| 39 |
+
def apply(self, x, frames: int, height: int, width: int, start_frame: int = 0):
|
| 40 |
+
"""Rotate ``x`` of shape ``[B, L, heads, head_dim]``.
|
| 41 |
+
|
| 42 |
+
Tokens past ``frames * height * width`` are left unchanged (padding).
|
| 43 |
+
"""
|
| 44 |
+
b, length, heads, dim = x.shape
|
| 45 |
+
seq = frames * height * width
|
| 46 |
+
if seq > length:
|
| 47 |
+
raise ValueError(f"grid {frames}x{height}x{width}={seq} exceeds sequence {length}")
|
| 48 |
+
# float32, never float64: the default device may be the GPU.
|
| 49 |
+
pairs = x[:, :seq].astype(mx.float32).reshape(b, seq, heads, dim // 2, 2)
|
| 50 |
+
real, imag = pairs[..., 0], pairs[..., 1]
|
| 51 |
+
cos, sin = self._grid(frames, height, width, start_frame)
|
| 52 |
+
cos = cos.reshape(1, seq, 1, dim // 2).astype(mx.float32)
|
| 53 |
+
sin = sin.reshape(1, seq, 1, dim // 2).astype(mx.float32)
|
| 54 |
+
out_r = real * cos - imag * sin
|
| 55 |
+
out_i = real * sin + imag * cos
|
| 56 |
+
rotated = mx.stack([out_r, out_i], axis=-1).reshape(b, seq, heads, dim).astype(x.dtype)
|
| 57 |
+
if seq == length:
|
| 58 |
+
return rotated
|
| 59 |
+
return mx.concatenate([rotated, x[:, seq:]], axis=1)
|
| 60 |
+
|
| 61 |
+
def _grid(self, frames, height, width, start_frame):
|
| 62 |
+
seq = frames * height * width
|
| 63 |
+
pieces_c = []
|
| 64 |
+
pieces_s = []
|
| 65 |
+
# temporal, then height, then width — the order Wan concatenates.
|
| 66 |
+
specs = (
|
| 67 |
+
(self.parts[0], start_frame, "t"),
|
| 68 |
+
(self.parts[1], 0, "h"),
|
| 69 |
+
(self.parts[2], 0, "w"),
|
| 70 |
+
)
|
| 71 |
+
for (cos, sin), offset, axis in specs:
|
| 72 |
+
n = {"t": frames, "h": height, "w": width}[axis]
|
| 73 |
+
c = cos[offset : offset + n]
|
| 74 |
+
s = sin[offset : offset + n]
|
| 75 |
+
if axis == "t":
|
| 76 |
+
shape = (frames, height, width, c.shape[-1])
|
| 77 |
+
c = mx.broadcast_to(c[:, None, None, :], shape)
|
| 78 |
+
s = mx.broadcast_to(s[:, None, None, :], shape)
|
| 79 |
+
elif axis == "h":
|
| 80 |
+
shape = (frames, height, width, c.shape[-1])
|
| 81 |
+
c = mx.broadcast_to(c[None, :, None, :], shape)
|
| 82 |
+
s = mx.broadcast_to(s[None, :, None, :], shape)
|
| 83 |
+
else:
|
| 84 |
+
shape = (frames, height, width, c.shape[-1])
|
| 85 |
+
c = mx.broadcast_to(c[None, None, :, :], shape)
|
| 86 |
+
s = mx.broadcast_to(s[None, None, :, :], shape)
|
| 87 |
+
pieces_c.append(c.reshape(seq, -1))
|
| 88 |
+
pieces_s.append(s.reshape(seq, -1))
|
| 89 |
+
return mx.concatenate(pieces_c, axis=-1), mx.concatenate(pieces_s, axis=-1)
|
fastframes_mlx/scheduler.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Rectified-flow schedule used by Wan2.1 and FastFrames.
|
| 2 |
+
|
| 3 |
+
``x_t = (1 - sigma) x_0 + sigma * noise``. A model timestep is snapped to the
|
| 4 |
+
nearest training-grid entry (1000 steps, shift 5). ``flow_to_x0`` is float32:
|
| 5 |
+
Metal has no float64, and the sigma grid is already float32. The official
|
| 6 |
+
scheduler promotes that same grid to float64 for the subtraction; the
|
| 7 |
+
difference is below the parity tolerance.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import mlx.core as mx
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class FlowMatchScheduler:
|
| 16 |
+
def __init__(self, num_inference_steps: int = 1000, num_train_timesteps: int = 1000, shift: float = 5.0):
|
| 17 |
+
self.num_train_timesteps = num_train_timesteps
|
| 18 |
+
self.shift = shift
|
| 19 |
+
self.set_timesteps(num_inference_steps)
|
| 20 |
+
|
| 21 |
+
def set_timesteps(self, num_inference_steps: int = 100):
|
| 22 |
+
sigmas = mx.linspace(1.0, 0.0, num_inference_steps + 1)[:-1]
|
| 23 |
+
self.sigmas = self.shift * sigmas / (1.0 + (self.shift - 1.0) * sigmas)
|
| 24 |
+
self.timesteps = self.sigmas * self.num_train_timesteps
|
| 25 |
+
|
| 26 |
+
def _index(self, timestep: mx.array) -> mx.array:
|
| 27 |
+
timestep = timestep.astype(mx.float32).reshape(-1)
|
| 28 |
+
dist = mx.abs(self.timesteps.reshape(1, -1) - timestep.reshape(-1, 1))
|
| 29 |
+
return mx.argmin(dist, axis=1)
|
| 30 |
+
|
| 31 |
+
def add_noise(self, original, noise, timestep):
|
| 32 |
+
sigma = self.sigmas[self._index(timestep)].reshape(-1, 1, 1, 1).astype(noise.dtype)
|
| 33 |
+
return ((1.0 - sigma) * original.astype(noise.dtype) + sigma * noise).astype(noise.dtype)
|
| 34 |
+
|
| 35 |
+
def flow_to_x0(self, flow, xt, timestep):
|
| 36 |
+
dtype = flow.dtype
|
| 37 |
+
sigma = self.sigmas[self._index(timestep)].reshape(-1, 1, 1, 1).astype(mx.float32)
|
| 38 |
+
out = xt.astype(mx.float32) - sigma * flow.astype(mx.float32)
|
| 39 |
+
return out.astype(dtype)
|
fastframes_mlx/t5.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""umT5-XXL encoder used by Wan2.1 (encoder only, 24 layers, 4096-d).
|
| 2 |
+
|
| 3 |
+
Relative attention bias is per layer (``shared_pos=False`` in the Wan checkpoint).
|
| 4 |
+
Attention is unscaled. Softmax is computed in float32. Dropout is omitted
|
| 5 |
+
because generation runs the encoder in eval mode.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
import mlx.core as mx
|
| 13 |
+
import mlx.nn as nn
|
| 14 |
+
|
| 15 |
+
from .ops import gelu_tanh, rms_norm
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _bucket(rel_pos, num_buckets, bidirectional, max_dist):
|
| 19 |
+
if bidirectional:
|
| 20 |
+
half = num_buckets // 2
|
| 21 |
+
rel_buckets = (rel_pos > 0).astype(mx.int32) * half
|
| 22 |
+
rel_pos = mx.abs(rel_pos)
|
| 23 |
+
n_buckets = half
|
| 24 |
+
else:
|
| 25 |
+
n_buckets = num_buckets
|
| 26 |
+
rel_buckets = mx.zeros(rel_pos.shape, dtype=mx.int32)
|
| 27 |
+
rel_pos = -mx.minimum(rel_pos, mx.zeros(rel_pos.shape, dtype=rel_pos.dtype))
|
| 28 |
+
max_exact = n_buckets // 2
|
| 29 |
+
# log(0) is only used for positions that take the "small" branch.
|
| 30 |
+
safe = mx.maximum(rel_pos.astype(mx.float32), 1.0)
|
| 31 |
+
large = max_exact + (
|
| 32 |
+
mx.log(safe / max_exact) / math.log(max_dist / max_exact) * (n_buckets - max_exact)
|
| 33 |
+
).astype(mx.int32)
|
| 34 |
+
large = mx.minimum(large, mx.full(large.shape, n_buckets - 1, dtype=mx.int32))
|
| 35 |
+
small = rel_pos < max_exact
|
| 36 |
+
return rel_buckets + mx.where(small, rel_pos.astype(mx.int32), large)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class T5RelativeEmbedding(nn.Module):
|
| 40 |
+
def __init__(self, num_buckets, num_heads, bidirectional=True, max_dist=128):
|
| 41 |
+
super().__init__()
|
| 42 |
+
self.num_buckets = num_buckets
|
| 43 |
+
self.num_heads = num_heads
|
| 44 |
+
self.bidirectional = bidirectional
|
| 45 |
+
self.max_dist = max_dist
|
| 46 |
+
self.embedding = nn.Embedding(num_buckets, num_heads)
|
| 47 |
+
|
| 48 |
+
def __call__(self, lq, lk):
|
| 49 |
+
rel = mx.arange(lk)[None, :] - mx.arange(lq)[:, None]
|
| 50 |
+
buckets = _bucket(rel, self.num_buckets, self.bidirectional, self.max_dist)
|
| 51 |
+
values = self.embedding(buckets)
|
| 52 |
+
return values.transpose(2, 0, 1)[None]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class T5Attention(nn.Module):
|
| 56 |
+
def __init__(self, dim, dim_attn, num_heads):
|
| 57 |
+
super().__init__()
|
| 58 |
+
self.num_heads = num_heads
|
| 59 |
+
self.head_dim = dim_attn // num_heads
|
| 60 |
+
self.q = nn.Linear(dim, dim_attn, bias=False)
|
| 61 |
+
self.k = nn.Linear(dim, dim_attn, bias=False)
|
| 62 |
+
self.v = nn.Linear(dim, dim_attn, bias=False)
|
| 63 |
+
self.o = nn.Linear(dim_attn, dim, bias=False)
|
| 64 |
+
|
| 65 |
+
def __call__(self, x, mask=None, pos_bias=None):
|
| 66 |
+
b = x.shape[0]
|
| 67 |
+
n, c = self.num_heads, self.head_dim
|
| 68 |
+
q = self.q(x).reshape(b, -1, n, c).transpose(0, 2, 1, 3)
|
| 69 |
+
k = self.k(x).reshape(b, -1, n, c).transpose(0, 2, 1, 3)
|
| 70 |
+
v = self.v(x).reshape(b, -1, n, c).transpose(0, 2, 1, 3)
|
| 71 |
+
scores = mx.matmul(q, mx.swapaxes(k, -1, -2))
|
| 72 |
+
if pos_bias is not None:
|
| 73 |
+
scores = scores + pos_bias.astype(scores.dtype)
|
| 74 |
+
if mask is not None:
|
| 75 |
+
# mask is 1 for tokens that may be attended to.
|
| 76 |
+
fill = mx.array(mx.finfo(scores.dtype).min, dtype=scores.dtype)
|
| 77 |
+
scores = mx.where(mask.reshape(b, 1, 1, -1) == 0, fill, scores)
|
| 78 |
+
weights = mx.softmax(scores.astype(mx.float32), axis=-1).astype(scores.dtype)
|
| 79 |
+
out = mx.matmul(weights, v).transpose(0, 2, 1, 3).reshape(b, -1, n * c)
|
| 80 |
+
return self.o(out)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class T5FeedForward(nn.Module):
|
| 84 |
+
def __init__(self, dim, dim_ffn):
|
| 85 |
+
super().__init__()
|
| 86 |
+
self.gate = nn.Linear(dim, dim_ffn, bias=False)
|
| 87 |
+
self.fc1 = nn.Linear(dim, dim_ffn, bias=False)
|
| 88 |
+
self.fc2 = nn.Linear(dim_ffn, dim, bias=False)
|
| 89 |
+
|
| 90 |
+
def __call__(self, x):
|
| 91 |
+
return self.fc2(self.fc1(x) * gelu_tanh(self.gate(x)))
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class T5LayerNorm(nn.Module):
|
| 95 |
+
def __init__(self, dim, eps=1e-6):
|
| 96 |
+
super().__init__()
|
| 97 |
+
self.eps = eps
|
| 98 |
+
self.weight = mx.ones((dim,))
|
| 99 |
+
|
| 100 |
+
def __call__(self, x):
|
| 101 |
+
return rms_norm(x, self.weight, self.eps)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class T5Block(nn.Module):
|
| 105 |
+
def __init__(self, dim, dim_attn, dim_ffn, num_heads, num_buckets, shared_pos):
|
| 106 |
+
super().__init__()
|
| 107 |
+
self.shared_pos = shared_pos
|
| 108 |
+
self.norm1 = T5LayerNorm(dim)
|
| 109 |
+
self.attn = T5Attention(dim, dim_attn, num_heads)
|
| 110 |
+
self.norm2 = T5LayerNorm(dim)
|
| 111 |
+
self.ffn = T5FeedForward(dim, dim_ffn)
|
| 112 |
+
self.pos_embedding = None if shared_pos else T5RelativeEmbedding(num_buckets, num_heads, True)
|
| 113 |
+
|
| 114 |
+
def __call__(self, x, mask=None, pos_bias=None):
|
| 115 |
+
e = pos_bias if self.shared_pos else self.pos_embedding(x.shape[1], x.shape[1])
|
| 116 |
+
x = x + self.attn(self.norm1(x), mask=mask, pos_bias=e)
|
| 117 |
+
x = x + self.ffn(self.norm2(x))
|
| 118 |
+
return x
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
class T5Encoder(nn.Module):
|
| 122 |
+
def __init__(
|
| 123 |
+
self,
|
| 124 |
+
vocab_size=256384,
|
| 125 |
+
dim=4096,
|
| 126 |
+
dim_attn=4096,
|
| 127 |
+
dim_ffn=10240,
|
| 128 |
+
num_heads=64,
|
| 129 |
+
num_layers=24,
|
| 130 |
+
num_buckets=32,
|
| 131 |
+
shared_pos=False,
|
| 132 |
+
):
|
| 133 |
+
super().__init__()
|
| 134 |
+
self.dim = dim
|
| 135 |
+
self.shared_pos = shared_pos
|
| 136 |
+
self.token_embedding = nn.Embedding(vocab_size, dim)
|
| 137 |
+
self.pos_embedding = T5RelativeEmbedding(num_buckets, num_heads, True) if shared_pos else None
|
| 138 |
+
self.blocks = [
|
| 139 |
+
T5Block(dim, dim_attn, dim_ffn, num_heads, num_buckets, shared_pos) for _ in range(num_layers)
|
| 140 |
+
]
|
| 141 |
+
self.norm = T5LayerNorm(dim)
|
| 142 |
+
|
| 143 |
+
def __call__(self, ids, mask=None):
|
| 144 |
+
x = self.token_embedding(ids)
|
| 145 |
+
e = self.pos_embedding(x.shape[1], x.shape[1]) if self.shared_pos else None
|
| 146 |
+
for block in self.blocks:
|
| 147 |
+
x = block(x, mask=mask, pos_bias=e)
|
| 148 |
+
return self.norm(x)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def umt5_xxl():
|
| 152 |
+
return T5Encoder()
|
fastframes_mlx/tokenizer.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Prompt tokenizer for the Wan umT5 encoder.
|
| 2 |
+
|
| 3 |
+
Cleaning matches Wan's ``HuggingfaceTokenizer`` with ``clean='whitespace'``
|
| 4 |
+
(ftfy, HTML unescape, collapsed whitespace). Token ids come from the
|
| 5 |
+
``tokenizer.json`` shipped in ``Wan-AI/Wan2.1-T2V-1.3B`` (``google/umt5-xxl``).
|
| 6 |
+
Sequences are truncated or padded to 512 tokens. The tokenizer's template
|
| 7 |
+
appends ``</s>``.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import html
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import mlx.core as mx
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def basic_clean(text: str) -> str:
|
| 19 |
+
import ftfy
|
| 20 |
+
|
| 21 |
+
text = ftfy.fix_text(text)
|
| 22 |
+
text = html.unescape(html.unescape(text))
|
| 23 |
+
return text.strip()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def whitespace_clean(text: str) -> str:
|
| 27 |
+
import regex as re
|
| 28 |
+
|
| 29 |
+
text = re.sub(r"\s+", " ", text)
|
| 30 |
+
return text.strip()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def clean_prompt(text: str) -> str:
|
| 34 |
+
return whitespace_clean(basic_clean(text))
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class UMT5Tokenizer:
|
| 38 |
+
def __init__(self, tokenizer_json: str | Path, seq_len: int = 512):
|
| 39 |
+
from tokenizers import Tokenizer
|
| 40 |
+
|
| 41 |
+
self.tokenizer = Tokenizer.from_file(str(tokenizer_json))
|
| 42 |
+
self.seq_len = seq_len
|
| 43 |
+
pad = self.tokenizer.token_to_id("<pad>")
|
| 44 |
+
self.pad_id = 0 if pad is None else pad
|
| 45 |
+
|
| 46 |
+
def encode(self, prompts):
|
| 47 |
+
if isinstance(prompts, str):
|
| 48 |
+
prompts = [prompts]
|
| 49 |
+
ids = []
|
| 50 |
+
mask = []
|
| 51 |
+
for prompt in prompts:
|
| 52 |
+
encoded = self.tokenizer.encode(clean_prompt(prompt))
|
| 53 |
+
row = list(encoded.ids)
|
| 54 |
+
if len(row) > self.seq_len:
|
| 55 |
+
row = row[: self.seq_len]
|
| 56 |
+
valid = len(row)
|
| 57 |
+
if len(row) < self.seq_len:
|
| 58 |
+
row = row + [self.pad_id] * (self.seq_len - len(row))
|
| 59 |
+
ids.append(row)
|
| 60 |
+
mask.append([1] * valid + [0] * (self.seq_len - valid))
|
| 61 |
+
return mx.array(ids, dtype=mx.int32), mx.array(mask, dtype=mx.int32)
|
fastframes_mlx/vae_layers.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from ml-explore/mlx-examples video/wan2.1 (MIT).
|
| 2 |
+
# Upstream copyright: Copyright © 2026 Apple Inc.
|
| 3 |
+
# Copyright © 2026 Apple Inc.
|
| 4 |
+
|
| 5 |
+
"""
|
| 6 |
+
Building blocks for Wan2.1 VAE.
|
| 7 |
+
|
| 8 |
+
All layers use channels-last format (NTHWC) as required by MLX.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import mlx.core as mx
|
| 12 |
+
import mlx.nn as nn
|
| 13 |
+
|
| 14 |
+
# Temporal cache depth: 2 frames for causal conv with kernel_size=3 along time.
|
| 15 |
+
CACHE_T = 2
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _normalize_tuple(value, n):
|
| 19 |
+
if isinstance(value, int):
|
| 20 |
+
return (value,) * n
|
| 21 |
+
return tuple(value)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def create_cache_entry(x, existing_cache=None):
|
| 25 |
+
"""Build temporal cache from the last CACHE_T frames of x, merging with existing cache."""
|
| 26 |
+
t = x.shape[1]
|
| 27 |
+
if t >= CACHE_T:
|
| 28 |
+
return x[:, -CACHE_T:, :, :, :]
|
| 29 |
+
else:
|
| 30 |
+
cache_x = x[:, -t:, :, :, :]
|
| 31 |
+
if existing_cache is not None:
|
| 32 |
+
old_frames = existing_cache[:, -(CACHE_T - t) :, :, :, :]
|
| 33 |
+
return mx.concatenate([old_frames, cache_x], axis=1)
|
| 34 |
+
else:
|
| 35 |
+
pad_t = CACHE_T - t
|
| 36 |
+
zeros = mx.zeros((x.shape[0], pad_t, *x.shape[2:]), dtype=x.dtype)
|
| 37 |
+
return mx.concatenate([zeros, cache_x], axis=1)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class CausalConv3d(nn.Module):
|
| 41 |
+
def __init__(
|
| 42 |
+
self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=True
|
| 43 |
+
):
|
| 44 |
+
super().__init__()
|
| 45 |
+
self.in_channels = in_channels
|
| 46 |
+
self.out_channels = out_channels
|
| 47 |
+
self.kernel_size = _normalize_tuple(kernel_size, 3)
|
| 48 |
+
self.stride = _normalize_tuple(stride, 3)
|
| 49 |
+
self.padding = _normalize_tuple(padding, 3)
|
| 50 |
+
self._temporal_pad = self.padding[0] * 2
|
| 51 |
+
self._spatial_pad_h = self.padding[1]
|
| 52 |
+
self._spatial_pad_w = self.padding[2]
|
| 53 |
+
|
| 54 |
+
scale = (
|
| 55 |
+
1.0
|
| 56 |
+
/ (
|
| 57 |
+
in_channels
|
| 58 |
+
* self.kernel_size[0]
|
| 59 |
+
* self.kernel_size[1]
|
| 60 |
+
* self.kernel_size[2]
|
| 61 |
+
)
|
| 62 |
+
** 0.5
|
| 63 |
+
)
|
| 64 |
+
self.weight = mx.random.uniform(
|
| 65 |
+
low=-scale,
|
| 66 |
+
high=scale,
|
| 67 |
+
shape=(out_channels, *self.kernel_size, in_channels),
|
| 68 |
+
)
|
| 69 |
+
if bias:
|
| 70 |
+
self.bias = mx.zeros((out_channels,))
|
| 71 |
+
|
| 72 |
+
def __call__(self, x, cache_x=None):
|
| 73 |
+
# Causal temporal padding (left-only), then symmetric spatial padding, then conv with padding=0.
|
| 74 |
+
temporal_pad = self._temporal_pad
|
| 75 |
+
if cache_x is not None and self._temporal_pad > 0:
|
| 76 |
+
x = mx.concatenate([cache_x, x], axis=1)
|
| 77 |
+
temporal_pad = max(0, self._temporal_pad - cache_x.shape[1])
|
| 78 |
+
|
| 79 |
+
if temporal_pad > 0:
|
| 80 |
+
x = mx.pad(x, [(0, 0), (temporal_pad, 0), (0, 0), (0, 0), (0, 0)])
|
| 81 |
+
|
| 82 |
+
if self._spatial_pad_h > 0 or self._spatial_pad_w > 0:
|
| 83 |
+
x = mx.pad(
|
| 84 |
+
x,
|
| 85 |
+
[
|
| 86 |
+
(0, 0),
|
| 87 |
+
(0, 0),
|
| 88 |
+
(self._spatial_pad_h, self._spatial_pad_h),
|
| 89 |
+
(self._spatial_pad_w, self._spatial_pad_w),
|
| 90 |
+
(0, 0),
|
| 91 |
+
],
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
y = mx.conv3d(x, self.weight, stride=self.stride, padding=0)
|
| 95 |
+
if "bias" in self:
|
| 96 |
+
y = y + self.bias
|
| 97 |
+
return y
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
class Resample(nn.Module):
|
| 101 |
+
def __init__(self, dim, mode):
|
| 102 |
+
assert mode in (
|
| 103 |
+
"upsample2d",
|
| 104 |
+
"upsample3d",
|
| 105 |
+
"downsample2d",
|
| 106 |
+
"downsample3d",
|
| 107 |
+
)
|
| 108 |
+
super().__init__()
|
| 109 |
+
self.dim = dim
|
| 110 |
+
self.mode = mode
|
| 111 |
+
|
| 112 |
+
if mode == "upsample2d":
|
| 113 |
+
self.upsample = nn.Upsample(scale_factor=(2.0, 2.0), mode="nearest")
|
| 114 |
+
self.conv = nn.Conv2d(
|
| 115 |
+
dim, dim // 2, kernel_size=3, stride=1, padding=0, bias=True
|
| 116 |
+
)
|
| 117 |
+
elif mode == "upsample3d":
|
| 118 |
+
self.upsample = nn.Upsample(scale_factor=(2.0, 2.0), mode="nearest")
|
| 119 |
+
self.conv = nn.Conv2d(
|
| 120 |
+
dim, dim // 2, kernel_size=3, stride=1, padding=0, bias=True
|
| 121 |
+
)
|
| 122 |
+
self.time_conv = CausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0))
|
| 123 |
+
elif mode in ("downsample2d", "downsample3d"):
|
| 124 |
+
self.conv = nn.Conv2d(
|
| 125 |
+
dim, dim, kernel_size=3, stride=2, padding=0, bias=True
|
| 126 |
+
)
|
| 127 |
+
if mode == "downsample3d":
|
| 128 |
+
self.time_conv = CausalConv3d(
|
| 129 |
+
dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
def __call__(self, x, cache=None):
|
| 133 |
+
b, t, h, w, c = x.shape
|
| 134 |
+
new_cache = None
|
| 135 |
+
|
| 136 |
+
if self.mode == "upsample3d":
|
| 137 |
+
if cache is None:
|
| 138 |
+
new_cache = mx.zeros((b, CACHE_T, h, w, c), dtype=x.dtype)
|
| 139 |
+
else:
|
| 140 |
+
cache_input = x
|
| 141 |
+
x = self.time_conv(x, cache)
|
| 142 |
+
new_cache = create_cache_entry(cache_input, cache)
|
| 143 |
+
x = x.reshape(b, t, h, w, 2, c)
|
| 144 |
+
x = x.transpose(0, 1, 4, 2, 3, 5)
|
| 145 |
+
x = x.reshape(b, t * 2, h, w, c)
|
| 146 |
+
|
| 147 |
+
t_out = x.shape[1]
|
| 148 |
+
c_out = x.shape[4]
|
| 149 |
+
x = x.reshape(b * t_out, x.shape[2], x.shape[3], c_out)
|
| 150 |
+
|
| 151 |
+
if self.mode in ("upsample2d", "upsample3d"):
|
| 152 |
+
x = self.upsample(x)
|
| 153 |
+
x = mx.pad(x, [(0, 0), (1, 1), (1, 1), (0, 0)])
|
| 154 |
+
x = self.conv(x)
|
| 155 |
+
elif self.mode in ("downsample2d", "downsample3d"):
|
| 156 |
+
x = mx.pad(x, [(0, 0), (0, 1), (0, 1), (0, 0)])
|
| 157 |
+
x = self.conv(x)
|
| 158 |
+
|
| 159 |
+
x = x.reshape(b, t_out, x.shape[1], x.shape[2], x.shape[3])
|
| 160 |
+
|
| 161 |
+
if self.mode == "downsample3d":
|
| 162 |
+
if cache is None:
|
| 163 |
+
new_cache = x
|
| 164 |
+
else:
|
| 165 |
+
x_with_cache = mx.concatenate([cache[:, -1:, :, :, :], x], axis=1)
|
| 166 |
+
new_cache = x[:, -1:, :, :, :]
|
| 167 |
+
x = self.time_conv(x_with_cache, None)
|
| 168 |
+
|
| 169 |
+
return x, new_cache
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
class ResidualBlock(nn.Module):
|
| 173 |
+
def __init__(self, in_dim, out_dim):
|
| 174 |
+
super().__init__()
|
| 175 |
+
self.in_dim = in_dim
|
| 176 |
+
self.out_dim = out_dim
|
| 177 |
+
self.norm1 = nn.RMSNorm(in_dim, eps=1e-12)
|
| 178 |
+
self.conv1 = CausalConv3d(in_dim, out_dim, 3, padding=1)
|
| 179 |
+
self.norm2 = nn.RMSNorm(out_dim, eps=1e-12)
|
| 180 |
+
self.conv2 = CausalConv3d(out_dim, out_dim, 3, padding=1)
|
| 181 |
+
if in_dim != out_dim:
|
| 182 |
+
self.shortcut = CausalConv3d(in_dim, out_dim, 1)
|
| 183 |
+
else:
|
| 184 |
+
self.shortcut = None
|
| 185 |
+
|
| 186 |
+
def __call__(self, x, cache1, cache2):
|
| 187 |
+
if self.shortcut is not None:
|
| 188 |
+
h = self.shortcut(x)
|
| 189 |
+
else:
|
| 190 |
+
h = x
|
| 191 |
+
|
| 192 |
+
residual = self.norm1(x)
|
| 193 |
+
residual = nn.silu(residual)
|
| 194 |
+
cache_input = residual
|
| 195 |
+
residual = self.conv1(residual, cache1)
|
| 196 |
+
new_cache1 = create_cache_entry(cache_input, cache1)
|
| 197 |
+
|
| 198 |
+
residual = self.norm2(residual)
|
| 199 |
+
residual = nn.silu(residual)
|
| 200 |
+
cache_input = residual
|
| 201 |
+
residual = self.conv2(residual, cache2)
|
| 202 |
+
new_cache2 = create_cache_entry(cache_input, cache2)
|
| 203 |
+
|
| 204 |
+
return h + residual, new_cache1, new_cache2
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
class AttentionBlock(nn.Module):
|
| 208 |
+
def __init__(self, dim):
|
| 209 |
+
super().__init__()
|
| 210 |
+
self.dim = dim
|
| 211 |
+
self.norm = nn.RMSNorm(dim, eps=1e-12)
|
| 212 |
+
self.to_qkv = nn.Linear(dim, dim * 3)
|
| 213 |
+
self.proj = nn.Linear(dim, dim)
|
| 214 |
+
|
| 215 |
+
def __call__(self, x):
|
| 216 |
+
identity = x
|
| 217 |
+
b, t, h, w, c = x.shape
|
| 218 |
+
x = x.reshape(b * t, h, w, c)
|
| 219 |
+
x = self.norm(x)
|
| 220 |
+
qkv = self.to_qkv(x)
|
| 221 |
+
qkv = qkv.reshape(b * t, h * w, 3, c)
|
| 222 |
+
q, k, v = qkv[:, :, 0, :], qkv[:, :, 1, :], qkv[:, :, 2, :]
|
| 223 |
+
q = q.reshape(b * t, 1, h * w, c)
|
| 224 |
+
k = k.reshape(b * t, 1, h * w, c)
|
| 225 |
+
v = v.reshape(b * t, 1, h * w, c)
|
| 226 |
+
scale = c**-0.5
|
| 227 |
+
attn = mx.fast.scaled_dot_product_attention(q, k, v, scale=scale)
|
| 228 |
+
attn = attn.squeeze(1).reshape(b * t, h, w, c)
|
| 229 |
+
out = self.proj(attn)
|
| 230 |
+
out = out.reshape(b, t, h, w, c)
|
| 231 |
+
return out + identity
|
fastframes_mlx/video.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Write an RGB clip with ffmpeg (libx264, yuv420p)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import subprocess
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def save_mp4(path, frames_uint8, fps: int = 16):
|
| 12 |
+
"""``frames_uint8`` is ``[T, H, W, 3]`` uint8, even width and height."""
|
| 13 |
+
frames = np.ascontiguousarray(frames_uint8)
|
| 14 |
+
if frames.ndim != 4 or frames.shape[-1] != 3:
|
| 15 |
+
raise ValueError(f"expected [T,H,W,3], got {frames.shape}")
|
| 16 |
+
if frames.shape[1] % 2 or frames.shape[2] % 2:
|
| 17 |
+
raise ValueError("yuv420p needs even height and width")
|
| 18 |
+
path = Path(path)
|
| 19 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 20 |
+
height, width = frames.shape[1], frames.shape[2]
|
| 21 |
+
cmd = [
|
| 22 |
+
"ffmpeg",
|
| 23 |
+
"-y",
|
| 24 |
+
"-loglevel",
|
| 25 |
+
"error",
|
| 26 |
+
"-f",
|
| 27 |
+
"rawvideo",
|
| 28 |
+
"-pix_fmt",
|
| 29 |
+
"rgb24",
|
| 30 |
+
"-s",
|
| 31 |
+
f"{width}x{height}",
|
| 32 |
+
"-r",
|
| 33 |
+
str(fps),
|
| 34 |
+
"-i",
|
| 35 |
+
"-",
|
| 36 |
+
"-an",
|
| 37 |
+
"-c:v",
|
| 38 |
+
"libx264",
|
| 39 |
+
"-pix_fmt",
|
| 40 |
+
"yuv420p",
|
| 41 |
+
str(path),
|
| 42 |
+
]
|
| 43 |
+
proc = subprocess.run(cmd, input=frames.tobytes(), capture_output=True)
|
| 44 |
+
if proc.returncode != 0:
|
| 45 |
+
raise RuntimeError(proc.stderr.decode() or "ffmpeg failed")
|
| 46 |
+
return path
|
fastframes_mlx/wan_vae.py
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from ml-explore/mlx-examples video/wan2.1 (MIT).
|
| 2 |
+
# Upstream copyright: Copyright © 2026 Apple Inc.
|
| 3 |
+
# Copyright © 2026 Apple Inc.
|
| 4 |
+
|
| 5 |
+
"""
|
| 6 |
+
Wan2.1 VAE encoder and decoder.
|
| 7 |
+
|
| 8 |
+
Encodes video frames to latents and decodes latents to video frames
|
| 9 |
+
using chunked processing with causal temporal caching.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import re
|
| 13 |
+
from typing import Dict, List, Optional
|
| 14 |
+
|
| 15 |
+
import mlx.core as mx
|
| 16 |
+
import mlx.nn as nn
|
| 17 |
+
|
| 18 |
+
from .vae_layers import (
|
| 19 |
+
AttentionBlock,
|
| 20 |
+
CausalConv3d,
|
| 21 |
+
Resample,
|
| 22 |
+
ResidualBlock,
|
| 23 |
+
create_cache_entry,
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class Decoder3d(nn.Module):
|
| 28 |
+
"""
|
| 29 |
+
VAE Decoder for video generation.
|
| 30 |
+
|
| 31 |
+
Input: [B, T, H/8, W/8, z_dim] (channels-last)
|
| 32 |
+
Output: [B, T*4, H, W, 3]
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
dim: int = 96,
|
| 38 |
+
z_dim: int = 16,
|
| 39 |
+
dim_mult: Optional[List[int]] = None,
|
| 40 |
+
num_res_blocks: int = 2,
|
| 41 |
+
attn_scales: Optional[List[float]] = None,
|
| 42 |
+
temporal_upsample: Optional[List[bool]] = None,
|
| 43 |
+
):
|
| 44 |
+
super().__init__()
|
| 45 |
+
if dim_mult is None:
|
| 46 |
+
dim_mult = [1, 2, 4, 4]
|
| 47 |
+
if attn_scales is None:
|
| 48 |
+
attn_scales = []
|
| 49 |
+
if temporal_upsample is None:
|
| 50 |
+
temporal_upsample = [True, True, False]
|
| 51 |
+
|
| 52 |
+
self.dim = dim
|
| 53 |
+
self.z_dim = z_dim
|
| 54 |
+
self.dim_mult = dim_mult
|
| 55 |
+
self.num_res_blocks = num_res_blocks
|
| 56 |
+
self.temporal_upsample = temporal_upsample
|
| 57 |
+
|
| 58 |
+
dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]]
|
| 59 |
+
scale = 1.0 / 2 ** (len(dim_mult) - 2)
|
| 60 |
+
|
| 61 |
+
self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1)
|
| 62 |
+
|
| 63 |
+
self.middle_res1 = ResidualBlock(dims[0], dims[0])
|
| 64 |
+
self.middle_attn = AttentionBlock(dims[0])
|
| 65 |
+
self.middle_res2 = ResidualBlock(dims[0], dims[0])
|
| 66 |
+
|
| 67 |
+
# Build upsample stages as nested lists
|
| 68 |
+
self.upsamples = []
|
| 69 |
+
for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
|
| 70 |
+
stage = []
|
| 71 |
+
if i == 1 or i == 2 or i == 3:
|
| 72 |
+
in_dim = in_dim // 2
|
| 73 |
+
for j in range(num_res_blocks + 1):
|
| 74 |
+
stage.append(ResidualBlock(in_dim, out_dim))
|
| 75 |
+
if scale in attn_scales:
|
| 76 |
+
stage.append(AttentionBlock(out_dim))
|
| 77 |
+
in_dim = out_dim
|
| 78 |
+
if i != len(dim_mult) - 1:
|
| 79 |
+
mode = "upsample3d" if temporal_upsample[i] else "upsample2d"
|
| 80 |
+
stage.append(Resample(out_dim, mode=mode))
|
| 81 |
+
scale *= 2.0
|
| 82 |
+
self.upsamples.append(stage)
|
| 83 |
+
|
| 84 |
+
self.head_norm = nn.RMSNorm(dims[-1], eps=1e-12)
|
| 85 |
+
self.head_conv = CausalConv3d(dims[-1], 3, 3, padding=1)
|
| 86 |
+
|
| 87 |
+
# Count temporal cache slots from architecture
|
| 88 |
+
n = 1 + 2 + 2 # conv1, middle_res1, middle_res2
|
| 89 |
+
for stage in self.upsamples:
|
| 90 |
+
for layer in stage:
|
| 91 |
+
if isinstance(layer, ResidualBlock):
|
| 92 |
+
n += 2
|
| 93 |
+
elif isinstance(layer, Resample) and hasattr(layer, "time_conv"):
|
| 94 |
+
n += 1
|
| 95 |
+
n += 1 # head_conv
|
| 96 |
+
self.num_cache_slots = n
|
| 97 |
+
|
| 98 |
+
def __call__(self, x, feat_cache):
|
| 99 |
+
cache_idx = 0
|
| 100 |
+
new_cache = []
|
| 101 |
+
|
| 102 |
+
cache_input = x
|
| 103 |
+
x = self.conv1(x, feat_cache[cache_idx])
|
| 104 |
+
new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx]))
|
| 105 |
+
cache_idx += 1
|
| 106 |
+
|
| 107 |
+
x, c1, c2 = self.middle_res1(
|
| 108 |
+
x, feat_cache[cache_idx], feat_cache[cache_idx + 1]
|
| 109 |
+
)
|
| 110 |
+
new_cache.append(c1)
|
| 111 |
+
new_cache.append(c2)
|
| 112 |
+
cache_idx += 2
|
| 113 |
+
|
| 114 |
+
x = self.middle_attn(x)
|
| 115 |
+
|
| 116 |
+
x, c1, c2 = self.middle_res2(
|
| 117 |
+
x, feat_cache[cache_idx], feat_cache[cache_idx + 1]
|
| 118 |
+
)
|
| 119 |
+
new_cache.append(c1)
|
| 120 |
+
new_cache.append(c2)
|
| 121 |
+
cache_idx += 2
|
| 122 |
+
|
| 123 |
+
for stage in self.upsamples:
|
| 124 |
+
for layer in stage:
|
| 125 |
+
if isinstance(layer, ResidualBlock):
|
| 126 |
+
x, c1, c2 = layer(
|
| 127 |
+
x, feat_cache[cache_idx], feat_cache[cache_idx + 1]
|
| 128 |
+
)
|
| 129 |
+
new_cache.append(c1)
|
| 130 |
+
new_cache.append(c2)
|
| 131 |
+
cache_idx += 2
|
| 132 |
+
elif isinstance(layer, AttentionBlock):
|
| 133 |
+
x = layer(x)
|
| 134 |
+
elif isinstance(layer, Resample):
|
| 135 |
+
x, c = layer(x, feat_cache[cache_idx])
|
| 136 |
+
if c is not None:
|
| 137 |
+
new_cache.append(c)
|
| 138 |
+
cache_idx += 1
|
| 139 |
+
|
| 140 |
+
x = self.head_norm(x)
|
| 141 |
+
x = nn.silu(x)
|
| 142 |
+
cache_input = x
|
| 143 |
+
x = self.head_conv(x, feat_cache[cache_idx])
|
| 144 |
+
new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx]))
|
| 145 |
+
cache_idx += 1
|
| 146 |
+
|
| 147 |
+
return x, new_cache
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class Encoder3d(nn.Module):
|
| 151 |
+
"""
|
| 152 |
+
VAE Encoder for video generation.
|
| 153 |
+
|
| 154 |
+
Input: [B, T, H, W, 3] (channels-last)
|
| 155 |
+
Output: [B, T', H/8, W/8, z_dim*2]
|
| 156 |
+
"""
|
| 157 |
+
|
| 158 |
+
def __init__(
|
| 159 |
+
self,
|
| 160 |
+
dim: int = 96,
|
| 161 |
+
z_dim: int = 16,
|
| 162 |
+
dim_mult: Optional[List[int]] = None,
|
| 163 |
+
num_res_blocks: int = 2,
|
| 164 |
+
attn_scales: Optional[List[float]] = None,
|
| 165 |
+
temporal_downsample: Optional[List[bool]] = None,
|
| 166 |
+
):
|
| 167 |
+
super().__init__()
|
| 168 |
+
if dim_mult is None:
|
| 169 |
+
dim_mult = [1, 2, 4, 4]
|
| 170 |
+
if attn_scales is None:
|
| 171 |
+
attn_scales = []
|
| 172 |
+
if temporal_downsample is None:
|
| 173 |
+
temporal_downsample = [False, True, True]
|
| 174 |
+
|
| 175 |
+
self.dim = dim
|
| 176 |
+
self.z_dim = z_dim
|
| 177 |
+
self.dim_mult = dim_mult
|
| 178 |
+
self.num_res_blocks = num_res_blocks
|
| 179 |
+
self.temporal_downsample = temporal_downsample
|
| 180 |
+
|
| 181 |
+
dims = [dim * u for u in [1] + dim_mult]
|
| 182 |
+
scale = 1.0
|
| 183 |
+
|
| 184 |
+
self.conv1 = CausalConv3d(3, dims[0], 3, padding=1)
|
| 185 |
+
|
| 186 |
+
# Build downsample stages as nested lists
|
| 187 |
+
self.downsamples = []
|
| 188 |
+
for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
|
| 189 |
+
stage = []
|
| 190 |
+
for j in range(num_res_blocks):
|
| 191 |
+
stage.append(ResidualBlock(in_dim, out_dim))
|
| 192 |
+
if scale in attn_scales:
|
| 193 |
+
stage.append(AttentionBlock(out_dim))
|
| 194 |
+
in_dim = out_dim
|
| 195 |
+
if i != len(dim_mult) - 1:
|
| 196 |
+
mode = "downsample3d" if temporal_downsample[i] else "downsample2d"
|
| 197 |
+
stage.append(Resample(out_dim, mode=mode))
|
| 198 |
+
scale /= 2.0
|
| 199 |
+
self.downsamples.append(stage)
|
| 200 |
+
|
| 201 |
+
self.middle_res1 = ResidualBlock(dims[-1], dims[-1])
|
| 202 |
+
self.middle_attn = AttentionBlock(dims[-1])
|
| 203 |
+
self.middle_res2 = ResidualBlock(dims[-1], dims[-1])
|
| 204 |
+
|
| 205 |
+
self.head_norm = nn.RMSNorm(dims[-1], eps=1e-12)
|
| 206 |
+
self.head_conv = CausalConv3d(dims[-1], z_dim * 2, 3, padding=1)
|
| 207 |
+
|
| 208 |
+
# Count temporal cache slots from architecture
|
| 209 |
+
n = 1 # conv1
|
| 210 |
+
for stage in self.downsamples:
|
| 211 |
+
for layer in stage:
|
| 212 |
+
if isinstance(layer, ResidualBlock):
|
| 213 |
+
n += 2
|
| 214 |
+
elif isinstance(layer, Resample) and hasattr(layer, "time_conv"):
|
| 215 |
+
n += 1
|
| 216 |
+
n += 2 + 2 + 1 # middle_res1, middle_res2, head_conv
|
| 217 |
+
self.num_cache_slots = n
|
| 218 |
+
|
| 219 |
+
def __call__(self, x, feat_cache):
|
| 220 |
+
cache_idx = 0
|
| 221 |
+
new_cache = []
|
| 222 |
+
|
| 223 |
+
cache_input = x
|
| 224 |
+
x = self.conv1(x, feat_cache[cache_idx])
|
| 225 |
+
new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx]))
|
| 226 |
+
cache_idx += 1
|
| 227 |
+
|
| 228 |
+
for stage in self.downsamples:
|
| 229 |
+
for layer in stage:
|
| 230 |
+
if isinstance(layer, ResidualBlock):
|
| 231 |
+
x, c1, c2 = layer(
|
| 232 |
+
x, feat_cache[cache_idx], feat_cache[cache_idx + 1]
|
| 233 |
+
)
|
| 234 |
+
new_cache.append(c1)
|
| 235 |
+
new_cache.append(c2)
|
| 236 |
+
cache_idx += 2
|
| 237 |
+
elif isinstance(layer, AttentionBlock):
|
| 238 |
+
x = layer(x)
|
| 239 |
+
elif isinstance(layer, Resample):
|
| 240 |
+
x, c = layer(x, feat_cache[cache_idx])
|
| 241 |
+
if c is not None:
|
| 242 |
+
new_cache.append(c)
|
| 243 |
+
cache_idx += 1
|
| 244 |
+
|
| 245 |
+
x, c1, c2 = self.middle_res1(
|
| 246 |
+
x, feat_cache[cache_idx], feat_cache[cache_idx + 1]
|
| 247 |
+
)
|
| 248 |
+
new_cache.append(c1)
|
| 249 |
+
new_cache.append(c2)
|
| 250 |
+
cache_idx += 2
|
| 251 |
+
|
| 252 |
+
x = self.middle_attn(x)
|
| 253 |
+
|
| 254 |
+
x, c1, c2 = self.middle_res2(
|
| 255 |
+
x, feat_cache[cache_idx], feat_cache[cache_idx + 1]
|
| 256 |
+
)
|
| 257 |
+
new_cache.append(c1)
|
| 258 |
+
new_cache.append(c2)
|
| 259 |
+
cache_idx += 2
|
| 260 |
+
|
| 261 |
+
x = self.head_norm(x)
|
| 262 |
+
x = nn.silu(x)
|
| 263 |
+
cache_input = x
|
| 264 |
+
x = self.head_conv(x, feat_cache[cache_idx])
|
| 265 |
+
new_cache.append(create_cache_entry(cache_input, feat_cache[cache_idx]))
|
| 266 |
+
cache_idx += 1
|
| 267 |
+
|
| 268 |
+
return x, new_cache
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
class WanVAE(nn.Module):
|
| 272 |
+
"""
|
| 273 |
+
High-level VAE wrapper for Wan2.1.
|
| 274 |
+
|
| 275 |
+
Encode: [F, H, W, C] video -> [F', H/8, W/8, z_dim] latent
|
| 276 |
+
Decode: [F, H, W, C] latent -> [F*4, H*8, W*8, 3] video clamped to [-1, 1]
|
| 277 |
+
"""
|
| 278 |
+
|
| 279 |
+
def __init__(self):
|
| 280 |
+
super().__init__()
|
| 281 |
+
self.encoder = Encoder3d()
|
| 282 |
+
self.conv1 = CausalConv3d(32, 32, 1)
|
| 283 |
+
self.decoder = Decoder3d()
|
| 284 |
+
self.conv2 = CausalConv3d(16, 16, 1)
|
| 285 |
+
|
| 286 |
+
self.mean = mx.array(
|
| 287 |
+
[
|
| 288 |
+
-0.7571,
|
| 289 |
+
-0.7089,
|
| 290 |
+
-0.9113,
|
| 291 |
+
0.1075,
|
| 292 |
+
-0.1745,
|
| 293 |
+
0.9653,
|
| 294 |
+
-0.1517,
|
| 295 |
+
1.5508,
|
| 296 |
+
0.4134,
|
| 297 |
+
-0.0715,
|
| 298 |
+
0.5517,
|
| 299 |
+
-0.3632,
|
| 300 |
+
-0.1922,
|
| 301 |
+
-0.9497,
|
| 302 |
+
0.2503,
|
| 303 |
+
-0.2921,
|
| 304 |
+
]
|
| 305 |
+
)
|
| 306 |
+
self.std = mx.array(
|
| 307 |
+
[
|
| 308 |
+
2.8184,
|
| 309 |
+
1.4541,
|
| 310 |
+
2.3275,
|
| 311 |
+
2.6558,
|
| 312 |
+
1.2196,
|
| 313 |
+
1.7708,
|
| 314 |
+
2.6052,
|
| 315 |
+
2.0743,
|
| 316 |
+
3.2687,
|
| 317 |
+
2.1526,
|
| 318 |
+
2.8652,
|
| 319 |
+
1.5579,
|
| 320 |
+
1.6382,
|
| 321 |
+
1.1253,
|
| 322 |
+
2.8251,
|
| 323 |
+
1.9160,
|
| 324 |
+
]
|
| 325 |
+
)
|
| 326 |
+
self.z_dim = 16
|
| 327 |
+
# Pre-compile for the frame-by-frame loop: avoids recompiling each frame.
|
| 328 |
+
self._compiled_decode = mx.compile(self.decoder.__call__)
|
| 329 |
+
self._compiled_encode = mx.compile(self.encoder.__call__)
|
| 330 |
+
|
| 331 |
+
def decode(self, z: mx.array) -> mx.array:
|
| 332 |
+
"""
|
| 333 |
+
Decode latent to video.
|
| 334 |
+
|
| 335 |
+
Args:
|
| 336 |
+
z: Latent tensor [F, H, W, C] (channels-last)
|
| 337 |
+
|
| 338 |
+
Returns:
|
| 339 |
+
Video tensor [F, H, W, C] clamped to [-1, 1] (channels-last)
|
| 340 |
+
"""
|
| 341 |
+
# Add batch dim: [F, H, W, C] -> [1, F, H, W, C]
|
| 342 |
+
z = z[None]
|
| 343 |
+
|
| 344 |
+
# Unscale latents
|
| 345 |
+
scale = 1.0 / self.std
|
| 346 |
+
z = z / scale.reshape(1, 1, 1, 1, self.z_dim) + self.mean.reshape(
|
| 347 |
+
1, 1, 1, 1, self.z_dim
|
| 348 |
+
)
|
| 349 |
+
|
| 350 |
+
# Pre-decoder conv
|
| 351 |
+
x = self.conv2(z)
|
| 352 |
+
|
| 353 |
+
# Decode one frame at a time. mx.eval per frame releases intermediates, keeping memory bounded.
|
| 354 |
+
num_frames = x.shape[1]
|
| 355 |
+
feat_cache = [None] * self.decoder.num_cache_slots
|
| 356 |
+
outputs = []
|
| 357 |
+
|
| 358 |
+
for i in range(num_frames):
|
| 359 |
+
frame = x[:, i : i + 1, :, :, :]
|
| 360 |
+
out_frame, feat_cache = self._compiled_decode(frame, feat_cache)
|
| 361 |
+
mx.eval(out_frame)
|
| 362 |
+
outputs.append(out_frame)
|
| 363 |
+
|
| 364 |
+
out = mx.concatenate(outputs, axis=1)
|
| 365 |
+
out = mx.clip(out, -1.0, 1.0)
|
| 366 |
+
|
| 367 |
+
# Remove batch dim: [1, F, H, W, C] -> [F, H, W, C]
|
| 368 |
+
return out[0]
|
| 369 |
+
|
| 370 |
+
def encode(self, x: mx.array) -> mx.array:
|
| 371 |
+
"""
|
| 372 |
+
Encode video to latent.
|
| 373 |
+
|
| 374 |
+
Args:
|
| 375 |
+
x: Video tensor [F, H, W, C] (channels-last)
|
| 376 |
+
|
| 377 |
+
Returns:
|
| 378 |
+
Latent tensor [F', H/8, W/8, C] (channels-last)
|
| 379 |
+
"""
|
| 380 |
+
# Add batch dim: [F, H, W, C] -> [1, F, H, W, C]
|
| 381 |
+
x = x[None]
|
| 382 |
+
|
| 383 |
+
num_frames = x.shape[1]
|
| 384 |
+
feat_cache = [None] * self.encoder.num_cache_slots
|
| 385 |
+
outputs = []
|
| 386 |
+
|
| 387 |
+
# First chunk is 1 frame (causal init), subsequent chunks are 4 frames (matching VAE temporal stride).
|
| 388 |
+
i = 0
|
| 389 |
+
chunk_idx = 0
|
| 390 |
+
while i < num_frames:
|
| 391 |
+
if chunk_idx == 0:
|
| 392 |
+
chunk = x[:, i : i + 1, :, :, :]
|
| 393 |
+
i += 1
|
| 394 |
+
else:
|
| 395 |
+
chunk = x[:, i : i + 4, :, :, :]
|
| 396 |
+
i += 4
|
| 397 |
+
|
| 398 |
+
out_chunk, feat_cache = self._compiled_encode(chunk, feat_cache)
|
| 399 |
+
mx.eval(out_chunk)
|
| 400 |
+
outputs.append(out_chunk)
|
| 401 |
+
chunk_idx += 1
|
| 402 |
+
|
| 403 |
+
out = mx.concatenate(outputs, axis=1)
|
| 404 |
+
|
| 405 |
+
# Post-encoder conv and extract mu
|
| 406 |
+
out = self.conv1(out)
|
| 407 |
+
mu = out[:, :, :, :, : self.z_dim]
|
| 408 |
+
|
| 409 |
+
# Scale: (mu - mean) * (1/std)
|
| 410 |
+
scale = 1.0 / self.std
|
| 411 |
+
mu = (mu - self.mean.reshape(1, 1, 1, 1, self.z_dim)) * scale.reshape(
|
| 412 |
+
1, 1, 1, 1, self.z_dim
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
# Remove batch dim: [1, F', H', W', C] -> [F', H', W', C]
|
| 416 |
+
return mu[0]
|
| 417 |
+
|
| 418 |
+
@staticmethod
|
| 419 |
+
def sanitize(weights: Dict[str, mx.array]) -> Dict[str, mx.array]:
|
| 420 |
+
"""Remap PyTorch VAE keys to MLX format."""
|
| 421 |
+
remapped = {}
|
| 422 |
+
for key, value in weights.items():
|
| 423 |
+
new_key = key
|
| 424 |
+
|
| 425 |
+
# Transpose convolution weights
|
| 426 |
+
if "weight" in new_key:
|
| 427 |
+
if len(value.shape) == 5:
|
| 428 |
+
value = mx.transpose(value, (0, 2, 3, 4, 1))
|
| 429 |
+
elif len(value.shape) == 4:
|
| 430 |
+
value = mx.transpose(value, (0, 2, 3, 1))
|
| 431 |
+
|
| 432 |
+
new_key = new_key.replace(".gamma", ".weight")
|
| 433 |
+
new_key = new_key.replace("decoder.middle.0.", "decoder.middle_res1.")
|
| 434 |
+
new_key = new_key.replace("decoder.middle.1.", "decoder.middle_attn.")
|
| 435 |
+
new_key = new_key.replace("decoder.middle.2.", "decoder.middle_res2.")
|
| 436 |
+
new_key = new_key.replace("decoder.head.0.", "decoder.head_norm.")
|
| 437 |
+
new_key = new_key.replace("decoder.head.2.", "decoder.head_conv.")
|
| 438 |
+
new_key = new_key.replace("encoder.middle.0.", "encoder.middle_res1.")
|
| 439 |
+
new_key = new_key.replace("encoder.middle.1.", "encoder.middle_attn.")
|
| 440 |
+
new_key = new_key.replace("encoder.middle.2.", "encoder.middle_res2.")
|
| 441 |
+
new_key = new_key.replace("encoder.head.0.", "encoder.head_norm.")
|
| 442 |
+
new_key = new_key.replace("encoder.head.2.", "encoder.head_conv.")
|
| 443 |
+
|
| 444 |
+
if "decoder.upsamples." in new_key:
|
| 445 |
+
new_key = _map_vae_upsample_key(new_key)
|
| 446 |
+
if "encoder.downsamples." in new_key:
|
| 447 |
+
new_key = _map_vae_downsample_key(new_key)
|
| 448 |
+
|
| 449 |
+
new_key = re.sub(r"\.residual\.0\.", ".norm1.", new_key)
|
| 450 |
+
new_key = re.sub(r"\.residual\.2\.", ".conv1.", new_key)
|
| 451 |
+
new_key = re.sub(r"\.residual\.3\.", ".norm2.", new_key)
|
| 452 |
+
new_key = re.sub(r"\.residual\.6\.", ".conv2.", new_key)
|
| 453 |
+
|
| 454 |
+
# Resample conv: .resample.1. -> .conv.
|
| 455 |
+
new_key = re.sub(r"\.resample\.1\.", ".conv.", new_key)
|
| 456 |
+
|
| 457 |
+
# Squeeze 1x1 conv weights to 2D for nn.Linear (to_qkv, proj)
|
| 458 |
+
if ("to_qkv" in new_key or "proj" in new_key) and "weight" in new_key:
|
| 459 |
+
if (
|
| 460 |
+
len(value.shape) == 4
|
| 461 |
+
and value.shape[1] == 1
|
| 462 |
+
and value.shape[2] == 1
|
| 463 |
+
):
|
| 464 |
+
value = value.reshape(value.shape[0], value.shape[3])
|
| 465 |
+
|
| 466 |
+
# Squeeze norm weights to 1D (required — nn.RMSNorm expects 1D)
|
| 467 |
+
if "norm" in new_key and "weight" in new_key:
|
| 468 |
+
if len(value.shape) > 1:
|
| 469 |
+
value = mx.squeeze(value)
|
| 470 |
+
|
| 471 |
+
remapped[new_key] = value
|
| 472 |
+
return remapped
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def _map_vae_upsample_key(key: str) -> str:
|
| 476 |
+
match = re.match(r"decoder\.upsamples\.(\d+)\.(.*)", key)
|
| 477 |
+
if not match:
|
| 478 |
+
return key
|
| 479 |
+
|
| 480 |
+
layer_idx = int(match.group(1))
|
| 481 |
+
rest = match.group(2)
|
| 482 |
+
|
| 483 |
+
# Decoder stages: (num_res_blocks+1) ResBlocks + 1 Resample each, except last (no Resample).
|
| 484 |
+
# Assumes attn_scales=[] (Wan2.1 default — no AttentionBlocks in stages).
|
| 485 |
+
num_res_blocks, num_stages = 2, 4
|
| 486 |
+
stage_sizes = [num_res_blocks + 2] * (num_stages - 1) + [num_res_blocks + 1]
|
| 487 |
+
stage = 0
|
| 488 |
+
local_idx = layer_idx
|
| 489 |
+
|
| 490 |
+
for s, size in enumerate(stage_sizes):
|
| 491 |
+
if local_idx < size:
|
| 492 |
+
stage = s
|
| 493 |
+
break
|
| 494 |
+
local_idx -= size
|
| 495 |
+
|
| 496 |
+
return f"decoder.upsamples.{stage}.{local_idx}.{rest}"
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def _map_vae_downsample_key(key: str) -> str:
|
| 500 |
+
match = re.match(r"encoder\.downsamples\.(\d+)\.(.*)", key)
|
| 501 |
+
if not match:
|
| 502 |
+
return key
|
| 503 |
+
|
| 504 |
+
layer_idx = int(match.group(1))
|
| 505 |
+
rest = match.group(2)
|
| 506 |
+
|
| 507 |
+
# Encoder stages: num_res_blocks ResBlocks + 1 Resample each, except last (no Resample).
|
| 508 |
+
# Assumes attn_scales=[] (Wan2.1 default — no AttentionBlocks in stages).
|
| 509 |
+
num_res_blocks, num_stages = 2, 4
|
| 510 |
+
stage_sizes = [num_res_blocks + 1] * (num_stages - 1) + [num_res_blocks]
|
| 511 |
+
stage = 0
|
| 512 |
+
local_idx = layer_idx
|
| 513 |
+
|
| 514 |
+
for s, size in enumerate(stage_sizes):
|
| 515 |
+
if local_idx < size:
|
| 516 |
+
stage = s
|
| 517 |
+
break
|
| 518 |
+
local_idx -= size
|
| 519 |
+
|
| 520 |
+
return f"encoder.downsamples.{stage}.{local_idx}.{rest}"
|
fastframes_mlx/weights.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Load official checkpoints into MLX modules.
|
| 2 |
+
|
| 3 |
+
Weights stay in the Hugging Face cache. This package does not re-host them.
|
| 4 |
+
|
| 5 |
+
* FastFrames-S refiner and pixel decoder: ``apple/FastFrames-S`` (apple-amlr).
|
| 6 |
+
* Generator: ``zhuhz22/Causal-Forcing`` ``causal-forcing++/framewise-2step.pt``
|
| 7 |
+
(Apache-2.0), EMA key ``generator_ema``.
|
| 8 |
+
* umT5 encoder, tokenizer, and Wan VAE: ``Wan-AI/Wan2.1-T2V-1.3B`` (Apache-2.0).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import mlx.core as mx
|
| 16 |
+
import mlx.nn as nn
|
| 17 |
+
from mlx.utils import tree_flatten
|
| 18 |
+
|
| 19 |
+
from .config import WAN_1_3B, FastFramesConfig
|
| 20 |
+
from .dit import CausalWanModel
|
| 21 |
+
from .kv_refiner import KVRefinementNetwork
|
| 22 |
+
from .pixel_decoder import PixelDecoder
|
| 23 |
+
from .t5 import umt5_xxl
|
| 24 |
+
|
| 25 |
+
_PREFIXES = ("model._fsdp_wrapped_module.", "_fsdp_wrapped_module.", "model.")
|
| 26 |
+
|
| 27 |
+
FILES = {
|
| 28 |
+
"refiner": ("apple/FastFrames-S", "fastframes-s/kv_refiner.safetensors"),
|
| 29 |
+
"decoder": ("apple/FastFrames-S", "fastframes-s/pixel_decoder.safetensors"),
|
| 30 |
+
"generator": ("zhuhz22/Causal-Forcing", "causal-forcing++/framewise-2step.pt"),
|
| 31 |
+
"t5": ("Wan-AI/Wan2.1-T2V-1.3B", "models_t5_umt5-xxl-enc-bf16.pth"),
|
| 32 |
+
"vae": ("Wan-AI/Wan2.1-T2V-1.3B", "Wan2.1_VAE.pth"),
|
| 33 |
+
"tokenizer": ("Wan-AI/Wan2.1-T2V-1.3B", "google/umt5-xxl/tokenizer.json"),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def download(name: str, cache_dir: str | Path | None = None) -> Path:
|
| 38 |
+
from huggingface_hub import hf_hub_download
|
| 39 |
+
|
| 40 |
+
repo, filename = FILES[name]
|
| 41 |
+
path = hf_hub_download(repo_id=repo, filename=filename, cache_dir=None if cache_dir is None else str(cache_dir))
|
| 42 |
+
return Path(path)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _strip(name: str) -> str:
|
| 46 |
+
for prefix in _PREFIXES:
|
| 47 |
+
if name.startswith(prefix):
|
| 48 |
+
return name[len(prefix) :]
|
| 49 |
+
return name
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def torch_to_mx(tensor):
|
| 53 |
+
import torch
|
| 54 |
+
|
| 55 |
+
tensor = tensor.detach().cpu().contiguous()
|
| 56 |
+
if tensor.dtype == torch.bfloat16:
|
| 57 |
+
return mx.array(tensor.view(torch.uint16).numpy()).view(mx.bfloat16)
|
| 58 |
+
return mx.array(tensor.numpy())
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def load_torch_state(path: Path, key: str | None = None) -> dict:
|
| 62 |
+
import torch
|
| 63 |
+
|
| 64 |
+
state = torch.load(path, map_location="cpu", weights_only=True)
|
| 65 |
+
if key is not None:
|
| 66 |
+
if key not in state:
|
| 67 |
+
raise KeyError(f"{path} has no {key!r}; top-level keys: {list(state)[:12]}")
|
| 68 |
+
state = state[key]
|
| 69 |
+
return {_strip(k): v for k, v in state.items()}
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def load_safetensors(path: Path) -> dict:
|
| 73 |
+
return dict(mx.load(str(path)))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _replace(key: str, pairs) -> str:
|
| 77 |
+
for src, dst in pairs:
|
| 78 |
+
key = key.replace(src, dst)
|
| 79 |
+
return key
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
DIT_KEYS = (
|
| 83 |
+
("text_embedding.0.", "text_embed_fc1."),
|
| 84 |
+
("text_embedding.2.", "text_embed_fc2."),
|
| 85 |
+
("time_embedding.0.", "time_embed_fc1."),
|
| 86 |
+
("time_embedding.2.", "time_embed_fc2."),
|
| 87 |
+
("time_projection.1.", "time_proj_fc."),
|
| 88 |
+
(".ffn.0.", ".ffn.fc1."),
|
| 89 |
+
(".ffn.2.", ".ffn.fc2."),
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
REFINER_KEYS = (
|
| 93 |
+
("t_embed.mlp.0.", "t_embed.fc1."),
|
| 94 |
+
("t_embed.mlp.2.", "t_embed.fc2."),
|
| 95 |
+
(".mlp.0.", ".mlp.fc1."),
|
| 96 |
+
(".mlp.2.", ".mlp.fc2."),
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
T5_DROP = ("ffn.gate.1.", "dropout")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
T5_KEYS = (("ffn.gate.0.", "ffn.gate."),)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _prepare(raw, renames=(), transpose_conv=True, drop_substrings=()):
|
| 106 |
+
prepared = {}
|
| 107 |
+
for key, value in raw.items():
|
| 108 |
+
if "freqs" in key or any(part in key for part in drop_substrings):
|
| 109 |
+
continue
|
| 110 |
+
key = _strip(key)
|
| 111 |
+
key = _replace(key, renames)
|
| 112 |
+
if not isinstance(value, mx.array):
|
| 113 |
+
value = torch_to_mx(value)
|
| 114 |
+
if transpose_conv and value.ndim == 5 and key.endswith("weight"):
|
| 115 |
+
value = mx.transpose(value, (0, 2, 3, 4, 1))
|
| 116 |
+
prepared[key] = value
|
| 117 |
+
return prepared
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _set_nested(module, key, value):
|
| 121 |
+
"""Assign one parameter.
|
| 122 |
+
|
| 123 |
+
``load_weights`` turns all-digit path components into list indexes, which
|
| 124 |
+
breaks ``phi_proj.768`` (a dict keyed by width). Walk lists and dicts
|
| 125 |
+
explicitly instead.
|
| 126 |
+
"""
|
| 127 |
+
obj = module
|
| 128 |
+
parts = key.split(".")
|
| 129 |
+
for part in parts[:-1]:
|
| 130 |
+
if isinstance(obj, list):
|
| 131 |
+
obj = obj[int(part)]
|
| 132 |
+
elif isinstance(obj, dict):
|
| 133 |
+
obj = obj[part]
|
| 134 |
+
else:
|
| 135 |
+
obj = getattr(obj, part)
|
| 136 |
+
leaf = parts[-1]
|
| 137 |
+
if isinstance(obj, dict):
|
| 138 |
+
obj[leaf] = value
|
| 139 |
+
else:
|
| 140 |
+
setattr(obj, leaf, value)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def assign(module, arrays, drop_substrings=()):
|
| 144 |
+
filtered = {}
|
| 145 |
+
for key, value in arrays.items():
|
| 146 |
+
if any(part in key for part in drop_substrings):
|
| 147 |
+
continue
|
| 148 |
+
filtered[key] = value
|
| 149 |
+
have = dict(tree_flatten(module.parameters()))
|
| 150 |
+
missing = sorted(set(have) - set(filtered))
|
| 151 |
+
extra = sorted(set(filtered) - set(have))
|
| 152 |
+
if missing or extra:
|
| 153 |
+
raise KeyError(
|
| 154 |
+
f"checkpoint does not match the module: "
|
| 155 |
+
f"{len(missing)} missing (e.g. {missing[:8]}), "
|
| 156 |
+
f"{len(extra)} unexpected (e.g. {extra[:8]})"
|
| 157 |
+
)
|
| 158 |
+
for key, value in filtered.items():
|
| 159 |
+
_set_nested(module, key, value)
|
| 160 |
+
mx.eval(module.parameters())
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def cast_floating(module, dtype):
|
| 164 |
+
items = [(key, value.astype(dtype)) for key, value in tree_flatten(module.parameters())]
|
| 165 |
+
for key, value in items:
|
| 166 |
+
_set_nested(module, key, value)
|
| 167 |
+
mx.eval(module.parameters())
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def quantize_linears(module, bits: int, group_size: int = 64):
|
| 171 |
+
"""Post-training quantization of ``nn.Linear`` only. Not a calibrated quant."""
|
| 172 |
+
|
| 173 |
+
def predicate(path, mod):
|
| 174 |
+
if not isinstance(mod, nn.Linear):
|
| 175 |
+
return False
|
| 176 |
+
# Group size must divide the input features. Skip the rare linear
|
| 177 |
+
# (none in FastFrames-S at group size 64) rather than failing the load.
|
| 178 |
+
return mod.weight.shape[-1] % group_size == 0
|
| 179 |
+
|
| 180 |
+
nn.quantize(module, bits=bits, group_size=group_size, class_predicate=predicate)
|
| 181 |
+
return module
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def load_fastframes_s(cache_dir=None, bits: int | None = None, group_size: int = 64, load_vae: bool = False):
|
| 185 |
+
"""Download (if needed) and build the FastFrames-S stack in bf16, decoder in fp32."""
|
| 186 |
+
cfg = FastFramesConfig()
|
| 187 |
+
dit_path = download("generator", cache_dir)
|
| 188 |
+
refiner_path = download("refiner", cache_dir)
|
| 189 |
+
decoder_path = download("decoder", cache_dir)
|
| 190 |
+
t5_path = download("t5", cache_dir)
|
| 191 |
+
tok_path = download("tokenizer", cache_dir)
|
| 192 |
+
|
| 193 |
+
dit = CausalWanModel(WAN_1_3B)
|
| 194 |
+
assign(dit, _prepare(load_torch_state(dit_path, key="generator_ema"), DIT_KEYS))
|
| 195 |
+
cast_floating(dit, mx.bfloat16)
|
| 196 |
+
if bits:
|
| 197 |
+
quantize_linears(dit, bits, group_size)
|
| 198 |
+
|
| 199 |
+
refiner = KVRefinementNetwork()
|
| 200 |
+
assign(refiner, _prepare(load_safetensors(refiner_path), REFINER_KEYS))
|
| 201 |
+
cast_floating(refiner, mx.bfloat16)
|
| 202 |
+
|
| 203 |
+
decoder = PixelDecoder()
|
| 204 |
+
assign(decoder, _prepare(load_safetensors(decoder_path)))
|
| 205 |
+
|
| 206 |
+
text = umt5_xxl()
|
| 207 |
+
assign(text, _prepare(load_torch_state(t5_path), T5_KEYS, drop_substrings=T5_DROP))
|
| 208 |
+
cast_floating(text, mx.bfloat16)
|
| 209 |
+
if bits:
|
| 210 |
+
quantize_linears(text, bits, group_size)
|
| 211 |
+
|
| 212 |
+
vae = None
|
| 213 |
+
if load_vae:
|
| 214 |
+
vae = load_vae_model(cache_dir)
|
| 215 |
+
return cfg, dit, refiner, decoder, text, tok_path, vae
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def load_vae_model(cache_dir=None):
|
| 219 |
+
from .wan_vae import WanVAE
|
| 220 |
+
|
| 221 |
+
path = download("vae", cache_dir)
|
| 222 |
+
state = load_torch_state(path)
|
| 223 |
+
arrays = {k: torch_to_mx(v) for k, v in state.items()}
|
| 224 |
+
model = WanVAE()
|
| 225 |
+
prepared = WanVAE.sanitize(arrays)
|
| 226 |
+
assign(model, prepared)
|
| 227 |
+
return model
|
pyproject.toml
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "fastframes-mlx"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "MLX port of Apple FastFrames-S for Apple Silicon. Weights are loaded from Hugging Face at runtime."
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = ">=3.10"
|
| 7 |
+
license = { text = "MIT" }
|
| 8 |
+
dependencies = [
|
| 9 |
+
"mlx>=0.29; sys_platform == 'darwin'",
|
| 10 |
+
"mlx[cpu]>=0.29; sys_platform == 'linux'",
|
| 11 |
+
"numpy",
|
| 12 |
+
"huggingface_hub",
|
| 13 |
+
"safetensors",
|
| 14 |
+
"torch",
|
| 15 |
+
"tokenizers",
|
| 16 |
+
"ftfy",
|
| 17 |
+
"regex",
|
| 18 |
+
"tqdm",
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
[project.optional-dependencies]
|
| 22 |
+
cpu = ["mlx[cpu]>=0.29"]
|
| 23 |
+
parity = ["einops"]
|
| 24 |
+
|
| 25 |
+
[project.scripts]
|
| 26 |
+
fastframes-mlx = "fastframes_mlx.generate:main"
|
| 27 |
+
|
| 28 |
+
[build-system]
|
| 29 |
+
requires = ["setuptools>=68"]
|
| 30 |
+
build-backend = "setuptools.build_meta"
|
| 31 |
+
|
| 32 |
+
[tool.setuptools.packages.find]
|
| 33 |
+
include = ["fastframes_mlx*"]
|
tests/data/fastframes_s_keys.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"refiner": [
|
| 3 |
+
"heads_grouped.gemms.0.dK_b",
|
| 4 |
+
"heads_grouped.gemms.0.dK_w",
|
| 5 |
+
"heads_grouped.gemms.0.dV_b",
|
| 6 |
+
"heads_grouped.gemms.0.dV_w",
|
| 7 |
+
"heads_grouped.gemms.0.fuse_b",
|
| 8 |
+
"heads_grouped.gemms.0.fuse_w",
|
| 9 |
+
"heads_grouped.gemms.0.m0_b",
|
| 10 |
+
"heads_grouped.gemms.0.m0_w",
|
| 11 |
+
"heads_grouped.gemms.0.m2_b",
|
| 12 |
+
"heads_grouped.gemms.0.m2_w",
|
| 13 |
+
"heads_grouped.gemms.0.mod_b",
|
| 14 |
+
"heads_grouped.gemms.0.mod_w",
|
| 15 |
+
"heads_grouped.gemms.0.proj_k_b",
|
| 16 |
+
"heads_grouped.gemms.0.proj_k_w",
|
| 17 |
+
"heads_grouped.gemms.0.proj_v_b",
|
| 18 |
+
"heads_grouped.gemms.0.proj_v_w",
|
| 19 |
+
"heads_grouped.gemms.0.v_logscale",
|
| 20 |
+
"heads_grouped.phi_proj.768.weight",
|
| 21 |
+
"stem_blocks.0.mlp.0.bias",
|
| 22 |
+
"stem_blocks.0.mlp.0.weight",
|
| 23 |
+
"stem_blocks.0.mlp.2.bias",
|
| 24 |
+
"stem_blocks.0.mlp.2.weight",
|
| 25 |
+
"stem_blocks.0.mod.lin.bias",
|
| 26 |
+
"stem_blocks.0.mod.lin.weight",
|
| 27 |
+
"stem_blocks.0.proj.bias",
|
| 28 |
+
"stem_blocks.0.proj.weight",
|
| 29 |
+
"stem_blocks.0.qkv.bias",
|
| 30 |
+
"stem_blocks.0.qkv.weight",
|
| 31 |
+
"stem_blocks.1.mlp.0.bias",
|
| 32 |
+
"stem_blocks.1.mlp.0.weight",
|
| 33 |
+
"stem_blocks.1.mlp.2.bias",
|
| 34 |
+
"stem_blocks.1.mlp.2.weight",
|
| 35 |
+
"stem_blocks.1.mod.lin.bias",
|
| 36 |
+
"stem_blocks.1.mod.lin.weight",
|
| 37 |
+
"stem_blocks.1.proj.bias",
|
| 38 |
+
"stem_blocks.1.proj.weight",
|
| 39 |
+
"stem_blocks.1.qkv.bias",
|
| 40 |
+
"stem_blocks.1.qkv.weight",
|
| 41 |
+
"stem_blocks.2.mlp.0.bias",
|
| 42 |
+
"stem_blocks.2.mlp.0.weight",
|
| 43 |
+
"stem_blocks.2.mlp.2.bias",
|
| 44 |
+
"stem_blocks.2.mlp.2.weight",
|
| 45 |
+
"stem_blocks.2.mod.lin.bias",
|
| 46 |
+
"stem_blocks.2.mod.lin.weight",
|
| 47 |
+
"stem_blocks.2.proj.bias",
|
| 48 |
+
"stem_blocks.2.proj.weight",
|
| 49 |
+
"stem_blocks.2.qkv.bias",
|
| 50 |
+
"stem_blocks.2.qkv.weight",
|
| 51 |
+
"stem_blocks.3.mlp.0.bias",
|
| 52 |
+
"stem_blocks.3.mlp.0.weight",
|
| 53 |
+
"stem_blocks.3.mlp.2.bias",
|
| 54 |
+
"stem_blocks.3.mlp.2.weight",
|
| 55 |
+
"stem_blocks.3.mod.lin.bias",
|
| 56 |
+
"stem_blocks.3.mod.lin.weight",
|
| 57 |
+
"stem_blocks.3.proj.bias",
|
| 58 |
+
"stem_blocks.3.proj.weight",
|
| 59 |
+
"stem_blocks.3.qkv.bias",
|
| 60 |
+
"stem_blocks.3.qkv.weight",
|
| 61 |
+
"stem_in.bias",
|
| 62 |
+
"stem_in.weight",
|
| 63 |
+
"stem_norm.bias",
|
| 64 |
+
"stem_norm.weight",
|
| 65 |
+
"t_embed.mlp.0.bias",
|
| 66 |
+
"t_embed.mlp.0.weight",
|
| 67 |
+
"t_embed.mlp.2.bias",
|
| 68 |
+
"t_embed.mlp.2.weight"
|
| 69 |
+
],
|
| 70 |
+
"decoder": [
|
| 71 |
+
"conv_high.0.dw.bias",
|
| 72 |
+
"conv_high.0.dw.weight",
|
| 73 |
+
"conv_high.0.norm.bias",
|
| 74 |
+
"conv_high.0.norm.weight",
|
| 75 |
+
"conv_high.0.pw1.bias",
|
| 76 |
+
"conv_high.0.pw1.weight",
|
| 77 |
+
"conv_high.0.pw2.bias",
|
| 78 |
+
"conv_high.0.pw2.weight",
|
| 79 |
+
"conv_low.0.dw.bias",
|
| 80 |
+
"conv_low.0.dw.weight",
|
| 81 |
+
"conv_low.0.norm.bias",
|
| 82 |
+
"conv_low.0.norm.weight",
|
| 83 |
+
"conv_low.0.pw1.bias",
|
| 84 |
+
"conv_low.0.pw1.weight",
|
| 85 |
+
"conv_low.0.pw2.bias",
|
| 86 |
+
"conv_low.0.pw2.weight",
|
| 87 |
+
"conv_low.1.dw.bias",
|
| 88 |
+
"conv_low.1.dw.weight",
|
| 89 |
+
"conv_low.1.norm.bias",
|
| 90 |
+
"conv_low.1.norm.weight",
|
| 91 |
+
"conv_low.1.pw1.bias",
|
| 92 |
+
"conv_low.1.pw1.weight",
|
| 93 |
+
"conv_low.1.pw2.bias",
|
| 94 |
+
"conv_low.1.pw2.weight",
|
| 95 |
+
"entry_norm.bias",
|
| 96 |
+
"entry_norm.weight",
|
| 97 |
+
"entry_pw.bias",
|
| 98 |
+
"entry_pw.weight",
|
| 99 |
+
"fc_out.bias",
|
| 100 |
+
"fc_out.weight",
|
| 101 |
+
"modulation"
|
| 102 |
+
]
|
| 103 |
+
}
|
tests/test_parity.py
ADDED
|
@@ -0,0 +1,583 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Numerical parity against the official FastFrames PyTorch modules.
|
| 2 |
+
|
| 3 |
+
The official ``attention`` helper requests cuDNN SDPA, which is not available
|
| 4 |
+
on this CPU. Tests patch it to ``backend='auto'`` (PyTorch's math kernel) and
|
| 5 |
+
compare with MLX SDPA. Both still cast attention inputs to bfloat16, as the
|
| 6 |
+
release does. Tolerances below are what this Linux CPU run actually met.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import os
|
| 12 |
+
import subprocess
|
| 13 |
+
import sys
|
| 14 |
+
import unittest
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import types
|
| 18 |
+
|
| 19 |
+
import mlx.core as mx
|
| 20 |
+
import numpy as np
|
| 21 |
+
import torch
|
| 22 |
+
|
| 23 |
+
# Official pipeline imports the Wan tokenizer, which imports transformers.
|
| 24 |
+
# The parity tests do not construct that tokenizer. Prompt cleaning is compared
|
| 25 |
+
# against the functions in tokenizers.py.
|
| 26 |
+
if "transformers" not in sys.modules:
|
| 27 |
+
_transformers = types.ModuleType("transformers")
|
| 28 |
+
_transformers.AutoTokenizer = object
|
| 29 |
+
sys.modules["transformers"] = _transformers
|
| 30 |
+
|
| 31 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 32 |
+
sys.path.insert(0, str(ROOT))
|
| 33 |
+
|
| 34 |
+
OFFICIAL = Path(os.environ.get("FASTFRAMES_SRC", "/tmp/ml-fastframes"))
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _ensure_official():
|
| 38 |
+
if (OFFICIAL / "fastframes" / "wan" / "model.py").exists():
|
| 39 |
+
return OFFICIAL
|
| 40 |
+
dest = ROOT / "third_party" / "ml-fastframes"
|
| 41 |
+
if not (dest / "fastframes" / "wan" / "model.py").exists():
|
| 42 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 43 |
+
subprocess.check_call(
|
| 44 |
+
["git", "clone", "--depth", "1", "https://github.com/apple-aiml-research/ml-fastframes.git", str(dest)]
|
| 45 |
+
)
|
| 46 |
+
return dest
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
OFFICIAL = _ensure_official()
|
| 50 |
+
sys.path.insert(0, str(OFFICIAL))
|
| 51 |
+
|
| 52 |
+
import fastframes.pipeline as official_pipeline # noqa: E402
|
| 53 |
+
import fastframes.wan.attention as official_attention # noqa: E402
|
| 54 |
+
import fastframes.wan.causal_model as official_causal # noqa: E402
|
| 55 |
+
import fastframes.wan.model as official_model # noqa: E402
|
| 56 |
+
|
| 57 |
+
_orig_attention = official_attention.attention
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _attention_auto(q, k, v, causal=False, dropout_p=0.0, backend="auto"):
|
| 61 |
+
return _orig_attention(q, k, v, causal=causal, dropout_p=dropout_p, backend="auto")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
official_attention.attention = _attention_auto
|
| 65 |
+
official_causal.attention = _attention_auto
|
| 66 |
+
official_model.attention = _attention_auto
|
| 67 |
+
official_model.cross_attention = lambda q, k, v: _attention_auto(q, k, v)
|
| 68 |
+
|
| 69 |
+
from fastframes.models.kv_refinement import KVRefinementNetwork # noqa: E402
|
| 70 |
+
from fastframes.models.pixel_decoder import PixelDecoder as OfficialDecoder # noqa: E402
|
| 71 |
+
from fastframes.models.pixel_decoder import _CausalConv3d # noqa: E402
|
| 72 |
+
from fastframes.pipeline import FF_S, FastFrames as OfficialFastFrames # noqa: E402
|
| 73 |
+
from fastframes.wan.causal_model import CausalWanModel, allocate_kv_cache # noqa: E402
|
| 74 |
+
from fastframes.wan.model import WanConfig # noqa: E402
|
| 75 |
+
from fastframes.wan.scheduler import FlowMatchScheduler as OfficialScheduler # noqa: E402
|
| 76 |
+
from fastframes.wan.t5 import T5Encoder # noqa: E402
|
| 77 |
+
|
| 78 |
+
from fastframes_mlx.config import FastFramesConfig, WanConfig as MlxWanConfig # noqa: E402
|
| 79 |
+
from fastframes_mlx.dit import CausalWanModel as MlxWan # noqa: E402
|
| 80 |
+
from fastframes_mlx.kv_refiner import KVRefinementNetwork as MlxRefiner # noqa: E402
|
| 81 |
+
from fastframes_mlx.ops import gelu_tanh, sinusoidal_embedding_1d # noqa: E402
|
| 82 |
+
from fastframes_mlx.pixel_decoder import CausalConv3d, PixelDecoder # noqa: E402
|
| 83 |
+
from fastframes_mlx.scheduler import FlowMatchScheduler # noqa: E402
|
| 84 |
+
from fastframes_mlx.t5 import T5Encoder as MlxT5 # noqa: E402
|
| 85 |
+
from fastframes_mlx.weights import DIT_KEYS, REFINER_KEYS, T5_KEYS, _prepare, assign # noqa: E402
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _max_err(a, b):
|
| 89 |
+
if isinstance(a, mx.array):
|
| 90 |
+
a = np.array(a.astype(mx.float32))
|
| 91 |
+
else:
|
| 92 |
+
a = np.asarray(a, dtype=np.float32)
|
| 93 |
+
if isinstance(b, mx.array):
|
| 94 |
+
b = np.array(b.astype(mx.float32))
|
| 95 |
+
else:
|
| 96 |
+
b = np.asarray(b, dtype=np.float32)
|
| 97 |
+
return float(np.max(np.abs(a - b)))
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _load(module, torch_module, renames=()):
|
| 101 |
+
raw = {k: v.detach() for k, v in torch_module.state_dict().items()}
|
| 102 |
+
assign(module, _prepare(raw, renames))
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
TINY = dict(dim=32, ffn_dim=64, num_heads=4, num_layers=2, text_dim=32, freq_dim=16, text_len=8)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
class ParityTest(unittest.TestCase):
|
| 109 |
+
def test_scheduler_matches_official_grid(self):
|
| 110 |
+
ours = FlowMatchScheduler(shift=5.0)
|
| 111 |
+
theirs = OfficialScheduler(shift=5.0)
|
| 112 |
+
self.assertLess(_max_err(ours.sigmas, theirs.sigmas.numpy()), 1e-6)
|
| 113 |
+
# timesteps are sigmas * 1000 in float32; 1e-7 of sigma is 1e-4 here.
|
| 114 |
+
self.assertLess(_max_err(ours.timesteps, theirs.timesteps.numpy()), 2e-4)
|
| 115 |
+
for value in (1000.0, 937.5, 833.3333129882812, 625.0, 0.0):
|
| 116 |
+
ours_i = int(np.array(ours._index(mx.array([value]))).reshape(-1)[0])
|
| 117 |
+
theirs_i = int(theirs._index(torch.tensor([value]), theirs.timesteps))
|
| 118 |
+
self.assertEqual(ours_i, theirs_i, value)
|
| 119 |
+
flow = torch.randn(2, 16, 4, 6)
|
| 120 |
+
xt = torch.randn(2, 16, 4, 6)
|
| 121 |
+
ts = torch.tensor([1000.0, 833.3333129882812])
|
| 122 |
+
ref = theirs.flow_to_x0(flow, xt, ts)
|
| 123 |
+
got = ours.flow_to_x0(mx.array(flow.numpy()), mx.array(xt.numpy()), mx.array(ts.numpy()))
|
| 124 |
+
self.assertLess(_max_err(got, ref.numpy()), 1e-5)
|
| 125 |
+
noise = torch.randn(2, 16, 4, 6)
|
| 126 |
+
ref_n = theirs.add_noise(xt, noise, ts)
|
| 127 |
+
got_n = ours.add_noise(mx.array(xt.numpy()), mx.array(noise.numpy()), mx.array(ts.numpy()))
|
| 128 |
+
self.assertLess(_max_err(got_n, ref_n.numpy()), 1e-6)
|
| 129 |
+
# t = 0 snaps to the last grid sigma, not to zero.
|
| 130 |
+
zero = theirs.flow_to_x0(flow[:1], xt[:1], torch.tensor([0.0]))
|
| 131 |
+
got_z = ours.flow_to_x0(mx.array(flow[:1].numpy()), mx.array(xt[:1].numpy()), mx.array([0.0]))
|
| 132 |
+
self.assertLess(_max_err(got_z, zero.numpy()), 1e-5)
|
| 133 |
+
|
| 134 |
+
def test_sinusoid_and_gelu(self):
|
| 135 |
+
from fastframes.wan.model import sinusoidal_embedding_1d as official_sin
|
| 136 |
+
|
| 137 |
+
pos = torch.tensor([0.0, 1000.0, 833.3333129882812])
|
| 138 |
+
ref = official_sin(16, pos)
|
| 139 |
+
got = sinusoidal_embedding_1d(16, mx.array(pos.numpy()))
|
| 140 |
+
# NumPy float64 sinusoid, stored as float32 for the GPU.
|
| 141 |
+
self.assertLess(_max_err(got, ref.numpy()), 2e-5)
|
| 142 |
+
self.assertEqual(got.dtype, mx.float32)
|
| 143 |
+
x = torch.randn(3, 8)
|
| 144 |
+
ref_g = torch.nn.GELU(approximate="tanh")(x)
|
| 145 |
+
self.assertLess(_max_err(gelu_tanh(mx.array(x.numpy())), ref_g.numpy()), 1e-5)
|
| 146 |
+
|
| 147 |
+
def test_rope_matches_complex128(self):
|
| 148 |
+
from fastframes.wan.model import causal_rope_apply, rope_params
|
| 149 |
+
|
| 150 |
+
head_dim = 32
|
| 151 |
+
d = head_dim
|
| 152 |
+
freqs = torch.cat(
|
| 153 |
+
[
|
| 154 |
+
rope_params(64, d - 4 * (d // 6)),
|
| 155 |
+
rope_params(64, 2 * (d // 6)),
|
| 156 |
+
rope_params(64, 2 * (d // 6)),
|
| 157 |
+
],
|
| 158 |
+
dim=1,
|
| 159 |
+
)
|
| 160 |
+
torch.manual_seed(0)
|
| 161 |
+
x = torch.randn(1, 6, 2, head_dim)
|
| 162 |
+
ref = causal_rope_apply(x, torch.tensor([[1, 2, 3]]), freqs, start_frame=2)
|
| 163 |
+
from fastframes_mlx.rope import RopeTables
|
| 164 |
+
|
| 165 |
+
tables = RopeTables(head_dim, max_seq_len=64)
|
| 166 |
+
got = tables.apply(mx.array(x.numpy()), 1, 2, 3, start_frame=2)
|
| 167 |
+
self.assertLess(_max_err(got, ref.numpy()), 1e-6)
|
| 168 |
+
|
| 169 |
+
def test_dit_keys_and_forward_frame(self):
|
| 170 |
+
torch.manual_seed(1)
|
| 171 |
+
cfg = WanConfig(**TINY)
|
| 172 |
+
ref = CausalWanModel(cfg).eval()
|
| 173 |
+
mlx_cfg = MlxWanConfig(**TINY)
|
| 174 |
+
got_m = MlxWan(mlx_cfg)
|
| 175 |
+
_load(got_m, ref, DIT_KEYS)
|
| 176 |
+
b, h, w = 1, 4, 6
|
| 177 |
+
tokens = (h // 2) * (w // 2)
|
| 178 |
+
latent = torch.randn(b, 1, 16, h, w)
|
| 179 |
+
t = torch.tensor([[1000.0]])
|
| 180 |
+
text = torch.randn(b, TINY["text_len"], TINY["text_dim"])
|
| 181 |
+
with torch.no_grad():
|
| 182 |
+
e0, e_head = ref.time_conditioning(t, torch.float32)
|
| 183 |
+
cache_t = ref.text_cache([row for row in text])
|
| 184 |
+
kv = allocate_kv_cache(ref, 2, tokens, dtype=torch.float32, device="cpu")
|
| 185 |
+
flow, hidden = ref.forward_frame(latent, e0, e_head, cache_t, kv, frame=0)
|
| 186 |
+
e0_m, e_head_m = got_m.time_conditioning(mx.array(t.numpy()), mx.float32)
|
| 187 |
+
cache_m = got_m.text_cache(mx.array(text.numpy()))
|
| 188 |
+
from fastframes_mlx.dit import LayerCache
|
| 189 |
+
|
| 190 |
+
kv_m = LayerCache(cfg.num_layers, 2)
|
| 191 |
+
flow_m, hidden_m = got_m.forward_frame(
|
| 192 |
+
mx.array(latent.numpy()), e0_m, e_head_m, cache_m, kv_m, 0
|
| 193 |
+
)
|
| 194 |
+
# bf16 SDPA inside both implementations. Two layers, short sequence.
|
| 195 |
+
self.assertLess(_max_err(flow_m, flow.numpy()), 2e-2)
|
| 196 |
+
self.assertLess(_max_err(hidden_m, hidden.numpy()), 2e-2)
|
| 197 |
+
# Second frame must attend to the first frame's cache.
|
| 198 |
+
latent2 = torch.randn(b, 1, 16, h, w)
|
| 199 |
+
with torch.no_grad():
|
| 200 |
+
flow2, _ = ref.forward_frame(latent2, e0, e_head, cache_t, kv, frame=1)
|
| 201 |
+
flow2_m, _ = got_m.forward_frame(mx.array(latent2.numpy()), e0_m, e_head_m, cache_m, kv_m, 1)
|
| 202 |
+
self.assertLess(_max_err(flow2_m, flow2.numpy()), 3e-2)
|
| 203 |
+
|
| 204 |
+
def test_refiner(self):
|
| 205 |
+
torch.manual_seed(2)
|
| 206 |
+
ref = KVRefinementNetwork(
|
| 207 |
+
dim=32, num_heads=4, head_dim=8, num_blocks=2, d_a=16, n_stem=1, stem_n_heads=4, stem_mlp_mult=2
|
| 208 |
+
).eval()
|
| 209 |
+
got = MlxRefiner(dim=32, num_heads=4, num_blocks=2, width=16, n_stem=1, stem_heads=4, stem_mlp=2)
|
| 210 |
+
_load(got, ref, REFINER_KEYS)
|
| 211 |
+
tokens = 6
|
| 212 |
+
stem = torch.randn(1, tokens, 32)
|
| 213 |
+
n_k = torch.randn(2, tokens, 32)
|
| 214 |
+
n_v = torch.randn(2, tokens, 32)
|
| 215 |
+
with torch.no_grad():
|
| 216 |
+
emb = ref.embed_timestep(833.3333129882812, 1, "cpu", torch.float32)
|
| 217 |
+
phi = ref.compute_phi(stem, emb)
|
| 218 |
+
k, v = ref.repair_all(n_k, n_v, phi, emb)
|
| 219 |
+
emb_m = got.embed_timestep(833.3333129882812, 1, mx.float32)
|
| 220 |
+
phi_m = got.compute_phi(mx.array(stem.numpy()), emb_m)
|
| 221 |
+
k_m, v_m = got.repair_all(mx.array(n_k.numpy()), mx.array(n_v.numpy()), phi_m, emb_m)
|
| 222 |
+
self.assertLess(_max_err(phi_m, phi.numpy()), 1e-4)
|
| 223 |
+
self.assertLess(_max_err(k_m, k.numpy()), 2e-4)
|
| 224 |
+
self.assertLess(_max_err(v_m, v.numpy()), 2e-4)
|
| 225 |
+
|
| 226 |
+
def test_pixel_decoder_and_causal_conv(self):
|
| 227 |
+
torch.manual_seed(3)
|
| 228 |
+
x = torch.randn(1, 8, 2, 5, 5)
|
| 229 |
+
ref_conv = _CausalConv3d(8, 8, 3, padding=1, groups=8)
|
| 230 |
+
got_conv = CausalConv3d(8, 8, 3, groups=8)
|
| 231 |
+
_load(got_conv, ref_conv)
|
| 232 |
+
with torch.no_grad():
|
| 233 |
+
ref_y = ref_conv(x)
|
| 234 |
+
got_y = got_conv(mx.array(x.numpy()))
|
| 235 |
+
self.assertLess(_max_err(got_y, ref_y.numpy()), 1e-5)
|
| 236 |
+
|
| 237 |
+
ref = OfficialDecoder(dim=32, pixel_patch_size=(2, 4, 4), n_low_blocks=1, n_high_blocks=1, ffn_mult=2).eval()
|
| 238 |
+
got = PixelDecoder(dim=32, pixel_patch_size=(2, 4, 4), n_low_blocks=1, n_high_blocks=1, ffn_mult=2)
|
| 239 |
+
_load(got, ref)
|
| 240 |
+
# Released initializers zero fc_out, which would hide every earlier op.
|
| 241 |
+
with torch.no_grad():
|
| 242 |
+
ref.fc_out.weight.normal_()
|
| 243 |
+
ref.fc_out.bias.normal_()
|
| 244 |
+
got.fc_out.weight = mx.array(ref.fc_out.weight.detach().numpy())
|
| 245 |
+
got.fc_out.bias = mx.array(ref.fc_out.bias.detach().numpy())
|
| 246 |
+
frames, tokens, dim = 2, 4, 32
|
| 247 |
+
hidden = torch.randn(1, frames * tokens, dim)
|
| 248 |
+
mod = torch.randn(1, frames, 1, dim)
|
| 249 |
+
grid = torch.tensor([[frames, 2, 2]])
|
| 250 |
+
with torch.no_grad():
|
| 251 |
+
ref_p = ref(hidden, mod, grid)
|
| 252 |
+
got_p = got(mx.array(hidden.numpy()), mx.array(mod.numpy()), (frames, 2, 2))
|
| 253 |
+
self.assertLess(_max_err(got_p, ref_p.numpy()), 2e-4)
|
| 254 |
+
self.assertEqual(tuple(got_p.shape), tuple(ref_p.shape))
|
| 255 |
+
|
| 256 |
+
def test_t5_block(self):
|
| 257 |
+
torch.manual_seed(4)
|
| 258 |
+
ref = T5Encoder(
|
| 259 |
+
vocab=50, dim=32, dim_attn=32, dim_ffn=64, num_heads=4, num_layers=2, num_buckets=8, shared_pos=False, dropout=0.0
|
| 260 |
+
).eval()
|
| 261 |
+
got = MlxT5(vocab_size=50, dim=32, dim_attn=32, dim_ffn=64, num_heads=4, num_layers=2, num_buckets=8, shared_pos=False)
|
| 262 |
+
_load(got, ref, T5_KEYS)
|
| 263 |
+
ids = torch.randint(0, 50, (2, 6))
|
| 264 |
+
mask = torch.ones(2, 6)
|
| 265 |
+
mask[:, -2:] = 0
|
| 266 |
+
with torch.no_grad():
|
| 267 |
+
ref_y = ref(ids, mask)
|
| 268 |
+
got_y = got(mx.array(ids.numpy()), mx.array(mask.numpy()))
|
| 269 |
+
self.assertLess(_max_err(got_y, ref_y.numpy()), 2e-4)
|
| 270 |
+
|
| 271 |
+
def test_prompt_cleaner_matches_official(self):
|
| 272 |
+
from fastframes.wan.tokenizers import basic_clean, whitespace_clean
|
| 273 |
+
|
| 274 |
+
from fastframes_mlx.tokenizer import basic_clean as our_basic
|
| 275 |
+
from fastframes_mlx.tokenizer import whitespace_clean as our_ws
|
| 276 |
+
|
| 277 |
+
text = " Hello,\n\nworld\t& friends "
|
| 278 |
+
self.assertEqual(our_basic(text), basic_clean(text))
|
| 279 |
+
self.assertEqual(our_ws(our_basic(text)), whitespace_clean(basic_clean(text)))
|
| 280 |
+
|
| 281 |
+
def test_two_frame_recipe_matches_official_pipeline(self):
|
| 282 |
+
torch.manual_seed(5)
|
| 283 |
+
cfg = WanConfig(**TINY)
|
| 284 |
+
dit = CausalWanModel(cfg).eval()
|
| 285 |
+
refiner = KVRefinementNetwork(
|
| 286 |
+
dim=32, num_heads=4, head_dim=8, num_blocks=2, d_a=16, n_stem=1, stem_n_heads=4, stem_mlp_mult=2
|
| 287 |
+
).eval()
|
| 288 |
+
decoder = OfficialDecoder(
|
| 289 |
+
dim=32, pixel_patch_size=(4, 4, 4), n_low_blocks=1, n_high_blocks=1, ffn_mult=2
|
| 290 |
+
).eval()
|
| 291 |
+
from fastframes.pipeline import FastFramesConfig as OffCfg
|
| 292 |
+
|
| 293 |
+
off_cfg = OffCfg(
|
| 294 |
+
name="tiny",
|
| 295 |
+
wan_model="unused",
|
| 296 |
+
first_frame_timesteps=(1000.0, 833.3333129882812),
|
| 297 |
+
timesteps=(1000.0, 833.3333129882812),
|
| 298 |
+
refine_timestep=833.3333129882812,
|
| 299 |
+
noise="sequential",
|
| 300 |
+
autocast=False,
|
| 301 |
+
fast_ops=False,
|
| 302 |
+
decoder_dtype=torch.float32,
|
| 303 |
+
contiguous_stem=True,
|
| 304 |
+
allow_tf32=False,
|
| 305 |
+
num_frames=2,
|
| 306 |
+
latent_size=(4, 4),
|
| 307 |
+
num_pixel_frames=5,
|
| 308 |
+
)
|
| 309 |
+
official = OfficialFastFrames(off_cfg, dit, refiner, decoder, text_encoder=None)
|
| 310 |
+
mlx_cfg = FastFramesConfig(
|
| 311 |
+
wan=MlxWanConfig(**TINY),
|
| 312 |
+
num_frames=2,
|
| 313 |
+
latent_height=4,
|
| 314 |
+
latent_width=4,
|
| 315 |
+
num_pixel_frames=5,
|
| 316 |
+
first_frame_timesteps=(1000.0, 833.3333129882812),
|
| 317 |
+
timesteps=(1000.0, 833.3333129882812),
|
| 318 |
+
refiner_width=16,
|
| 319 |
+
refiner_stem_blocks=1,
|
| 320 |
+
refiner_stem_heads=4,
|
| 321 |
+
refiner_stem_mlp=2,
|
| 322 |
+
pixel_patch=(4, 4, 4),
|
| 323 |
+
decoder_low_blocks=1,
|
| 324 |
+
decoder_high_blocks=1,
|
| 325 |
+
decoder_ffn=2,
|
| 326 |
+
)
|
| 327 |
+
mlx_dit = MlxWan(mlx_cfg.wan)
|
| 328 |
+
mlx_ref = MlxRefiner(dim=32, num_heads=4, num_blocks=2, width=16, n_stem=1, stem_heads=4, stem_mlp=2)
|
| 329 |
+
mlx_dec = PixelDecoder(dim=32, pixel_patch_size=(4, 4, 4), n_low_blocks=1, n_high_blocks=1, ffn_mult=2)
|
| 330 |
+
_load(mlx_dit, dit, DIT_KEYS)
|
| 331 |
+
_load(mlx_ref, refiner, REFINER_KEYS)
|
| 332 |
+
_load(mlx_dec, decoder)
|
| 333 |
+
# The release runs the DiT and the refiner in bf16 (the pipeline even
|
| 334 |
+
# hardcodes the time embedding dtype). The pixel decoder stays fp32.
|
| 335 |
+
dit.to(torch.bfloat16)
|
| 336 |
+
refiner.to(torch.bfloat16)
|
| 337 |
+
from fastframes_mlx.weights import cast_floating
|
| 338 |
+
|
| 339 |
+
cast_floating(mlx_dit, mx.bfloat16)
|
| 340 |
+
cast_floating(mlx_ref, mx.bfloat16)
|
| 341 |
+
from fastframes_mlx.pipeline import FastFrames
|
| 342 |
+
|
| 343 |
+
port = FastFrames(mlx_cfg, mlx_dit, mlx_ref, mlx_dec)
|
| 344 |
+
rng = np.random.default_rng(0)
|
| 345 |
+
noise = rng.standard_normal((1, 2, 16, 4, 4), dtype=np.float32)
|
| 346 |
+
transitions = [rng.standard_normal((1, 16, 4, 4), dtype=np.float32) for _ in range(4)]
|
| 347 |
+
cursor = {"i": 0}
|
| 348 |
+
|
| 349 |
+
def off_noise(self, seed):
|
| 350 |
+
def transition(step, frame):
|
| 351 |
+
eps = torch.tensor(transitions[cursor["i"]], dtype=torch.bfloat16)
|
| 352 |
+
cursor["i"] += 1
|
| 353 |
+
return eps
|
| 354 |
+
|
| 355 |
+
return torch.tensor(noise, dtype=torch.bfloat16), transition
|
| 356 |
+
|
| 357 |
+
def mlx_noise(self, seed, dtype):
|
| 358 |
+
cursor_m = {"i": 0}
|
| 359 |
+
|
| 360 |
+
def transition(step, frame):
|
| 361 |
+
eps = mx.array(transitions[cursor_m["i"]]).astype(dtype)
|
| 362 |
+
cursor_m["i"] += 1
|
| 363 |
+
return eps
|
| 364 |
+
|
| 365 |
+
return mx.array(noise).astype(dtype), transition
|
| 366 |
+
|
| 367 |
+
official._noise = off_noise.__get__(official, OfficialFastFrames)
|
| 368 |
+
port._initial_noise = mlx_noise.__get__(port, FastFrames)
|
| 369 |
+
|
| 370 |
+
def alloc(*args, **kwargs):
|
| 371 |
+
kwargs["dtype"] = torch.bfloat16
|
| 372 |
+
kwargs["device"] = "cpu"
|
| 373 |
+
return allocate_kv_cache(*args, **kwargs)
|
| 374 |
+
|
| 375 |
+
official_pipeline.allocate_kv_cache = alloc
|
| 376 |
+
text = torch.randn(1, TINY["text_len"], TINY["text_dim"], dtype=torch.bfloat16)
|
| 377 |
+
with torch.no_grad():
|
| 378 |
+
lat_r, feat_r, mod_r = official.rollout(text, seed=0)
|
| 379 |
+
text_m = mx.array(text.view(torch.uint16).numpy()).view(mx.bfloat16)
|
| 380 |
+
lat_m, feat_m, mod_m, _times = port.rollout(text_m, seed=0)
|
| 381 |
+
# bf16 matmul on this CPU. Measured max abs: latents 0.015625,
|
| 382 |
+
# pre-head features 0.03125, time modulation 3e-5.
|
| 383 |
+
self.assertLess(_max_err(lat_m, lat_r.float().numpy()), 0.02)
|
| 384 |
+
self.assertLess(_max_err(feat_m, feat_r.float().numpy()), 0.04)
|
| 385 |
+
self.assertLess(_max_err(mod_m, mod_r.float().numpy()), 1e-4)
|
| 386 |
+
with torch.no_grad():
|
| 387 |
+
pix_r = official.decode(feat_r, mod_r)
|
| 388 |
+
pix_m = port.decode_pixels(feat_m, mod_m)
|
| 389 |
+
self.assertEqual(tuple(pix_m.shape), tuple(pix_r.shape))
|
| 390 |
+
self.assertLess(_max_err(pix_m, pix_r.float().numpy()), 2e-3)
|
| 391 |
+
|
| 392 |
+
def test_released_checkpoint_keys_match_modules(self):
|
| 393 |
+
import json
|
| 394 |
+
|
| 395 |
+
from mlx.utils import tree_flatten
|
| 396 |
+
|
| 397 |
+
from fastframes_mlx.kv_refiner import KVRefinementNetwork
|
| 398 |
+
from fastframes_mlx.pixel_decoder import PixelDecoder
|
| 399 |
+
from fastframes_mlx.weights import REFINER_KEYS, _prepare
|
| 400 |
+
|
| 401 |
+
spec = json.loads((ROOT / "tests" / "data" / "fastframes_s_keys.json").read_text())
|
| 402 |
+
refiner = KVRefinementNetwork()
|
| 403 |
+
decoder = PixelDecoder()
|
| 404 |
+
# Empty tensors are enough: _prepare only looks at rank for conv transpose.
|
| 405 |
+
# Build dummy arrays from the key list by asking the module for shapes.
|
| 406 |
+
have_r = dict(tree_flatten(refiner.parameters()))
|
| 407 |
+
raw = {}
|
| 408 |
+
# Invert the rename so checkpoint keys map onto module keys, then check sets.
|
| 409 |
+
prepared = {}
|
| 410 |
+
for key in spec["refiner"]:
|
| 411 |
+
new = key
|
| 412 |
+
for src, dst in REFINER_KEYS:
|
| 413 |
+
new = new.replace(src, dst)
|
| 414 |
+
prepared[new] = key
|
| 415 |
+
self.assertEqual(set(prepared), set(have_r), sorted(set(prepared) ^ set(have_r))[:12])
|
| 416 |
+
have_d = dict(tree_flatten(decoder.parameters()))
|
| 417 |
+
self.assertEqual(set(spec["decoder"]), set(have_d), sorted(set(spec["decoder"]) ^ set(have_d))[:12])
|
| 418 |
+
del raw
|
| 419 |
+
|
| 420 |
+
def test_vae_causal_conv_matches_official(self):
|
| 421 |
+
from fastframes.wan.vae import CausalConv3d as OfficialConv
|
| 422 |
+
|
| 423 |
+
from fastframes_mlx.vae_layers import CausalConv3d as MlxConv
|
| 424 |
+
|
| 425 |
+
torch.manual_seed(6)
|
| 426 |
+
ref = OfficialConv(4, 3, 3, padding=1)
|
| 427 |
+
x = torch.randn(1, 4, 2, 6, 6)
|
| 428 |
+
with torch.no_grad():
|
| 429 |
+
y = ref(x)
|
| 430 |
+
got = MlxConv(4, 3, 3, padding=1)
|
| 431 |
+
got.weight = mx.transpose(mx.array(ref.weight.detach().numpy()), (0, 2, 3, 4, 1))
|
| 432 |
+
got.bias = mx.array(ref.bias.detach().numpy())
|
| 433 |
+
xm = mx.transpose(mx.array(x.numpy()), (0, 2, 3, 4, 1))
|
| 434 |
+
ym = mx.transpose(got(xm), (0, 4, 1, 2, 3))
|
| 435 |
+
self.assertLess(_max_err(ym, y.numpy()), 1e-5)
|
| 436 |
+
|
| 437 |
+
def test_smoke_cli_writes_mp4(self):
|
| 438 |
+
out = ROOT / "outputs" / "smoke.mp4"
|
| 439 |
+
if out.exists():
|
| 440 |
+
out.unlink()
|
| 441 |
+
env = os.environ.copy()
|
| 442 |
+
env["PYTHONPATH"] = str(ROOT)
|
| 443 |
+
proc = subprocess.run(
|
| 444 |
+
[sys.executable, "-m", "fastframes_mlx.generate", "--smoke", "--out", str(out), "--seed", "1"],
|
| 445 |
+
cwd=str(ROOT),
|
| 446 |
+
env=env,
|
| 447 |
+
capture_output=True,
|
| 448 |
+
text=True,
|
| 449 |
+
check=False,
|
| 450 |
+
)
|
| 451 |
+
self.assertEqual(proc.returncode, 0, proc.stderr)
|
| 452 |
+
self.assertIn("latent frame 00", proc.stdout)
|
| 453 |
+
self.assertIn("MLX allocator peak", proc.stdout)
|
| 454 |
+
self.assertTrue(out.exists() and out.stat().st_size > 0)
|
| 455 |
+
|
| 456 |
+
def test_quantize_linear_runs(self):
|
| 457 |
+
from fastframes_mlx.weights import quantize_linears
|
| 458 |
+
|
| 459 |
+
model = MlxWan(MlxWanConfig(**TINY))
|
| 460 |
+
quantize_linears(model, bits=4, group_size=32)
|
| 461 |
+
from fastframes_mlx.dit import LayerCache
|
| 462 |
+
|
| 463 |
+
latent = mx.random.normal((1, 1, 16, 4, 4))
|
| 464 |
+
e0, e_head = model.time_conditioning(mx.array([[500.0]]), mx.float32)
|
| 465 |
+
text = mx.random.normal((1, TINY["text_len"], TINY["text_dim"]))
|
| 466 |
+
cache = model.text_cache(text)
|
| 467 |
+
flow, hidden = model.forward_frame(latent, e0, e_head, cache, LayerCache(2, 1), 0)
|
| 468 |
+
mx.eval(flow, hidden)
|
| 469 |
+
self.assertTrue(np.isfinite(np.array(flow.astype(mx.float32))).all())
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
class _ForbidFloat64:
|
| 473 |
+
"""Reject float64 MLX arrays. Metal raises on those, and CPU tests would not.
|
| 474 |
+
|
| 475 |
+
``mx.array`` stays acceptable to ``isinstance`` via ``__instancecheck__``,
|
| 476 |
+
because ``mlx.nn.Module`` checks ``isinstance(value, mx.array)``.
|
| 477 |
+
"""
|
| 478 |
+
|
| 479 |
+
def __enter__(self):
|
| 480 |
+
self.cls = type(mx.zeros((1,)))
|
| 481 |
+
self.orig_astype = self.cls.astype
|
| 482 |
+
self.orig_array = mx.array
|
| 483 |
+
self.orig = {
|
| 484 |
+
"arange": mx.arange,
|
| 485 |
+
"zeros": mx.zeros,
|
| 486 |
+
"ones": mx.ones,
|
| 487 |
+
"linspace": mx.linspace,
|
| 488 |
+
}
|
| 489 |
+
orig_array = self.orig_array
|
| 490 |
+
|
| 491 |
+
def _bad(where):
|
| 492 |
+
raise AssertionError(f"float64 MLX array from {where}")
|
| 493 |
+
|
| 494 |
+
class _Meta(type):
|
| 495 |
+
def __instancecheck__(cls, obj):
|
| 496 |
+
return isinstance(obj, orig_array)
|
| 497 |
+
|
| 498 |
+
class _Array(metaclass=_Meta):
|
| 499 |
+
def __new__(cls, *args, **kwargs):
|
| 500 |
+
if kwargs.get("dtype", False) is mx.float64:
|
| 501 |
+
_bad("array")
|
| 502 |
+
out = orig_array(*args, **kwargs)
|
| 503 |
+
if out.dtype is mx.float64:
|
| 504 |
+
_bad("array")
|
| 505 |
+
return out
|
| 506 |
+
|
| 507 |
+
def _wrap(name, fn):
|
| 508 |
+
def wrapped(*args, **kwargs):
|
| 509 |
+
if kwargs.get("dtype", False) is mx.float64:
|
| 510 |
+
_bad(name)
|
| 511 |
+
out = fn(*args, **kwargs)
|
| 512 |
+
if getattr(out, "dtype", None) is mx.float64:
|
| 513 |
+
_bad(name)
|
| 514 |
+
return out
|
| 515 |
+
|
| 516 |
+
return wrapped
|
| 517 |
+
|
| 518 |
+
def astype(arr, dtype, *args, **kwargs):
|
| 519 |
+
if dtype is mx.float64:
|
| 520 |
+
_bad("astype")
|
| 521 |
+
return self.orig_astype(arr, dtype, *args, **kwargs)
|
| 522 |
+
|
| 523 |
+
mx.array = _Array
|
| 524 |
+
for name, fn in self.orig.items():
|
| 525 |
+
setattr(mx, name, _wrap(name, fn))
|
| 526 |
+
self.cls.astype = astype
|
| 527 |
+
return self
|
| 528 |
+
|
| 529 |
+
def __exit__(self, *exc):
|
| 530 |
+
mx.array = self.orig_array
|
| 531 |
+
for name, fn in self.orig.items():
|
| 532 |
+
setattr(mx, name, fn)
|
| 533 |
+
self.cls.astype = self.orig_astype
|
| 534 |
+
return False
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
class GpuFloat64Test(unittest.TestCase):
|
| 538 |
+
def test_forward_and_smoke_reject_float64(self):
|
| 539 |
+
"""Run a DiT step and ``--smoke`` on the GPU when Metal/CUDA exists.
|
| 540 |
+
|
| 541 |
+
Float64 MLX arrays are rejected either way, so the M5 crash
|
| 542 |
+
(``float64 is not supported on the GPU``) fails this test on CPU too.
|
| 543 |
+
"""
|
| 544 |
+
from fastframes_mlx.dit import LayerCache
|
| 545 |
+
from fastframes_mlx.generate import _smoke_stack, main
|
| 546 |
+
|
| 547 |
+
previous = mx.default_device()
|
| 548 |
+
try:
|
| 549 |
+
mx.set_default_device(mx.gpu)
|
| 550 |
+
mx.eval(mx.array([1.0], dtype=mx.float32) + 1)
|
| 551 |
+
on_gpu = True
|
| 552 |
+
except (ValueError, RuntimeError):
|
| 553 |
+
mx.set_default_device(previous)
|
| 554 |
+
on_gpu = False
|
| 555 |
+
try:
|
| 556 |
+
with _ForbidFloat64():
|
| 557 |
+
with self.assertRaises(AssertionError):
|
| 558 |
+
mx.arange(3, dtype=mx.float64)
|
| 559 |
+
model, prompt = _smoke_stack()
|
| 560 |
+
e0, e_head = model.dit.time_conditioning(mx.array([[1000.0]], dtype=mx.float32), mx.float32)
|
| 561 |
+
latent = mx.random.normal((1, 1, 16, model.config.latent_height, model.config.latent_width))
|
| 562 |
+
cache = model.dit.text_cache(prompt)
|
| 563 |
+
flow, hidden = model.dit.forward_frame(
|
| 564 |
+
latent, e0, e_head, cache, LayerCache(model.dit.num_layers, 1), 0
|
| 565 |
+
)
|
| 566 |
+
mx.eval(flow, hidden)
|
| 567 |
+
self.assertNotEqual(flow.dtype, mx.float64)
|
| 568 |
+
self.assertNotEqual(hidden.dtype, mx.float64)
|
| 569 |
+
self.assertTrue(np.isfinite(np.array(flow.astype(mx.float32))).all())
|
| 570 |
+
if on_gpu:
|
| 571 |
+
self.assertEqual(flow.device, mx.default_device())
|
| 572 |
+
out = ROOT / "outputs" / "gpu_smoke.mp4"
|
| 573 |
+
if out.exists():
|
| 574 |
+
out.unlink()
|
| 575 |
+
status = main(["--smoke", "--out", str(out), "--seed", "2"])
|
| 576 |
+
self.assertEqual(status, 0)
|
| 577 |
+
self.assertTrue(out.exists() and out.stat().st_size > 0)
|
| 578 |
+
finally:
|
| 579 |
+
mx.set_default_device(previous)
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
if __name__ == "__main__":
|
| 583 |
+
unittest.main()
|