Text-to-Video
MLX
apple-silicon
pratikgx's picture
Add unofficial MLX port of Apple FastFrames-S (code only; official weights loaded at runtime)
e87ef2c verified
Raw History Blame Contribute Delete
4.26 kB
"""Small numeric helpers shared by the MLX modules.
LayerNorm and RMSNorm follow the Wan / FastFrames definitions: the reduction is
in float32, the result is cast back to the activation dtype, and RMSNorm
multiplies the weight after that cast.
"""
from __future__ import annotations
import math
import numpy as np
import mlx.core as mx
def gelu_tanh(x: mx.array) -> mx.array:
"""PyTorch ``GELU(approximate='tanh')`` and the umT5 encoder GELU."""
c = math.sqrt(2.0 / math.pi)
return 0.5 * x * (1.0 + mx.tanh(c * (x + 0.044715 * (x * x * x))))
def gelu_erf(x: mx.array) -> mx.array:
"""Exact PyTorch ``GELU()`` (erf), used by the pixel-decoder ConvNeXt blocks."""
return 0.5 * x * (1.0 + mx.erf(x / math.sqrt(2.0)))
def silu(x: mx.array) -> mx.array:
return x * mx.sigmoid(x)
def layer_norm(x, eps, weight=None, bias=None):
xf = x.astype(mx.float32)
mean = mx.mean(xf, axis=-1, keepdims=True)
var = mx.mean(mx.square(xf - mean), axis=-1, keepdims=True)
y = (xf - mean) * mx.rsqrt(var + eps)
if weight is not None:
y = y * weight.astype(mx.float32)
if bias is not None:
y = y + bias.astype(mx.float32)
return y.astype(x.dtype)
def rms_norm(x, weight, eps):
xf = x.astype(mx.float32)
y = xf * mx.rsqrt(mx.mean(mx.square(xf), axis=-1, keepdims=True) + eps)
return y.astype(x.dtype) * weight
def sinusoidal_embedding_1d(dim: int, position: mx.array) -> mx.array:
"""Wan / DiT timestep embedding.
Angles are NumPy float64, matching PyTorch's float64 sinusoid. The array
handed back to MLX is float32 so it can live on the GPU.
"""
if dim % 2 != 0:
raise ValueError(f"sinusoidal dim must be even, got {dim}")
half = dim // 2
pos = np.asarray(position.astype(mx.float32)).astype(np.float64).reshape(-1)
div = np.arange(half, dtype=np.float64) / np.float64(half)
sinusoid = pos[:, None] * np.power(np.float64(10000.0), -div)[None, :]
emb = np.concatenate([np.cos(sinusoid), np.sin(sinusoid)], axis=1).astype(np.float32)
return mx.array(emb)
def sdpa(q, k, v):
"""Scaled dot-product attention.
``q, k, v`` are ``[B, L, H, D]``. Matches FastFrames' ``attention``: cast to
bfloat16, scale by ``1/sqrt(D)``, cast back to ``q.dtype``. The official
release asks for the cuDNN SDPA kernel; this is the MLX SDPA kernel (math
on CPU).
"""
out_dtype = q.dtype
q = q.transpose(0, 2, 1, 3).astype(mx.bfloat16)
k = k.transpose(0, 2, 1, 3).astype(mx.bfloat16)
v = v.transpose(0, 2, 1, 3).astype(mx.bfloat16)
scale = q.shape[-1] ** -0.5
out = mx.fast.scaled_dot_product_attention(q, k, v, scale=scale)
return out.transpose(0, 2, 1, 3).astype(out_dtype)
def sdpa_heads_first(q, k, v):
"""SDPA for ``[B, H, L, D]`` queries. No forced bfloat16 cast (refiner stem)."""
scale = q.shape[-1] ** -0.5
return mx.fast.scaled_dot_product_attention(q, k, v, scale=scale)
def conv3d_ncdhw(x, weight, bias=None, stride=(1, 1, 1)):
"""Conv3d on ``[N, C, D, H, W]``. ``weight`` is MLX layout ``[O, Kd, Kh, Kw, I]``."""
xt = mx.transpose(x, (0, 2, 3, 4, 1))
y = mx.conv3d(xt, weight.astype(xt.dtype), stride=stride, padding=0)
if bias is not None:
y = y + bias.astype(y.dtype)
return mx.transpose(y, (0, 4, 1, 2, 3))
def depthwise_conv3d_ncdhw(x, weight, bias=None):
"""Depthwise conv3d, stride 1, padding 0. ``weight`` is ``[C, Kd, Kh, Kw, 1]``."""
xt = mx.transpose(x, (0, 2, 3, 4, 1))
_, d, h, w, _ = xt.shape
kd, kh, kw = int(weight.shape[1]), int(weight.shape[2]), int(weight.shape[3])
od, oh, ow = d - kd + 1, h - kh + 1, w - kw + 1
if od < 1 or oh < 1 or ow < 1:
raise ValueError(f"depthwise conv output is empty for input {(d, h, w)} kernel {(kd, kh, kw)}")
acc = None
w = weight.astype(xt.dtype)
for kt in range(kd):
for ih in range(kh):
for iw in range(kw):
sl = xt[:, kt : kt + od, ih : ih + oh, iw : iw + ow, :]
coeff = w[:, kt, ih, iw, 0]
term = sl * coeff
acc = term if acc is None else acc + term
if bias is not None:
acc = acc + bias.astype(acc.dtype)
return mx.transpose(acc, (0, 4, 1, 2, 3))