"""Small numeric helpers shared by the MLX modules. LayerNorm and RMSNorm follow the Wan / FastFrames definitions: the reduction is in float32, the result is cast back to the activation dtype, and RMSNorm multiplies the weight after that cast. """ from __future__ import annotations import math import numpy as np import mlx.core as mx def gelu_tanh(x: mx.array) -> mx.array: """PyTorch ``GELU(approximate='tanh')`` and the umT5 encoder GELU.""" c = math.sqrt(2.0 / math.pi) return 0.5 * x * (1.0 + mx.tanh(c * (x + 0.044715 * (x * x * x)))) def gelu_erf(x: mx.array) -> mx.array: """Exact PyTorch ``GELU()`` (erf), used by the pixel-decoder ConvNeXt blocks.""" return 0.5 * x * (1.0 + mx.erf(x / math.sqrt(2.0))) def silu(x: mx.array) -> mx.array: return x * mx.sigmoid(x) def layer_norm(x, eps, weight=None, bias=None): xf = x.astype(mx.float32) mean = mx.mean(xf, axis=-1, keepdims=True) var = mx.mean(mx.square(xf - mean), axis=-1, keepdims=True) y = (xf - mean) * mx.rsqrt(var + eps) if weight is not None: y = y * weight.astype(mx.float32) if bias is not None: y = y + bias.astype(mx.float32) return y.astype(x.dtype) def rms_norm(x, weight, eps): xf = x.astype(mx.float32) y = xf * mx.rsqrt(mx.mean(mx.square(xf), axis=-1, keepdims=True) + eps) return y.astype(x.dtype) * weight def sinusoidal_embedding_1d(dim: int, position: mx.array) -> mx.array: """Wan / DiT timestep embedding. Angles are NumPy float64, matching PyTorch's float64 sinusoid. The array handed back to MLX is float32 so it can live on the GPU. """ if dim % 2 != 0: raise ValueError(f"sinusoidal dim must be even, got {dim}") half = dim // 2 pos = np.asarray(position.astype(mx.float32)).astype(np.float64).reshape(-1) div = np.arange(half, dtype=np.float64) / np.float64(half) sinusoid = pos[:, None] * np.power(np.float64(10000.0), -div)[None, :] emb = np.concatenate([np.cos(sinusoid), np.sin(sinusoid)], axis=1).astype(np.float32) return mx.array(emb) def sdpa(q, k, v): """Scaled dot-product attention. ``q, k, v`` are ``[B, L, H, D]``. Matches FastFrames' ``attention``: cast to bfloat16, scale by ``1/sqrt(D)``, cast back to ``q.dtype``. The official release asks for the cuDNN SDPA kernel; this is the MLX SDPA kernel (math on CPU). """ out_dtype = q.dtype q = q.transpose(0, 2, 1, 3).astype(mx.bfloat16) k = k.transpose(0, 2, 1, 3).astype(mx.bfloat16) v = v.transpose(0, 2, 1, 3).astype(mx.bfloat16) scale = q.shape[-1] ** -0.5 out = mx.fast.scaled_dot_product_attention(q, k, v, scale=scale) return out.transpose(0, 2, 1, 3).astype(out_dtype) def sdpa_heads_first(q, k, v): """SDPA for ``[B, H, L, D]`` queries. No forced bfloat16 cast (refiner stem).""" scale = q.shape[-1] ** -0.5 return mx.fast.scaled_dot_product_attention(q, k, v, scale=scale) def conv3d_ncdhw(x, weight, bias=None, stride=(1, 1, 1)): """Conv3d on ``[N, C, D, H, W]``. ``weight`` is MLX layout ``[O, Kd, Kh, Kw, I]``.""" xt = mx.transpose(x, (0, 2, 3, 4, 1)) y = mx.conv3d(xt, weight.astype(xt.dtype), stride=stride, padding=0) if bias is not None: y = y + bias.astype(y.dtype) return mx.transpose(y, (0, 4, 1, 2, 3)) def depthwise_conv3d_ncdhw(x, weight, bias=None): """Depthwise conv3d, stride 1, padding 0. ``weight`` is ``[C, Kd, Kh, Kw, 1]``.""" xt = mx.transpose(x, (0, 2, 3, 4, 1)) _, d, h, w, _ = xt.shape kd, kh, kw = int(weight.shape[1]), int(weight.shape[2]), int(weight.shape[3]) od, oh, ow = d - kd + 1, h - kh + 1, w - kw + 1 if od < 1 or oh < 1 or ow < 1: raise ValueError(f"depthwise conv output is empty for input {(d, h, w)} kernel {(kd, kh, kw)}") acc = None w = weight.astype(xt.dtype) for kt in range(kd): for ih in range(kh): for iw in range(kw): sl = xt[:, kt : kt + od, ih : ih + oh, iw : iw + ow, :] coeff = w[:, kt, ih, iw, 0] term = sl * coeff acc = term if acc is None else acc + term if bias is not None: acc = acc + bias.astype(acc.dtype) return mx.transpose(acc, (0, 4, 1, 2, 3))