neojev-mlx / vision_mlx.py
dowands's picture
Add files using upload-large-folder tool
74630d5 verified
Raw History Blame Contribute Delete
10.3 kB
"""Qwen3.5 vision tower and multimodal RoPE for the MLX backend.
The vision tower is a trimmed port of mlx_vlm/models/qwen3_vl/vision.py from mlx-vlm 0.7.10
(https://github.com/Blaizzy/mlx-vlm): one image per call, no deepstack (Qwen3.5 has none). It loads the
unquantized `vision_tower.*` weights that mlx-lm drops. The interleaved M-RoPE layout follows
mlx_vlm/models/rope_utils.py. Those parts are used under the MIT License:
MIT License
Copyright (c) 2025 Prince Canuma
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
MRope replaces the 1-D rope of the full-attention layers of an mlx-lm Qwen3.5 text model. With
`positions` unset it calls the original rope, so text-only calls are bit-identical to stock mlx-lm.
"""
import glob
import json
import os
import mlx.core as mx
import mlx.nn as nn
def _rotate_half(x):
x1, x2 = mx.split(x, 2, axis=-1)
return mx.concatenate([-x2, x1], axis=-1)
class _PatchEmbed(nn.Module):
def __init__(self, cfg):
super().__init__()
self.c, self.t, self.p, self.d = cfg["in_channels"], cfg["temporal_patch_size"], cfg["patch_size"], cfg["hidden_size"]
k = [self.t, self.p, self.p]
self.proj = nn.Conv3d(self.c, self.d, kernel_size=k, stride=k, bias=True)
def __call__(self, x):
x = x.reshape(-1, self.c, self.t, self.p, self.p).moveaxis(1, 4)
return self.proj(x).reshape(-1, self.d)
class _Attention(nn.Module):
def __init__(self, dim, num_heads):
super().__init__()
self.num_heads = num_heads
self.scale = (dim // num_heads) ** -0.5
self.qkv = nn.Linear(dim, dim * 3, bias=True)
self.proj = nn.Linear(dim, dim)
def __call__(self, x, cos, sin):
n = x.shape[0]
q, k, v = mx.split(self.qkv(x).reshape(n, 3, self.num_heads, -1).transpose(1, 0, 2, 3), 3)
q, k, v = q[0], k[0], v[0] # n, heads, head_dim
q = (q.astype(mx.float32) * cos + _rotate_half(q.astype(mx.float32)) * sin).astype(x.dtype)
k = (k.astype(mx.float32) * cos + _rotate_half(k.astype(mx.float32)) * sin).astype(x.dtype)
q, k, v = (a.transpose(1, 0, 2)[None] for a in (q, k, v)) # 1, heads, n, head_dim
o = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale)
return self.proj(o[0].transpose(1, 0, 2).reshape(n, -1))
class _MLP(nn.Module):
def __init__(self, dim, hidden):
super().__init__()
self.linear_fc1 = nn.Linear(dim, hidden, bias=True)
self.linear_fc2 = nn.Linear(hidden, dim, bias=True)
def __call__(self, x):
return self.linear_fc2(nn.gelu_approx(self.linear_fc1(x)))
class _Block(nn.Module):
def __init__(self, cfg):
super().__init__()
self.norm1 = nn.LayerNorm(cfg["hidden_size"], eps=1e-6)
self.norm2 = nn.LayerNorm(cfg["hidden_size"], eps=1e-6)
self.attn = _Attention(cfg["hidden_size"], cfg["num_heads"])
self.mlp = _MLP(cfg["hidden_size"], cfg["intermediate_size"])
def __call__(self, x, cos, sin):
x = x + self.attn(self.norm1(x), cos, sin)
return x + self.mlp(self.norm2(x))
class _Merger(nn.Module):
def __init__(self, cfg):
super().__init__()
self.hidden = cfg["hidden_size"] * cfg["spatial_merge_size"] ** 2
self.norm = nn.LayerNorm(cfg["hidden_size"], eps=1e-6)
self.linear_fc1 = nn.Linear(self.hidden, self.hidden)
self.linear_fc2 = nn.Linear(self.hidden, cfg["out_hidden_size"])
def __call__(self, x):
x = self.norm(x).reshape(-1, self.hidden)
return self.linear_fc2(nn.gelu(self.linear_fc1(x)))
class VisionModel(nn.Module):
def __init__(self, cfg):
super().__init__()
self.merge = cfg["spatial_merge_size"]
self.patch_embed = _PatchEmbed(cfg)
self.pos_embed = nn.Embedding(cfg["num_position_embeddings"], cfg["hidden_size"])
self.side = int(cfg["num_position_embeddings"] ** 0.5)
self.blocks = [_Block(cfg) for _ in range(cfg["depth"])]
self.merger = _Merger(cfg)
hd = cfg["hidden_size"] // cfg["num_heads"] // 2
self._inv_freq = 1.0 / (10000.0 ** (mx.arange(0, hd, 2, dtype=mx.float32) / hd))
def _block_order(self, a, h, w):
"""Row-major (h, w, ...) -> the processor's 2x2 merge-window order."""
m = self.merge
return a.reshape(h // m, m, w // m, m, -1).transpose(0, 2, 1, 3, 4).reshape(h * w, -1)
def _pos_embed(self, h, w):
s = self.side
hi, wi = mx.linspace(0, s - 1, h), mx.linspace(0, s - 1, w)
hf, wf = hi.astype(mx.int32), wi.astype(mx.int32)
hc, wc = mx.minimum(hf + 1, s - 1), mx.minimum(wf + 1, s - 1)
dh, dw = hi - hf.astype(mx.float32), wi - wf.astype(mx.float32)
e = self.pos_embed.weight
out = 0
for rows, cols, wgt in ((hf, wf, (1 - dh)[:, None] * (1 - dw)[None]), (hf, wc, (1 - dh)[:, None] * dw[None]),
(hc, wf, dh[:, None] * (1 - dw)[None]), (hc, wc, dh[:, None] * dw[None])):
idx = (rows[:, None] * s + cols[None]).reshape(-1)
out = out + e[idx] * wgt.reshape(-1, 1).astype(e.dtype)
return self._block_order(out, h, w)
def _rope(self, h, w):
rows = mx.broadcast_to(mx.arange(h)[:, None], (h, w)).reshape(-1, 1)
cols = mx.broadcast_to(mx.arange(w)[None], (h, w)).reshape(-1, 1)
coords = self._block_order(mx.concatenate([rows, cols], axis=1), h, w).astype(mx.float32)
f = mx.concatenate([coords[:, :1] * self._inv_freq, coords[:, 1:] * self._inv_freq], axis=-1)
f = mx.concatenate([f, f], axis=-1)[:, None, :] # n, 1, head_dim
return mx.cos(f), mx.sin(f)
def __call__(self, patches, grid_thw):
"""patches [h*w, 1536] from vision_common.prepare_image, grid_thw (1, h, w) -> [h*w/4, out_hidden]."""
t, h, w = (int(v) for v in grid_thw)
assert t == 1, "one image per call"
x = self.patch_embed(patches.astype(self.patch_embed.proj.weight.dtype))
x = x + self._pos_embed(h, w)
cos, sin = self._rope(h, w)
for blk in self.blocks:
x = blk(x, cos, sin)
return self.merger(x)
def load_vision(path):
"""VisionModel with the weights in `path`, or None when the model folder has no vision tower."""
cfg = json.load(open(os.path.join(path, "config.json")))
vc = cfg.get("vision_config")
if not vc:
return None
files = sorted(glob.glob(os.path.join(path, "*.safetensors")))
w = {}
for f in files:
for k, v in mx.load(f).items():
if k.startswith("vision_tower."):
w[k[len("vision_tower."):]] = v
if not w:
return None
pw = w["patch_embed.proj.weight"]
if pw.shape[-1] == vc["in_channels"] and pw.shape[1] == vc["temporal_patch_size"]:
pass # already MLX layout: out, t, kh, kw, in
else: # PyTorch layout: out, in, t, kh, kw
w["patch_embed.proj.weight"] = pw.transpose(0, 2, 3, 4, 1)
w = {k: v for k, v in w.items() if "position_ids" not in k}
m = VisionModel(vc)
m.load_weights(list(w.items()))
mx.eval(m.parameters())
return m
class MRope:
"""Interleaved multimodal RoPE (Qwen3.5) in place of mlx-lm's rope in the full-attention layers.
Set `positions` (int array [3, L], absolute positions of the L tokens in the next forward call) before
the call and reset it to None afterwards; while None the original rope runs unchanged.
"""
def __init__(self, base_rope, mrope_section):
self.base = base_rope
self.positions = None
dims, theta = base_rope.dims, base_rope.base
half = dims // 2
self.dims = dims
self.inv_freq = 1.0 / (theta ** (mx.arange(0, dims, 2, dtype=mx.float32) / dims))
sel = [0] * half
for axis, start in ((1, 1), (2, 2)):
for i in range(start, min(mrope_section[axis] * 3, half), 3):
sel[i] = axis
self.selector = mx.array(sel, dtype=mx.int32)
def __call__(self, x, offset=0):
if self.positions is None:
return self.base(x, offset=offset)
p = mx.take(self.positions, self.selector, axis=0).astype(mx.float32) # half, L
ang = p.T * self.inv_freq # L, half
cos, sin = mx.cos(ang), mx.sin(ang)
d = self.dims
xr = x[..., :d].astype(mx.float32)
x1, x2 = xr[..., : d // 2], xr[..., d // 2:]
rot = mx.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1).astype(x.dtype)
return mx.concatenate([rot, x[..., d:]], axis=-1) if x.shape[-1] > d else rot
def install_mrope(lm, mrope_section=None):
"""Swap the rope of every full-attention layer of an mlx-lm Qwen3.5 TextModel for one shared MRope."""
if mrope_section is None:
rp = getattr(lm.args, "rope_parameters", None) or {}
mrope_section = rp.get("mrope_section", [11, 11, 10])
mr = None
for layer in lm.model.layers:
if not layer.is_linear:
if mr is None:
mr = MRope(layer.self_attn.rope, mrope_section)
layer.self_attn.rope = mr
return mr