"""Qwen3.5 vision tower and multimodal RoPE for the MLX backend. The vision tower is a trimmed port of mlx_vlm/models/qwen3_vl/vision.py from mlx-vlm 0.7.10 (https://github.com/Blaizzy/mlx-vlm): one image per call, no deepstack (Qwen3.5 has none). It loads the unquantized `vision_tower.*` weights that mlx-lm drops. The interleaved M-RoPE layout follows mlx_vlm/models/rope_utils.py. Those parts are used under the MIT License: MIT License Copyright (c) 2025 Prince Canuma Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. MRope replaces the 1-D rope of the full-attention layers of an mlx-lm Qwen3.5 text model. With `positions` unset it calls the original rope, so text-only calls are bit-identical to stock mlx-lm. """ import glob import json import os import mlx.core as mx import mlx.nn as nn def _rotate_half(x): x1, x2 = mx.split(x, 2, axis=-1) return mx.concatenate([-x2, x1], axis=-1) class _PatchEmbed(nn.Module): def __init__(self, cfg): super().__init__() self.c, self.t, self.p, self.d = cfg["in_channels"], cfg["temporal_patch_size"], cfg["patch_size"], cfg["hidden_size"] k = [self.t, self.p, self.p] self.proj = nn.Conv3d(self.c, self.d, kernel_size=k, stride=k, bias=True) def __call__(self, x): x = x.reshape(-1, self.c, self.t, self.p, self.p).moveaxis(1, 4) return self.proj(x).reshape(-1, self.d) class _Attention(nn.Module): def __init__(self, dim, num_heads): super().__init__() self.num_heads = num_heads self.scale = (dim // num_heads) ** -0.5 self.qkv = nn.Linear(dim, dim * 3, bias=True) self.proj = nn.Linear(dim, dim) def __call__(self, x, cos, sin): n = x.shape[0] q, k, v = mx.split(self.qkv(x).reshape(n, 3, self.num_heads, -1).transpose(1, 0, 2, 3), 3) q, k, v = q[0], k[0], v[0] # n, heads, head_dim q = (q.astype(mx.float32) * cos + _rotate_half(q.astype(mx.float32)) * sin).astype(x.dtype) k = (k.astype(mx.float32) * cos + _rotate_half(k.astype(mx.float32)) * sin).astype(x.dtype) q, k, v = (a.transpose(1, 0, 2)[None] for a in (q, k, v)) # 1, heads, n, head_dim o = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale) return self.proj(o[0].transpose(1, 0, 2).reshape(n, -1)) class _MLP(nn.Module): def __init__(self, dim, hidden): super().__init__() self.linear_fc1 = nn.Linear(dim, hidden, bias=True) self.linear_fc2 = nn.Linear(hidden, dim, bias=True) def __call__(self, x): return self.linear_fc2(nn.gelu_approx(self.linear_fc1(x))) class _Block(nn.Module): def __init__(self, cfg): super().__init__() self.norm1 = nn.LayerNorm(cfg["hidden_size"], eps=1e-6) self.norm2 = nn.LayerNorm(cfg["hidden_size"], eps=1e-6) self.attn = _Attention(cfg["hidden_size"], cfg["num_heads"]) self.mlp = _MLP(cfg["hidden_size"], cfg["intermediate_size"]) def __call__(self, x, cos, sin): x = x + self.attn(self.norm1(x), cos, sin) return x + self.mlp(self.norm2(x)) class _Merger(nn.Module): def __init__(self, cfg): super().__init__() self.hidden = cfg["hidden_size"] * cfg["spatial_merge_size"] ** 2 self.norm = nn.LayerNorm(cfg["hidden_size"], eps=1e-6) self.linear_fc1 = nn.Linear(self.hidden, self.hidden) self.linear_fc2 = nn.Linear(self.hidden, cfg["out_hidden_size"]) def __call__(self, x): x = self.norm(x).reshape(-1, self.hidden) return self.linear_fc2(nn.gelu(self.linear_fc1(x))) class VisionModel(nn.Module): def __init__(self, cfg): super().__init__() self.merge = cfg["spatial_merge_size"] self.patch_embed = _PatchEmbed(cfg) self.pos_embed = nn.Embedding(cfg["num_position_embeddings"], cfg["hidden_size"]) self.side = int(cfg["num_position_embeddings"] ** 0.5) self.blocks = [_Block(cfg) for _ in range(cfg["depth"])] self.merger = _Merger(cfg) hd = cfg["hidden_size"] // cfg["num_heads"] // 2 self._inv_freq = 1.0 / (10000.0 ** (mx.arange(0, hd, 2, dtype=mx.float32) / hd)) def _block_order(self, a, h, w): """Row-major (h, w, ...) -> the processor's 2x2 merge-window order.""" m = self.merge return a.reshape(h // m, m, w // m, m, -1).transpose(0, 2, 1, 3, 4).reshape(h * w, -1) def _pos_embed(self, h, w): s = self.side hi, wi = mx.linspace(0, s - 1, h), mx.linspace(0, s - 1, w) hf, wf = hi.astype(mx.int32), wi.astype(mx.int32) hc, wc = mx.minimum(hf + 1, s - 1), mx.minimum(wf + 1, s - 1) dh, dw = hi - hf.astype(mx.float32), wi - wf.astype(mx.float32) e = self.pos_embed.weight out = 0 for rows, cols, wgt in ((hf, wf, (1 - dh)[:, None] * (1 - dw)[None]), (hf, wc, (1 - dh)[:, None] * dw[None]), (hc, wf, dh[:, None] * (1 - dw)[None]), (hc, wc, dh[:, None] * dw[None])): idx = (rows[:, None] * s + cols[None]).reshape(-1) out = out + e[idx] * wgt.reshape(-1, 1).astype(e.dtype) return self._block_order(out, h, w) def _rope(self, h, w): rows = mx.broadcast_to(mx.arange(h)[:, None], (h, w)).reshape(-1, 1) cols = mx.broadcast_to(mx.arange(w)[None], (h, w)).reshape(-1, 1) coords = self._block_order(mx.concatenate([rows, cols], axis=1), h, w).astype(mx.float32) f = mx.concatenate([coords[:, :1] * self._inv_freq, coords[:, 1:] * self._inv_freq], axis=-1) f = mx.concatenate([f, f], axis=-1)[:, None, :] # n, 1, head_dim return mx.cos(f), mx.sin(f) def __call__(self, patches, grid_thw): """patches [h*w, 1536] from vision_common.prepare_image, grid_thw (1, h, w) -> [h*w/4, out_hidden].""" t, h, w = (int(v) for v in grid_thw) assert t == 1, "one image per call" x = self.patch_embed(patches.astype(self.patch_embed.proj.weight.dtype)) x = x + self._pos_embed(h, w) cos, sin = self._rope(h, w) for blk in self.blocks: x = blk(x, cos, sin) return self.merger(x) def load_vision(path): """VisionModel with the weights in `path`, or None when the model folder has no vision tower.""" cfg = json.load(open(os.path.join(path, "config.json"))) vc = cfg.get("vision_config") if not vc: return None files = sorted(glob.glob(os.path.join(path, "*.safetensors"))) w = {} for f in files: for k, v in mx.load(f).items(): if k.startswith("vision_tower."): w[k[len("vision_tower."):]] = v if not w: return None pw = w["patch_embed.proj.weight"] if pw.shape[-1] == vc["in_channels"] and pw.shape[1] == vc["temporal_patch_size"]: pass # already MLX layout: out, t, kh, kw, in else: # PyTorch layout: out, in, t, kh, kw w["patch_embed.proj.weight"] = pw.transpose(0, 2, 3, 4, 1) w = {k: v for k, v in w.items() if "position_ids" not in k} m = VisionModel(vc) m.load_weights(list(w.items())) mx.eval(m.parameters()) return m class MRope: """Interleaved multimodal RoPE (Qwen3.5) in place of mlx-lm's rope in the full-attention layers. Set `positions` (int array [3, L], absolute positions of the L tokens in the next forward call) before the call and reset it to None afterwards; while None the original rope runs unchanged. """ def __init__(self, base_rope, mrope_section): self.base = base_rope self.positions = None dims, theta = base_rope.dims, base_rope.base half = dims // 2 self.dims = dims self.inv_freq = 1.0 / (theta ** (mx.arange(0, dims, 2, dtype=mx.float32) / dims)) sel = [0] * half for axis, start in ((1, 1), (2, 2)): for i in range(start, min(mrope_section[axis] * 3, half), 3): sel[i] = axis self.selector = mx.array(sel, dtype=mx.int32) def __call__(self, x, offset=0): if self.positions is None: return self.base(x, offset=offset) p = mx.take(self.positions, self.selector, axis=0).astype(mx.float32) # half, L ang = p.T * self.inv_freq # L, half cos, sin = mx.cos(ang), mx.sin(ang) d = self.dims xr = x[..., :d].astype(mx.float32) x1, x2 = xr[..., : d // 2], xr[..., d // 2:] rot = mx.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1).astype(x.dtype) return mx.concatenate([rot, x[..., d:]], axis=-1) if x.shape[-1] > d else rot def install_mrope(lm, mrope_section=None): """Swap the rope of every full-attention layer of an mlx-lm Qwen3.5 TextModel for one shared MRope.""" if mrope_section is None: rp = getattr(lm.args, "rope_parameters", None) or {} mrope_section = rp.get("mrope_section", [11, 11, 10]) mr = None for layer in lm.model.layers: if not layer.is_linear: if mr is None: mr = MRope(layer.self_attn.rope, mrope_section) layer.self_attn.rope = mr return mr