Text Classification
MLX
Safetensors
qwen3_5
neojev
jev
system-one
typed-decisions
classification
calibration
vision
apple-silicon
4-bit precision
Instructions to use dowands/neojev-mlx with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use dowands/neojev-mlx with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download dowands/neojev-mlx --local-dir neojev-mlx
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download vision_mlx.py from dowands/neojev-mlx: direct link, hf CLI and curl.
- Browser
- Download file 10.3 kB
-
https://huggingface.co/dowands/neojev-mlx/resolve/main/vision_mlx.py
- Command line
-
hf download hf://dowands/neojev-mlx/vision_mlx.py
-
curl -L -o vision_mlx.py https://huggingface.co/dowands/neojev-mlx/resolve/main/vision_mlx.py
10.3 kB
| """Qwen3.5 vision tower and multimodal RoPE for the MLX backend. | |
| The vision tower is a trimmed port of mlx_vlm/models/qwen3_vl/vision.py from mlx-vlm 0.7.10 | |
| (https://github.com/Blaizzy/mlx-vlm): one image per call, no deepstack (Qwen3.5 has none). It loads the | |
| unquantized `vision_tower.*` weights that mlx-lm drops. The interleaved M-RoPE layout follows | |
| mlx_vlm/models/rope_utils.py. Those parts are used under the MIT License: | |
| MIT License | |
| Copyright (c) 2025 Prince Canuma | |
| Permission is hereby granted, free of charge, to any person obtaining a copy | |
| of this software and associated documentation files (the "Software"), to deal | |
| in the Software without restriction, including without limitation the rights | |
| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
| copies of the Software, and to permit persons to whom the Software is | |
| furnished to do so, subject to the following conditions: | |
| The above copyright notice and this permission notice shall be included in all | |
| copies or substantial portions of the Software. | |
| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| SOFTWARE. | |
| MRope replaces the 1-D rope of the full-attention layers of an mlx-lm Qwen3.5 text model. With | |
| `positions` unset it calls the original rope, so text-only calls are bit-identical to stock mlx-lm. | |
| """ | |
| import glob | |
| import json | |
| import os | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| def _rotate_half(x): | |
| x1, x2 = mx.split(x, 2, axis=-1) | |
| return mx.concatenate([-x2, x1], axis=-1) | |
| class _PatchEmbed(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.c, self.t, self.p, self.d = cfg["in_channels"], cfg["temporal_patch_size"], cfg["patch_size"], cfg["hidden_size"] | |
| k = [self.t, self.p, self.p] | |
| self.proj = nn.Conv3d(self.c, self.d, kernel_size=k, stride=k, bias=True) | |
| def __call__(self, x): | |
| x = x.reshape(-1, self.c, self.t, self.p, self.p).moveaxis(1, 4) | |
| return self.proj(x).reshape(-1, self.d) | |
| class _Attention(nn.Module): | |
| def __init__(self, dim, num_heads): | |
| super().__init__() | |
| self.num_heads = num_heads | |
| self.scale = (dim // num_heads) ** -0.5 | |
| self.qkv = nn.Linear(dim, dim * 3, bias=True) | |
| self.proj = nn.Linear(dim, dim) | |
| def __call__(self, x, cos, sin): | |
| n = x.shape[0] | |
| q, k, v = mx.split(self.qkv(x).reshape(n, 3, self.num_heads, -1).transpose(1, 0, 2, 3), 3) | |
| q, k, v = q[0], k[0], v[0] # n, heads, head_dim | |
| q = (q.astype(mx.float32) * cos + _rotate_half(q.astype(mx.float32)) * sin).astype(x.dtype) | |
| k = (k.astype(mx.float32) * cos + _rotate_half(k.astype(mx.float32)) * sin).astype(x.dtype) | |
| q, k, v = (a.transpose(1, 0, 2)[None] for a in (q, k, v)) # 1, heads, n, head_dim | |
| o = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale) | |
| return self.proj(o[0].transpose(1, 0, 2).reshape(n, -1)) | |
| class _MLP(nn.Module): | |
| def __init__(self, dim, hidden): | |
| super().__init__() | |
| self.linear_fc1 = nn.Linear(dim, hidden, bias=True) | |
| self.linear_fc2 = nn.Linear(hidden, dim, bias=True) | |
| def __call__(self, x): | |
| return self.linear_fc2(nn.gelu_approx(self.linear_fc1(x))) | |
| class _Block(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(cfg["hidden_size"], eps=1e-6) | |
| self.norm2 = nn.LayerNorm(cfg["hidden_size"], eps=1e-6) | |
| self.attn = _Attention(cfg["hidden_size"], cfg["num_heads"]) | |
| self.mlp = _MLP(cfg["hidden_size"], cfg["intermediate_size"]) | |
| def __call__(self, x, cos, sin): | |
| x = x + self.attn(self.norm1(x), cos, sin) | |
| return x + self.mlp(self.norm2(x)) | |
| class _Merger(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.hidden = cfg["hidden_size"] * cfg["spatial_merge_size"] ** 2 | |
| self.norm = nn.LayerNorm(cfg["hidden_size"], eps=1e-6) | |
| self.linear_fc1 = nn.Linear(self.hidden, self.hidden) | |
| self.linear_fc2 = nn.Linear(self.hidden, cfg["out_hidden_size"]) | |
| def __call__(self, x): | |
| x = self.norm(x).reshape(-1, self.hidden) | |
| return self.linear_fc2(nn.gelu(self.linear_fc1(x))) | |
| class VisionModel(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.merge = cfg["spatial_merge_size"] | |
| self.patch_embed = _PatchEmbed(cfg) | |
| self.pos_embed = nn.Embedding(cfg["num_position_embeddings"], cfg["hidden_size"]) | |
| self.side = int(cfg["num_position_embeddings"] ** 0.5) | |
| self.blocks = [_Block(cfg) for _ in range(cfg["depth"])] | |
| self.merger = _Merger(cfg) | |
| hd = cfg["hidden_size"] // cfg["num_heads"] // 2 | |
| self._inv_freq = 1.0 / (10000.0 ** (mx.arange(0, hd, 2, dtype=mx.float32) / hd)) | |
| def _block_order(self, a, h, w): | |
| """Row-major (h, w, ...) -> the processor's 2x2 merge-window order.""" | |
| m = self.merge | |
| return a.reshape(h // m, m, w // m, m, -1).transpose(0, 2, 1, 3, 4).reshape(h * w, -1) | |
| def _pos_embed(self, h, w): | |
| s = self.side | |
| hi, wi = mx.linspace(0, s - 1, h), mx.linspace(0, s - 1, w) | |
| hf, wf = hi.astype(mx.int32), wi.astype(mx.int32) | |
| hc, wc = mx.minimum(hf + 1, s - 1), mx.minimum(wf + 1, s - 1) | |
| dh, dw = hi - hf.astype(mx.float32), wi - wf.astype(mx.float32) | |
| e = self.pos_embed.weight | |
| out = 0 | |
| for rows, cols, wgt in ((hf, wf, (1 - dh)[:, None] * (1 - dw)[None]), (hf, wc, (1 - dh)[:, None] * dw[None]), | |
| (hc, wf, dh[:, None] * (1 - dw)[None]), (hc, wc, dh[:, None] * dw[None])): | |
| idx = (rows[:, None] * s + cols[None]).reshape(-1) | |
| out = out + e[idx] * wgt.reshape(-1, 1).astype(e.dtype) | |
| return self._block_order(out, h, w) | |
| def _rope(self, h, w): | |
| rows = mx.broadcast_to(mx.arange(h)[:, None], (h, w)).reshape(-1, 1) | |
| cols = mx.broadcast_to(mx.arange(w)[None], (h, w)).reshape(-1, 1) | |
| coords = self._block_order(mx.concatenate([rows, cols], axis=1), h, w).astype(mx.float32) | |
| f = mx.concatenate([coords[:, :1] * self._inv_freq, coords[:, 1:] * self._inv_freq], axis=-1) | |
| f = mx.concatenate([f, f], axis=-1)[:, None, :] # n, 1, head_dim | |
| return mx.cos(f), mx.sin(f) | |
| def __call__(self, patches, grid_thw): | |
| """patches [h*w, 1536] from vision_common.prepare_image, grid_thw (1, h, w) -> [h*w/4, out_hidden].""" | |
| t, h, w = (int(v) for v in grid_thw) | |
| assert t == 1, "one image per call" | |
| x = self.patch_embed(patches.astype(self.patch_embed.proj.weight.dtype)) | |
| x = x + self._pos_embed(h, w) | |
| cos, sin = self._rope(h, w) | |
| for blk in self.blocks: | |
| x = blk(x, cos, sin) | |
| return self.merger(x) | |
| def load_vision(path): | |
| """VisionModel with the weights in `path`, or None when the model folder has no vision tower.""" | |
| cfg = json.load(open(os.path.join(path, "config.json"))) | |
| vc = cfg.get("vision_config") | |
| if not vc: | |
| return None | |
| files = sorted(glob.glob(os.path.join(path, "*.safetensors"))) | |
| w = {} | |
| for f in files: | |
| for k, v in mx.load(f).items(): | |
| if k.startswith("vision_tower."): | |
| w[k[len("vision_tower."):]] = v | |
| if not w: | |
| return None | |
| pw = w["patch_embed.proj.weight"] | |
| if pw.shape[-1] == vc["in_channels"] and pw.shape[1] == vc["temporal_patch_size"]: | |
| pass # already MLX layout: out, t, kh, kw, in | |
| else: # PyTorch layout: out, in, t, kh, kw | |
| w["patch_embed.proj.weight"] = pw.transpose(0, 2, 3, 4, 1) | |
| w = {k: v for k, v in w.items() if "position_ids" not in k} | |
| m = VisionModel(vc) | |
| m.load_weights(list(w.items())) | |
| mx.eval(m.parameters()) | |
| return m | |
| class MRope: | |
| """Interleaved multimodal RoPE (Qwen3.5) in place of mlx-lm's rope in the full-attention layers. | |
| Set `positions` (int array [3, L], absolute positions of the L tokens in the next forward call) before | |
| the call and reset it to None afterwards; while None the original rope runs unchanged. | |
| """ | |
| def __init__(self, base_rope, mrope_section): | |
| self.base = base_rope | |
| self.positions = None | |
| dims, theta = base_rope.dims, base_rope.base | |
| half = dims // 2 | |
| self.dims = dims | |
| self.inv_freq = 1.0 / (theta ** (mx.arange(0, dims, 2, dtype=mx.float32) / dims)) | |
| sel = [0] * half | |
| for axis, start in ((1, 1), (2, 2)): | |
| for i in range(start, min(mrope_section[axis] * 3, half), 3): | |
| sel[i] = axis | |
| self.selector = mx.array(sel, dtype=mx.int32) | |
| def __call__(self, x, offset=0): | |
| if self.positions is None: | |
| return self.base(x, offset=offset) | |
| p = mx.take(self.positions, self.selector, axis=0).astype(mx.float32) # half, L | |
| ang = p.T * self.inv_freq # L, half | |
| cos, sin = mx.cos(ang), mx.sin(ang) | |
| d = self.dims | |
| xr = x[..., :d].astype(mx.float32) | |
| x1, x2 = xr[..., : d // 2], xr[..., d // 2:] | |
| rot = mx.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1).astype(x.dtype) | |
| return mx.concatenate([rot, x[..., d:]], axis=-1) if x.shape[-1] > d else rot | |
| def install_mrope(lm, mrope_section=None): | |
| """Swap the rope of every full-attention layer of an mlx-lm Qwen3.5 TextModel for one shared MRope.""" | |
| if mrope_section is None: | |
| rp = getattr(lm.args, "rope_parameters", None) or {} | |
| mrope_section = rp.get("mrope_section", [11, 11, 10]) | |
| mr = None | |
| for layer in lm.model.layers: | |
| if not layer.is_linear: | |
| if mr is None: | |
| mr = MRope(layer.self_attn.rope, mrope_section) | |
| layer.self_attn.rope = mr | |
| return mr | |