"""Qwen3-VL image tower, DeepStack, and H3's image-release presentation.""" from __future__ import annotations import logging import math import mlx.core as mx import numpy as np from PIL import Image from .ops import attention, layer_norm, rotate from .weights import Linear log = logging.getLogger(__name__) def bound_reference(image, low, high): short = min(image.size) if low <= short <= high: return image scale = (low if short < low else high) / short size = tuple(max(32, round(v * scale / 32) * 32) for v in image.size) return image.resize(size, Image.Resampling.LANCZOS) def preprocess_image(image): image = bound_reference(image.convert("RGB"), 1024, 1024) w, h = image.size if max(w, h) / min(w, h) > 200: raise ValueError("Reference image aspect ratio is too large") nh, nw = max(32, round(h / 32) * 32), max(32, round(w / 32) * 32) if nh * nw > 16777216: beta = math.sqrt(h * w / 16777216) nh, nw = ( max(32, math.floor(h / beta / 32) * 32), max(32, math.floor(w / beta / 32) * 32), ) image = image.resize((nw, nh), Image.Resampling.BICUBIC) values = (np.asarray(image).astype(np.float32) / 255 - 0.5) / 0.5 patches = np.stack((values.transpose(2, 0, 1),) * 2) gh, gw = nh // 16, nw // 16 patches = patches.reshape(1, 2, 3, gh // 2, 2, 16, gw // 2, 2, 16) patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8).reshape(gh * gw, -1) return patches, (gh, gw) def image_presentation(encoder, prompt, images): ids, tags, grids, patches = [], [], [], [] positions = [] cursor = 0 tokenizer, cfg = encoder.tokenizer, encoder.config def text(tokens): nonlocal cursor ids.extend(tokens) tags.extend([1] * len(tokens)) positions.extend([[cursor + i] * 3 for i in range(len(tokens))]) cursor += len(tokens) for index, image in enumerate(images, 1): patch, (gh, gw) = preprocess_image(image) patches.append(patch) grids.append((gh, gw)) text(tokenizer.encode(f"Picture {index}: ", add_special_tokens=False)[:4096]) ids.append(cfg["vision_start_token_id"]) tags.append(0) positions.append([cursor] * 3) cursor += 1 hh, ww = gh // 2, gw // 2 for y in range(hh): for x in range(ww): ids.append(cfg["image_token_id"]) tags.append(0) positions.append([cursor, cursor + y, cursor + x]) cursor += max(hh, ww) ids.append(cfg["vision_end_token_id"]) tags.append(0) positions.append([cursor] * 3) cursor += 1 text(tokenizer.encode(prompt, add_special_tokens=False)[:4096]) pad = -len(ids) % 8 ids += [0] * pad positions += [positions[-1]] * pad return ids, mx.array(tags, mx.int32), np.asarray(positions).T, (patches, grids) def gelu(x, *, approximate=False): y = x.astype(mx.float32) if approximate: y = 0.5 * y * (1 + mx.tanh(math.sqrt(2 / math.pi) * (y + 0.044715 * y * y * y))) else: y = 0.5 * y * (1 + mx.erf(y / math.sqrt(2))) return y.astype(x.dtype) def _merger(store, prefix, x, *, postshuffle=False): if postshuffle: x = x.reshape(-1, 4608) x = layer_norm( x, store.get(prefix + ".norm.weight"), store.get(prefix + ".norm.bias"), 1e-6 ) x = x.reshape(-1, 4608) return Linear.load(store, prefix + ".linear_fc2")( gelu(Linear.load(store, prefix + ".linear_fc1")(x)) ) def _position_embedding(table, h, w): grid = math.isqrt(table.shape[0]) yy = np.linspace(0, grid - 1, h, dtype=np.float32) xx = np.linspace(0, grid - 1, w, dtype=np.float32) y0, x0 = yy.astype(np.int32), xx.astype(np.int32) y1, x1 = np.minimum(y0 + 1, grid - 1), np.minimum(x0 + 1, grid - 1) dy, dx = yy - y0, xx - x0 indices = [ (y0[:, None] * grid + x0).ravel(), (y0[:, None] * grid + x1).ravel(), (y1[:, None] * grid + x0).ravel(), (y1[:, None] * grid + x1).ravel(), ] weights = [ ((1 - dy)[:, None] * (1 - dx)).ravel(), ((1 - dy)[:, None] * dx).ravel(), (dy[:, None] * (1 - dx)).ravel(), (dy[:, None] * dx).ravel(), ] values = [ table[mx.array(i)] * mx.array(v).astype(table.dtype)[:, None] for i, v in zip(indices, weights, strict=False) ] out = values[0] + values[1] + values[2] + values[3] return ( out.reshape(h // 2, 2, w // 2, 2, -1) .transpose(0, 2, 1, 3, 4) .reshape(h * w, -1) ) def encode_vision(store, config, payload): patches, grids = payload prefix = "model.visual" weight = store.get(prefix + ".patch_embed.proj.weight").reshape( config["hidden_size"], -1 ) embed = Linear(weight, store.get(prefix + ".patch_embed.proj.bias")) table = store.get(prefix + ".pos_embed.weight") x = mx.concatenate( [ embed(mx.array(p).astype(mx.bfloat16)) + _position_embedding(table, h, w) for p, (h, w) in zip(patches, grids, strict=False) ] ) coords = [] for h, w in grids: yy, xx = np.meshgrid(np.arange(h), np.arange(w), indexing="ij") c = ( np.stack((yy, xx), axis=-1) .reshape(h // 2, 2, w // 2, 2, 2) .transpose(0, 2, 1, 3, 4) .reshape(-1, 2) ) coords.append(c) pos = mx.array(np.concatenate(coords), mx.float32) dim = config["hidden_size"] // config["num_heads"] inv = 1 / (10000 ** (mx.arange(0, dim // 2, 2, dtype=mx.float32) / (dim // 2))) angles = (pos[..., None] * inv).reshape(pos.shape[0], dim // 2) angles = mx.concatenate((angles, angles), axis=-1) rope = mx.cos(angles)[None, :, None], mx.sin(angles)[None, :, None] lengths = [h * w for h, w in grids] deep = [] for i in range(config["depth"]): p = f"{prefix}.blocks.{i}" h = layer_norm( x, store.get(p + ".norm1.weight"), store.get(p + ".norm1.bias"), 1e-6 ) qkv = Linear.load(store, p + ".attn.qkv")(h).reshape( 1, x.shape[0], 3, config["num_heads"], dim ) q, k, v = (qkv[:, :, j] for j in range(3)) # Vision RoPE computes both products and their sum in FP32, unlike text. q = rotate(q.astype(mx.float32), *rope).astype(mx.bfloat16) k = rotate(k.astype(mx.float32), *rope).astype(mx.bfloat16) chunks, offset = [], 0 for n in lengths: chunks.append( attention( q[:, offset : offset + n], k[:, offset : offset + n], v[:, offset : offset + n], ).reshape(n, -1) ) offset += n x = x + Linear.load(store, p + ".attn.proj")(mx.concatenate(chunks)) h = layer_norm( x, store.get(p + ".norm2.weight"), store.get(p + ".norm2.bias"), 1e-6 ) h = gelu(Linear.load(store, p + ".mlp.linear_fc1")(h), approximate=True) x = x + Linear.load(store, p + ".mlp.linear_fc2")(h) mx.eval(x) if i in config["deepstack_visual_indexes"]: index = config["deepstack_visual_indexes"].index(i) deep.append( _merger( store, f"{prefix}.deepstack_merger_list.{index}", x, postshuffle=True, ) ) mx.eval(deep[-1]) if (i + 1) % 9 == 0: log.info("Vision encoder %d/%d", i + 1, config["depth"]) mx.clear_cache() out = _merger(store, prefix + ".merger", x) mx.eval(out) return out, deep