devin-lai's picture
Release v0.4.0: selective MLX model downloads
3da88e3 verified
Raw History Blame Contribute Delete
7.76 kB
"""Qwen3-VL image tower, DeepStack, and H3's image-release presentation."""
from __future__ import annotations
import logging
import math
import mlx.core as mx
import numpy as np
from PIL import Image
from .ops import attention, layer_norm, rotate
from .weights import Linear
log = logging.getLogger(__name__)
def bound_reference(image, low, high):
short = min(image.size)
if low <= short <= high:
return image
scale = (low if short < low else high) / short
size = tuple(max(32, round(v * scale / 32) * 32) for v in image.size)
return image.resize(size, Image.Resampling.LANCZOS)
def preprocess_image(image):
image = bound_reference(image.convert("RGB"), 1024, 1024)
w, h = image.size
if max(w, h) / min(w, h) > 200:
raise ValueError("Reference image aspect ratio is too large")
nh, nw = max(32, round(h / 32) * 32), max(32, round(w / 32) * 32)
if nh * nw > 16777216:
beta = math.sqrt(h * w / 16777216)
nh, nw = (
max(32, math.floor(h / beta / 32) * 32),
max(32, math.floor(w / beta / 32) * 32),
)
image = image.resize((nw, nh), Image.Resampling.BICUBIC)
values = (np.asarray(image).astype(np.float32) / 255 - 0.5) / 0.5
patches = np.stack((values.transpose(2, 0, 1),) * 2)
gh, gw = nh // 16, nw // 16
patches = patches.reshape(1, 2, 3, gh // 2, 2, 16, gw // 2, 2, 16)
patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8).reshape(gh * gw, -1)
return patches, (gh, gw)
def image_presentation(encoder, prompt, images):
ids, tags, grids, patches = [], [], [], []
positions = []
cursor = 0
tokenizer, cfg = encoder.tokenizer, encoder.config
def text(tokens):
nonlocal cursor
ids.extend(tokens)
tags.extend([1] * len(tokens))
positions.extend([[cursor + i] * 3 for i in range(len(tokens))])
cursor += len(tokens)
for index, image in enumerate(images, 1):
patch, (gh, gw) = preprocess_image(image)
patches.append(patch)
grids.append((gh, gw))
text(tokenizer.encode(f"Picture {index}: ", add_special_tokens=False)[:4096])
ids.append(cfg["vision_start_token_id"])
tags.append(0)
positions.append([cursor] * 3)
cursor += 1
hh, ww = gh // 2, gw // 2
for y in range(hh):
for x in range(ww):
ids.append(cfg["image_token_id"])
tags.append(0)
positions.append([cursor, cursor + y, cursor + x])
cursor += max(hh, ww)
ids.append(cfg["vision_end_token_id"])
tags.append(0)
positions.append([cursor] * 3)
cursor += 1
text(tokenizer.encode(prompt, add_special_tokens=False)[:4096])
pad = -len(ids) % 8
ids += [0] * pad
positions += [positions[-1]] * pad
return ids, mx.array(tags, mx.int32), np.asarray(positions).T, (patches, grids)
def gelu(x, *, approximate=False):
y = x.astype(mx.float32)
if approximate:
y = 0.5 * y * (1 + mx.tanh(math.sqrt(2 / math.pi) * (y + 0.044715 * y * y * y)))
else:
y = 0.5 * y * (1 + mx.erf(y / math.sqrt(2)))
return y.astype(x.dtype)
def _merger(store, prefix, x, *, postshuffle=False):
if postshuffle:
x = x.reshape(-1, 4608)
x = layer_norm(
x, store.get(prefix + ".norm.weight"), store.get(prefix + ".norm.bias"), 1e-6
)
x = x.reshape(-1, 4608)
return Linear.load(store, prefix + ".linear_fc2")(
gelu(Linear.load(store, prefix + ".linear_fc1")(x))
)
def _position_embedding(table, h, w):
grid = math.isqrt(table.shape[0])
yy = np.linspace(0, grid - 1, h, dtype=np.float32)
xx = np.linspace(0, grid - 1, w, dtype=np.float32)
y0, x0 = yy.astype(np.int32), xx.astype(np.int32)
y1, x1 = np.minimum(y0 + 1, grid - 1), np.minimum(x0 + 1, grid - 1)
dy, dx = yy - y0, xx - x0
indices = [
(y0[:, None] * grid + x0).ravel(),
(y0[:, None] * grid + x1).ravel(),
(y1[:, None] * grid + x0).ravel(),
(y1[:, None] * grid + x1).ravel(),
]
weights = [
((1 - dy)[:, None] * (1 - dx)).ravel(),
((1 - dy)[:, None] * dx).ravel(),
(dy[:, None] * (1 - dx)).ravel(),
(dy[:, None] * dx).ravel(),
]
values = [
table[mx.array(i)] * mx.array(v).astype(table.dtype)[:, None]
for i, v in zip(indices, weights, strict=False)
]
out = values[0] + values[1] + values[2] + values[3]
return (
out.reshape(h // 2, 2, w // 2, 2, -1)
.transpose(0, 2, 1, 3, 4)
.reshape(h * w, -1)
)
def encode_vision(store, config, payload):
patches, grids = payload
prefix = "model.visual"
weight = store.get(prefix + ".patch_embed.proj.weight").reshape(
config["hidden_size"], -1
)
embed = Linear(weight, store.get(prefix + ".patch_embed.proj.bias"))
table = store.get(prefix + ".pos_embed.weight")
x = mx.concatenate(
[
embed(mx.array(p).astype(mx.bfloat16)) + _position_embedding(table, h, w)
for p, (h, w) in zip(patches, grids, strict=False)
]
)
coords = []
for h, w in grids:
yy, xx = np.meshgrid(np.arange(h), np.arange(w), indexing="ij")
c = (
np.stack((yy, xx), axis=-1)
.reshape(h // 2, 2, w // 2, 2, 2)
.transpose(0, 2, 1, 3, 4)
.reshape(-1, 2)
)
coords.append(c)
pos = mx.array(np.concatenate(coords), mx.float32)
dim = config["hidden_size"] // config["num_heads"]
inv = 1 / (10000 ** (mx.arange(0, dim // 2, 2, dtype=mx.float32) / (dim // 2)))
angles = (pos[..., None] * inv).reshape(pos.shape[0], dim // 2)
angles = mx.concatenate((angles, angles), axis=-1)
rope = mx.cos(angles)[None, :, None], mx.sin(angles)[None, :, None]
lengths = [h * w for h, w in grids]
deep = []
for i in range(config["depth"]):
p = f"{prefix}.blocks.{i}"
h = layer_norm(
x, store.get(p + ".norm1.weight"), store.get(p + ".norm1.bias"), 1e-6
)
qkv = Linear.load(store, p + ".attn.qkv")(h).reshape(
1, x.shape[0], 3, config["num_heads"], dim
)
q, k, v = (qkv[:, :, j] for j in range(3))
# Vision RoPE computes both products and their sum in FP32, unlike text.
q = rotate(q.astype(mx.float32), *rope).astype(mx.bfloat16)
k = rotate(k.astype(mx.float32), *rope).astype(mx.bfloat16)
chunks, offset = [], 0
for n in lengths:
chunks.append(
attention(
q[:, offset : offset + n],
k[:, offset : offset + n],
v[:, offset : offset + n],
).reshape(n, -1)
)
offset += n
x = x + Linear.load(store, p + ".attn.proj")(mx.concatenate(chunks))
h = layer_norm(
x, store.get(p + ".norm2.weight"), store.get(p + ".norm2.bias"), 1e-6
)
h = gelu(Linear.load(store, p + ".mlp.linear_fc1")(h), approximate=True)
x = x + Linear.load(store, p + ".mlp.linear_fc2")(h)
mx.eval(x)
if i in config["deepstack_visual_indexes"]:
index = config["deepstack_visual_indexes"].index(i)
deep.append(
_merger(
store,
f"{prefix}.deepstack_merger_list.{index}",
x,
postshuffle=True,
)
)
mx.eval(deep[-1])
if (i + 1) % 9 == 0:
log.info("Vision encoder %d/%d", i + 1, config["depth"])
mx.clear_cache()
out = _merger(store, prefix + ".merger", x)
mx.eval(out)
return out, deep