Text-to-Image
MLX
MiniMax H3
Safetensors
English
apple-silicon
macos
image-to-image
image-editing
image-generation
Instructions to use devin-lai/MiniMax-H3-Image-mlx with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use devin-lai/MiniMax-H3-Image-mlx with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download devin-lai/MiniMax-H3-Image-mlx --local-dir MiniMax-H3-Image-mlx
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download h3_mlx/vision.py from devin-lai/MiniMax-H3-Image-mlx: direct link, hf CLI and curl.
- Browser
- Download file 7.76 kB
-
https://huggingface.co/devin-lai/MiniMax-H3-Image-mlx/resolve/main/h3_mlx/vision.py
- Command line
-
hf download hf://devin-lai/MiniMax-H3-Image-mlx/h3_mlx/vision.py
-
curl -L -o vision.py https://huggingface.co/devin-lai/MiniMax-H3-Image-mlx/resolve/main/h3_mlx/vision.py
7.76 kB
| """Qwen3-VL image tower, DeepStack, and H3's image-release presentation.""" | |
| from __future__ import annotations | |
| import logging | |
| import math | |
| import mlx.core as mx | |
| import numpy as np | |
| from PIL import Image | |
| from .ops import attention, layer_norm, rotate | |
| from .weights import Linear | |
| log = logging.getLogger(__name__) | |
| def bound_reference(image, low, high): | |
| short = min(image.size) | |
| if low <= short <= high: | |
| return image | |
| scale = (low if short < low else high) / short | |
| size = tuple(max(32, round(v * scale / 32) * 32) for v in image.size) | |
| return image.resize(size, Image.Resampling.LANCZOS) | |
| def preprocess_image(image): | |
| image = bound_reference(image.convert("RGB"), 1024, 1024) | |
| w, h = image.size | |
| if max(w, h) / min(w, h) > 200: | |
| raise ValueError("Reference image aspect ratio is too large") | |
| nh, nw = max(32, round(h / 32) * 32), max(32, round(w / 32) * 32) | |
| if nh * nw > 16777216: | |
| beta = math.sqrt(h * w / 16777216) | |
| nh, nw = ( | |
| max(32, math.floor(h / beta / 32) * 32), | |
| max(32, math.floor(w / beta / 32) * 32), | |
| ) | |
| image = image.resize((nw, nh), Image.Resampling.BICUBIC) | |
| values = (np.asarray(image).astype(np.float32) / 255 - 0.5) / 0.5 | |
| patches = np.stack((values.transpose(2, 0, 1),) * 2) | |
| gh, gw = nh // 16, nw // 16 | |
| patches = patches.reshape(1, 2, 3, gh // 2, 2, 16, gw // 2, 2, 16) | |
| patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8).reshape(gh * gw, -1) | |
| return patches, (gh, gw) | |
| def image_presentation(encoder, prompt, images): | |
| ids, tags, grids, patches = [], [], [], [] | |
| positions = [] | |
| cursor = 0 | |
| tokenizer, cfg = encoder.tokenizer, encoder.config | |
| def text(tokens): | |
| nonlocal cursor | |
| ids.extend(tokens) | |
| tags.extend([1] * len(tokens)) | |
| positions.extend([[cursor + i] * 3 for i in range(len(tokens))]) | |
| cursor += len(tokens) | |
| for index, image in enumerate(images, 1): | |
| patch, (gh, gw) = preprocess_image(image) | |
| patches.append(patch) | |
| grids.append((gh, gw)) | |
| text(tokenizer.encode(f"Picture {index}: ", add_special_tokens=False)[:4096]) | |
| ids.append(cfg["vision_start_token_id"]) | |
| tags.append(0) | |
| positions.append([cursor] * 3) | |
| cursor += 1 | |
| hh, ww = gh // 2, gw // 2 | |
| for y in range(hh): | |
| for x in range(ww): | |
| ids.append(cfg["image_token_id"]) | |
| tags.append(0) | |
| positions.append([cursor, cursor + y, cursor + x]) | |
| cursor += max(hh, ww) | |
| ids.append(cfg["vision_end_token_id"]) | |
| tags.append(0) | |
| positions.append([cursor] * 3) | |
| cursor += 1 | |
| text(tokenizer.encode(prompt, add_special_tokens=False)[:4096]) | |
| pad = -len(ids) % 8 | |
| ids += [0] * pad | |
| positions += [positions[-1]] * pad | |
| return ids, mx.array(tags, mx.int32), np.asarray(positions).T, (patches, grids) | |
| def gelu(x, *, approximate=False): | |
| y = x.astype(mx.float32) | |
| if approximate: | |
| y = 0.5 * y * (1 + mx.tanh(math.sqrt(2 / math.pi) * (y + 0.044715 * y * y * y))) | |
| else: | |
| y = 0.5 * y * (1 + mx.erf(y / math.sqrt(2))) | |
| return y.astype(x.dtype) | |
| def _merger(store, prefix, x, *, postshuffle=False): | |
| if postshuffle: | |
| x = x.reshape(-1, 4608) | |
| x = layer_norm( | |
| x, store.get(prefix + ".norm.weight"), store.get(prefix + ".norm.bias"), 1e-6 | |
| ) | |
| x = x.reshape(-1, 4608) | |
| return Linear.load(store, prefix + ".linear_fc2")( | |
| gelu(Linear.load(store, prefix + ".linear_fc1")(x)) | |
| ) | |
| def _position_embedding(table, h, w): | |
| grid = math.isqrt(table.shape[0]) | |
| yy = np.linspace(0, grid - 1, h, dtype=np.float32) | |
| xx = np.linspace(0, grid - 1, w, dtype=np.float32) | |
| y0, x0 = yy.astype(np.int32), xx.astype(np.int32) | |
| y1, x1 = np.minimum(y0 + 1, grid - 1), np.minimum(x0 + 1, grid - 1) | |
| dy, dx = yy - y0, xx - x0 | |
| indices = [ | |
| (y0[:, None] * grid + x0).ravel(), | |
| (y0[:, None] * grid + x1).ravel(), | |
| (y1[:, None] * grid + x0).ravel(), | |
| (y1[:, None] * grid + x1).ravel(), | |
| ] | |
| weights = [ | |
| ((1 - dy)[:, None] * (1 - dx)).ravel(), | |
| ((1 - dy)[:, None] * dx).ravel(), | |
| (dy[:, None] * (1 - dx)).ravel(), | |
| (dy[:, None] * dx).ravel(), | |
| ] | |
| values = [ | |
| table[mx.array(i)] * mx.array(v).astype(table.dtype)[:, None] | |
| for i, v in zip(indices, weights, strict=False) | |
| ] | |
| out = values[0] + values[1] + values[2] + values[3] | |
| return ( | |
| out.reshape(h // 2, 2, w // 2, 2, -1) | |
| .transpose(0, 2, 1, 3, 4) | |
| .reshape(h * w, -1) | |
| ) | |
| def encode_vision(store, config, payload): | |
| patches, grids = payload | |
| prefix = "model.visual" | |
| weight = store.get(prefix + ".patch_embed.proj.weight").reshape( | |
| config["hidden_size"], -1 | |
| ) | |
| embed = Linear(weight, store.get(prefix + ".patch_embed.proj.bias")) | |
| table = store.get(prefix + ".pos_embed.weight") | |
| x = mx.concatenate( | |
| [ | |
| embed(mx.array(p).astype(mx.bfloat16)) + _position_embedding(table, h, w) | |
| for p, (h, w) in zip(patches, grids, strict=False) | |
| ] | |
| ) | |
| coords = [] | |
| for h, w in grids: | |
| yy, xx = np.meshgrid(np.arange(h), np.arange(w), indexing="ij") | |
| c = ( | |
| np.stack((yy, xx), axis=-1) | |
| .reshape(h // 2, 2, w // 2, 2, 2) | |
| .transpose(0, 2, 1, 3, 4) | |
| .reshape(-1, 2) | |
| ) | |
| coords.append(c) | |
| pos = mx.array(np.concatenate(coords), mx.float32) | |
| dim = config["hidden_size"] // config["num_heads"] | |
| inv = 1 / (10000 ** (mx.arange(0, dim // 2, 2, dtype=mx.float32) / (dim // 2))) | |
| angles = (pos[..., None] * inv).reshape(pos.shape[0], dim // 2) | |
| angles = mx.concatenate((angles, angles), axis=-1) | |
| rope = mx.cos(angles)[None, :, None], mx.sin(angles)[None, :, None] | |
| lengths = [h * w for h, w in grids] | |
| deep = [] | |
| for i in range(config["depth"]): | |
| p = f"{prefix}.blocks.{i}" | |
| h = layer_norm( | |
| x, store.get(p + ".norm1.weight"), store.get(p + ".norm1.bias"), 1e-6 | |
| ) | |
| qkv = Linear.load(store, p + ".attn.qkv")(h).reshape( | |
| 1, x.shape[0], 3, config["num_heads"], dim | |
| ) | |
| q, k, v = (qkv[:, :, j] for j in range(3)) | |
| # Vision RoPE computes both products and their sum in FP32, unlike text. | |
| q = rotate(q.astype(mx.float32), *rope).astype(mx.bfloat16) | |
| k = rotate(k.astype(mx.float32), *rope).astype(mx.bfloat16) | |
| chunks, offset = [], 0 | |
| for n in lengths: | |
| chunks.append( | |
| attention( | |
| q[:, offset : offset + n], | |
| k[:, offset : offset + n], | |
| v[:, offset : offset + n], | |
| ).reshape(n, -1) | |
| ) | |
| offset += n | |
| x = x + Linear.load(store, p + ".attn.proj")(mx.concatenate(chunks)) | |
| h = layer_norm( | |
| x, store.get(p + ".norm2.weight"), store.get(p + ".norm2.bias"), 1e-6 | |
| ) | |
| h = gelu(Linear.load(store, p + ".mlp.linear_fc1")(h), approximate=True) | |
| x = x + Linear.load(store, p + ".mlp.linear_fc2")(h) | |
| mx.eval(x) | |
| if i in config["deepstack_visual_indexes"]: | |
| index = config["deepstack_visual_indexes"].index(i) | |
| deep.append( | |
| _merger( | |
| store, | |
| f"{prefix}.deepstack_merger_list.{index}", | |
| x, | |
| postshuffle=True, | |
| ) | |
| ) | |
| mx.eval(deep[-1]) | |
| if (i + 1) % 9 == 0: | |
| log.info("Vision encoder %d/%d", i + 1, config["depth"]) | |
| mx.clear_cache() | |
| out = _merger(store, prefix + ".merger", x) | |
| mx.eval(out) | |
| return out, deep | |