"""SAM 2.1 (facebook/sam2.1-hiera-large, Apache-2.0) in MLX: a port of transformers' `Sam2Model` for single images (Hiera-L image encoder + FPN neck, SAM 2 prompt encoder and mask decoder; no video memory). import mlx_model model = mlx_model.load("bf16") # a folder with config.json + model.safetensors emb = mlx_model.embed(model, rgb) # rgb HxWx3 uint8 -> image features (+ size) masks, scores, low_res = mlx_model.predict(model, emb, points=[[x, y]], labels=[1], box=None, multimask=True) Prompts in original-image pixels; masks at the original size (logits > 0). Single-mask output picks the most stable mask as SAM 2 does ("dynamic multimask via stability"). Runs on its own next to its weights (mlx, numpy). """ from __future__ import annotations import json from pathlib import Path import mlx.core as mx import mlx.nn as nn import numpy as np CONFIG = {"model_type": "sam2", "image_size": 1024, "hidden_size": 144, "blocks_per_stage": [2, 6, 36, 4], "embed_dim_per_stage": [144, 288, 576, 1152], "heads_per_stage": [2, 4, 8, 16], "window_size_per_stage": [8, 4, 16, 8], "global_attention_blocks": [23, 33, 43], "pos_embed_bg": [7, 7], "fpn_hidden_size": 256, "fpn_top_down_levels": [2, 3], "stability_delta": 0.05, "stability_thresh": 0.98} PRECISIONS = ["bf16", "fp16", "8bit", "6bit", "4bit"] MEAN, STD = np.array([0.485, 0.456, 0.406], np.float32), np.array([0.229, 0.224, 0.225], np.float32) GROUP = 64 def quantizable(path: str, module) -> bool: return isinstance(module, nn.Linear) and module.weight.shape[-1] % GROUP == 0 def resize(x, size): """torch F.interpolate(mode="bilinear", align_corners=False), NHWC.""" b, h, w, c = x.shape oh, ow = int(size[0]), int(size[1]) if (oh, ow) == (h, w): return x def axis(n_in, n_out): s = np.clip((np.arange(n_out, dtype=np.float64) + 0.5) * (n_in / n_out) - 0.5, 0, None) i0 = np.minimum(np.floor(s).astype(np.int32), n_in - 1) return mx.array(i0), mx.array(np.minimum(i0 + 1, n_in - 1)), mx.array((s - i0).astype(np.float32)) y0, y1, fy = axis(h, oh) x0, x1, fx = axis(w, ow) fy, fx = fy.astype(x.dtype)[None, :, None, None], fx.astype(x.dtype)[None, None, :, None] top, bot = x[:, y0], x[:, y1] top = top[:, :, x0] * (1 - fx) + top[:, :, x1] * fx bot = bot[:, :, x0] * (1 - fx) + bot[:, :, x1] * fx return top * (1 - fy) + bot * fy def bicubic_matrix(n_in: int, n_out: int) -> np.ndarray: """torch F.interpolate(mode="bicubic", align_corners=False) along one axis as an (n_out, n_in) matrix (A = -0.75, source index not clamped, taps clamped to the border).""" A = -0.75 m = np.zeros((n_out, n_in), np.float64) scale = n_in / n_out for o in range(n_out): s = (o + 0.5) * scale - 0.5 f = int(np.floor(s)) t = s - f w = [((A * (t + 1) - 5 * A) * (t + 1) + 8 * A) * (t + 1) - 4 * A, ((A + 2) * t - (A + 3)) * t * t + 1, ((A + 2) * (1 - t) - (A + 3)) * (1 - t) * (1 - t) + 1, ((A * (2 - t) - 5 * A) * (2 - t) + 8 * A) * (2 - t) - 4 * A] for j in range(4): m[o, min(max(f - 1 + j, 0), n_in - 1)] += w[j] return m.astype(np.float32) def maxpool2(x): b, h, w, c = x.shape return x[:, :h // 2 * 2, :w // 2 * 2].reshape(b, h // 2, 2, w // 2, 2, c).max(axis=(2, 4)) def window_partition(x, ws): b, h, w, c = x.shape ph, pw = (-h) % ws, (-w) % ws if ph or pw: x = mx.pad(x, [(0, 0), (0, ph), (0, pw), (0, 0)]) hp, wp = h + ph, w + pw x = x.reshape(b, hp // ws, ws, wp // ws, ws, c).transpose(0, 1, 3, 2, 4, 5).reshape(-1, ws, ws, c) return x, (hp, wp) def window_unpartition(x, ws, pad_hw, hw): hp, wp = pad_hw h, w = hw b = x.shape[0] // ((hp // ws) * (wp // ws)) x = x.reshape(b, hp // ws, wp // ws, ws, ws, -1).transpose(0, 1, 3, 2, 4, 5).reshape(b, hp, wp, -1) return x[:, :h, :w] class MultiScaleAttention(nn.Module): def __init__(self, dim, dim_out, heads, q_stride): super().__init__() self.heads, self.q_stride = heads, q_stride self.scale = (dim_out // heads) ** -0.5 self.qkv = nn.Linear(dim, dim_out * 3) self.proj = nn.Linear(dim_out, dim_out) def __call__(self, x): b, h, w, _ = x.shape qkv = self.qkv(x).reshape(b, h * w, 3, self.heads, -1) q, k, v = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2] if self.q_stride: q = maxpool2(q.reshape(b, h, w, -1)) h, w = q.shape[1:3] q = q.reshape(b, h * w, self.heads, -1) q, k, v = (t.transpose(0, 2, 1, 3) for t in (q, k, v)) o = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale) return self.proj(o.transpose(0, 2, 1, 3).reshape(b, h, w, -1)) class Mlp(nn.Module): """Sam2FeedForward: proj_in, (layers), proj_out.""" def __init__(self, din, dh, dout, n, act=nn.relu, sigmoid=False): super().__init__() self.proj_in, self.proj_out = nn.Linear(din, dh), nn.Linear(dh, dout) self.layers = [nn.Linear(dh, dh) for _ in range(n - 2)] self.act, self.sigmoid = act, sigmoid def __call__(self, x): x = self.act(self.proj_in(x)) for l in self.layers: x = self.act(l(x)) x = self.proj_out(x) return mx.sigmoid(x) if self.sigmoid else x class Block(nn.Module): def __init__(self, c, stage, idx, total): super().__init__() first = stage > 0 and idx == 0 self.dim = c["embed_dim_per_stage"][stage - 1] if first else c["embed_dim_per_stage"][stage] self.dim_out = c["embed_dim_per_stage"][stage] ws = c["window_size_per_stage"][stage - 1] if first else c["window_size_per_stage"][stage] self.ws = 0 if total in c["global_attention_blocks"] else ws self.q_stride = 0 < stage <= 3 and idx == 0 self.layer_norm1 = nn.LayerNorm(self.dim, eps=1e-6) self.attn = MultiScaleAttention(self.dim, self.dim_out, c["heads_per_stage"][stage], self.q_stride) self.layer_norm2 = nn.LayerNorm(self.dim_out, eps=1e-6) self.mlp = Mlp(self.dim_out, self.dim_out * 4, self.dim_out, 2, act=nn.gelu) if self.dim != self.dim_out: self.proj = nn.Linear(self.dim, self.dim_out) def __call__(self, x): res = x x = self.layer_norm1(x) if self.dim != self.dim_out: res = self.proj(x) if self.q_stride: res = maxpool2(res) ws = self.ws if ws > 0: hw = x.shape[1:3] x, pad = window_partition(x, ws) x = self.attn(x) if self.q_stride: ws = self.ws // 2 hw = res.shape[1:3] pad = (hw[0] + (-hw[0]) % ws, hw[1] + (-hw[1]) % ws) if self.ws > 0: x = window_unpartition(x, ws, pad, hw) x = res + x return x + self.mlp(self.layer_norm2(x)) class Hiera(nn.Module): def __init__(self, c): super().__init__() d = c["hidden_size"] self.patch_embed = nn.Conv2d(3, d, 7, stride=4, padding=3) self.pos_embed = mx.zeros((1, *c["pos_embed_bg"], d)) # NHWC self.pos_embed_window = mx.zeros((1, c["window_size_per_stage"][0], c["window_size_per_stage"][0], d)) self.blocks, total = [], 0 for s, n in enumerate(c["blocks_per_stage"]): for i in range(n): self.blocks.append(Block(c, s, i, total)) total += 1 self.stage_ends = list(np.cumsum(c["blocks_per_stage"]) - 1) def pos(self, h, w): pe = self.pos_embed.astype(mx.float32) mh = mx.array(bicubic_matrix(pe.shape[1], h)) mw = mx.array(bicubic_matrix(pe.shape[2], w)) pe = mx.einsum("oh,bhwc->bowc", mh, pe) pe = mx.einsum("pw,bowc->bopc", mw, pe) ws = self.pos_embed_window.shape[1] win = mx.tile(self.pos_embed_window.astype(mx.float32), (1, h // ws, w // ws, 1)) return pe + win def __call__(self, x): x = self.patch_embed(x) x = x + self.pos(x.shape[1], x.shape[2]).astype(x.dtype) outs = [] for i, blk in enumerate(self.blocks): x = blk(x) if i in self.stage_ends: outs.append(x) return outs class Neck(nn.Module): def __init__(self, c): super().__init__() self.convs = [nn.Conv2d(d, c["fpn_hidden_size"], 1) for d in c["embed_dim_per_stage"][::-1]] self.top_down = c["fpn_top_down_levels"] def __call__(self, feats): n = len(self.convs) - 1 out, prev = [], None for i in range(n, -1, -1): lat = self.convs[n - i](feats[i]) if i in self.top_down and i != n: b, h, w, ch = prev.shape up = mx.broadcast_to(prev[:, :, None, :, None, :], (b, h, 2, w, 2, ch)).reshape(b, 2 * h, 2 * w, ch) prev = lat + up else: prev = lat out.append(prev) return out[::-1][:3] # 256, 128, 64 px feature maps class LayerNorm2d(nn.Module): def __init__(self, c, eps=1e-6): super().__init__() self.weight, self.bias, self.eps = mx.ones((c,)), mx.zeros((c,)), eps def __call__(self, x): y = x.astype(mx.float32) u = y.mean(-1, keepdims=True) s = ((y - u) ** 2).mean(-1, keepdims=True) return self.weight * ((y - u) * mx.rsqrt(s + self.eps)).astype(x.dtype) + self.bias class Attention(nn.Module): def __init__(self, dim=256, heads=8, down=2): super().__init__() inner = dim // down self.heads = heads self.q_proj, self.k_proj, self.v_proj = nn.Linear(dim, inner), nn.Linear(dim, inner), nn.Linear(dim, inner) self.o_proj = nn.Linear(inner, dim) def __call__(self, q, k, v): q, k, v = self.q_proj(q), self.k_proj(k), self.v_proj(v) b, nq, c = q.shape sp = lambda t: t.reshape(b, t.shape[1], self.heads, c // self.heads).transpose(0, 2, 1, 3) o = mx.fast.scaled_dot_product_attention(sp(q), sp(k), sp(v), scale=(c // self.heads) ** -0.5) return self.o_proj(o.transpose(0, 2, 1, 3).reshape(b, nq, c)) class TwoWayBlock(nn.Module): def __init__(self, skip_pe): super().__init__() self.skip_pe = skip_pe self.self_attn = Attention(down=1) self.cross_attn_token_to_image, self.cross_attn_image_to_token = Attention(), Attention() self.mlp = Mlp(256, 2048, 256, 2) self.layer_norm1, self.layer_norm2 = nn.LayerNorm(256), nn.LayerNorm(256) self.layer_norm3, self.layer_norm4 = nn.LayerNorm(256), nn.LayerNorm(256) def __call__(self, q, k, qpe, kpe): if self.skip_pe: q = self.self_attn(q, q, q) else: t = q + qpe q = q + self.self_attn(t, t, q) q = self.layer_norm1(q) q = self.layer_norm2(q + self.cross_attn_token_to_image(q + qpe, k + kpe, k)) q = self.layer_norm3(q + self.mlp(q)) k = self.layer_norm4(k + self.cross_attn_image_to_token(k + kpe, q + qpe, q)) return q, k class TwoWayTransformer(nn.Module): def __init__(self): super().__init__() self.layers = [TwoWayBlock(i == 0) for i in range(2)] self.final_attn_token_to_image = Attention() self.layer_norm_final_attn = nn.LayerNorm(256) def __call__(self, img, pe, tokens): b, h, w, c = img.shape k, kpe, q = img.reshape(b, h * w, c), pe.reshape(b, h * w, c), tokens for layer in self.layers: q, k = layer(q, k, tokens, kpe) q = self.layer_norm_final_attn(q + self.final_attn_token_to_image(q + tokens, k + kpe, k)) return q, k class MaskDecoder(nn.Module): def __init__(self): super().__init__() self.obj_score_token, self.iou_token, self.mask_tokens = mx.zeros((1, 256)), mx.zeros((1, 256)), mx.zeros((4, 256)) self.transformer = TwoWayTransformer() self.upscale_conv1 = nn.ConvTranspose2d(256, 64, 2, stride=2) self.upscale_layer_norm = LayerNorm2d(64) self.upscale_conv2 = nn.ConvTranspose2d(64, 32, 2, stride=2) self.output_hypernetworks_mlps = [Mlp(256, 256, 32, 3) for _ in range(4)] self.iou_prediction_head = Mlp(256, 256, 4, 3, sigmoid=True) self.conv_s0, self.conv_s1 = nn.Conv2d(256, 32, 1), nn.Conv2d(256, 64, 1) self.pred_obj_score_head = Mlp(256, 256, 1, 3) def __call__(self, img, pe, sparse, dense, s0, s1): b = sparse.shape[0] out = mx.concatenate([self.obj_score_token, self.iou_token, self.mask_tokens], 0) tokens = mx.concatenate([mx.broadcast_to(out[None], (b,) + out.shape), sparse], 1) src = mx.broadcast_to(img + dense, (b,) + img.shape[1:]) hs, src = self.transformer(src, mx.broadcast_to(pe, (b,) + pe.shape[1:]).astype(src.dtype), tokens) _, h, w, c = img.shape up = self.upscale_conv1(src.reshape(b, h, w, c)) + s1 up = nn.gelu(self.upscale_layer_norm(up)) up = nn.gelu(self.upscale_conv2(up) + s0) hyper = mx.stack([self.output_hypernetworks_mlps[i](hs[:, 2 + i]) for i in range(4)], 1) _, uh, uw, uc = up.shape masks = (hyper @ up.reshape(b, uh * uw, uc).transpose(0, 2, 1)).reshape(b, 4, uh, uw) return masks, self.iou_prediction_head(hs[:, 1]), self.pred_obj_score_head(hs[:, 0]) class PromptEncoder(nn.Module): def __init__(self, size): super().__init__() self.size = size self.positional_embedding = mx.zeros((2, 128)) self.point_embed = mx.zeros((4, 256)) self.not_a_point_embed = mx.zeros((1, 256)) self.no_mask_embed = mx.zeros((1, 256)) def pe(self, coords, gauss): c = (2 * coords - 1) @ gauss.astype(mx.float32) c = 2 * np.pi * c return mx.concatenate([mx.sin(c), mx.cos(c)], -1) def __call__(self, points, labels, box): sparse = [] g = self.positional_embedding if points is not None: pts, lab = points + 0.5, labels if box is None: pts = mx.concatenate([pts, mx.zeros((1, 1, 2))], 1) lab = mx.concatenate([lab, -mx.ones((1, 1))], 1) e = self.pe(pts / self.size, g) l = lab[..., None] e = mx.where(l == -1, self.not_a_point_embed.astype(mx.float32), e) idx = mx.maximum(lab, 0).astype(mx.int32) e = e + self.point_embed.astype(mx.float32)[idx] * (l >= 0) sparse.append(e) if box is not None: e = self.pe((box + 0.5).reshape(1, 2, 2) / self.size, g) e = e + self.point_embed[2:4].astype(mx.float32)[None] sparse.append(mx.concatenate([e, self.not_a_point_embed.astype(mx.float32)[None]], 1)) return mx.concatenate(sparse, 1).astype(self.no_mask_embed.dtype) class Model(nn.Module): def __init__(self, config: dict = CONFIG): super().__init__() self.config, self.size = config, config["image_size"] self.backbone = Hiera(config) self.neck = Neck(config) self.no_memory_embedding = mx.zeros((1, 1, 256)) self.shared_image_embedding = mx.zeros((2, 128)) self.prompt_encoder = PromptEncoder(self.size) self.mask_decoder = MaskDecoder() def image_pe(self): n = self.size // 16 g = (mx.arange(n, dtype=mx.float32) + 0.5) / n grid = mx.stack([mx.broadcast_to(g[None, :], (n, n)), mx.broadcast_to(g[:, None], (n, n))], -1) return self.prompt_encoder.pe(grid, self.shared_image_embedding)[None] def load(folder) -> Model: folder = Path(folder) config = json.loads((folder / "config.json").read_text()) model = Model(config) q = config.get("quantization") if q: nn.quantize(model, group_size=q["group_size"], bits=q["bits"], class_predicate=quantizable) model.load_weights(str(folder / "model.safetensors"), strict=True) mx.eval(model.parameters()) return model def embed(model: Model, rgb: np.ndarray) -> dict: """Sam2ImageProcessor: the image resized to 1024x1024 (bilinear, no aspect kept), ImageNet-normalized.""" s = model.size x = resize(mx.array(rgb.astype(np.float32))[None], (s, s)) x = (mx.clip(mx.round(x), 0, 255) / 255.0 - mx.array(MEAN)) / mx.array(STD) return embed_pixels(model, x, rgb.shape[:2]) def embed_pixels(model: Model, x, original_size) -> dict: """The encoder on already preprocessed pixels (1, 1024, 1024, 3), e.g. Sam2Processor's pixel_values in NHWC (its torchvision uint8 resize differs from `embed`'s by up to one level per pixel).""" h, w = original_size dt = model.no_memory_embedding.dtype f0, f1, f2 = model.neck(model.backbone(mx.array(x).astype(dt))) dec = model.mask_decoder feats = {"s0": dec.conv_s0(f0), "s1": dec.conv_s1(f1), "img": f2 + model.no_memory_embedding.reshape(1, 1, 1, -1), "original_size": (h, w)} mx.eval(feats["s0"], feats["s1"], feats["img"]) return feats def predict(model: Model, emb: dict, points=None, labels=None, box=None, multimask=True): h, w = emb["original_size"] s = model.size sc = np.array([s / w, s / h], np.float32) pts = lab = bx = None if points is not None: pts = mx.array(np.asarray(points, np.float32)[None] * sc) lab = mx.array(np.asarray(labels, np.float32)[None]) if box is not None: bx = mx.array(np.asarray(box, np.float32).reshape(1, 4) * np.tile(sc, 2)) sparse = model.prompt_encoder(pts, lab, bx) dense = model.prompt_encoder.no_mask_embed.reshape(1, 1, 1, -1) masks, iou, _ = model.mask_decoder(emb["img"], model.image_pe(), sparse, dense, emb["s0"], emb["s1"]) masks, iou = masks.astype(mx.float32)[0], iou.astype(mx.float32)[0] if multimask: masks, iou = masks[1:], iou[1:] else: single = masks[0] stab = ((single > 0.05).sum() / mx.maximum((single > -0.05).sum(), 1)).item() if (single > -0.05).sum().item() > 0 and stab < model.config["stability_thresh"]: j = int(mx.argmax(iou[1:]).item()) + 1 else: j = 0 masks, iou = masks[j:j + 1], iou[j:j + 1] up = resize(masks[..., None].transpose(0, 1, 2, 3), (h, w))[..., 0] return np.array(up > 0), np.array(iou), np.array(masks) def convert(source: Path) -> dict: """runs/source/model.safetensors (transformers layout) -> this port's names; the video-memory parts dropped.""" from safetensors.numpy import load_file sd = load_file(str(Path(source) / "model.safetensors")) keep = ("vision_encoder.", "prompt_encoder.", "mask_decoder.", "shared_image_embedding.", "no_memory_embedding") out = {} for k, v in sd.items(): if not k.startswith(keep): continue n = k.replace("vision_encoder.backbone.", "backbone.").replace("vision_encoder.neck.", "neck.") n = n.replace("backbone.patch_embed.projection.", "backbone.patch_embed.") n = n.replace("shared_image_embedding.positional_embedding", "shared_image_embedding") n = n.replace("prompt_encoder.shared_embedding.positional_embedding", "prompt_encoder.positional_embedding") for t in ("point_embed", "not_a_point_embed", "no_mask_embed", "obj_score_token", "iou_token", "mask_tokens"): if n.endswith(f".{t}.weight"): n = n[:-len(".weight")] if n.startswith("prompt_encoder.mask_embed"): continue # mask prompts aren't exposed here if n in ("backbone.pos_embed", "backbone.pos_embed_window"): v = v.transpose(0, 2, 3, 1) elif v.ndim == 4: v = v.transpose(1, 2, 3, 0) if "upscale_conv" in n else v.transpose(0, 2, 3, 1) out[n] = v.astype(np.float32) return out