sam2.1-hiera-large-MLX / 4bit /mlx_model.py
jennyxue's picture
4bit build of SAM 2.1
b326041 verified
Raw History Blame Contribute Delete
20 kB
"""SAM 2.1 (facebook/sam2.1-hiera-large, Apache-2.0) in MLX: a port of transformers' `Sam2Model` for single images
(Hiera-L image encoder + FPN neck, SAM 2 prompt encoder and mask decoder; no video memory).
import mlx_model
model = mlx_model.load("bf16") # a folder with config.json + model.safetensors
emb = mlx_model.embed(model, rgb) # rgb HxWx3 uint8 -> image features (+ size)
masks, scores, low_res = mlx_model.predict(model, emb, points=[[x, y]], labels=[1], box=None, multimask=True)
Prompts in original-image pixels; masks at the original size (logits > 0). Single-mask output picks the most stable
mask as SAM 2 does ("dynamic multimask via stability"). Runs on its own next to its weights (mlx, numpy).
"""
from __future__ import annotations
import json
from pathlib import Path
import mlx.core as mx
import mlx.nn as nn
import numpy as np
CONFIG = {"model_type": "sam2", "image_size": 1024, "hidden_size": 144, "blocks_per_stage": [2, 6, 36, 4],
"embed_dim_per_stage": [144, 288, 576, 1152], "heads_per_stage": [2, 4, 8, 16],
"window_size_per_stage": [8, 4, 16, 8], "global_attention_blocks": [23, 33, 43], "pos_embed_bg": [7, 7],
"fpn_hidden_size": 256, "fpn_top_down_levels": [2, 3], "stability_delta": 0.05, "stability_thresh": 0.98}
PRECISIONS = ["bf16", "fp16", "8bit", "6bit", "4bit"]
MEAN, STD = np.array([0.485, 0.456, 0.406], np.float32), np.array([0.229, 0.224, 0.225], np.float32)
GROUP = 64
def quantizable(path: str, module) -> bool:
return isinstance(module, nn.Linear) and module.weight.shape[-1] % GROUP == 0
def resize(x, size):
"""torch F.interpolate(mode="bilinear", align_corners=False), NHWC."""
b, h, w, c = x.shape
oh, ow = int(size[0]), int(size[1])
if (oh, ow) == (h, w):
return x
def axis(n_in, n_out):
s = np.clip((np.arange(n_out, dtype=np.float64) + 0.5) * (n_in / n_out) - 0.5, 0, None)
i0 = np.minimum(np.floor(s).astype(np.int32), n_in - 1)
return mx.array(i0), mx.array(np.minimum(i0 + 1, n_in - 1)), mx.array((s - i0).astype(np.float32))
y0, y1, fy = axis(h, oh)
x0, x1, fx = axis(w, ow)
fy, fx = fy.astype(x.dtype)[None, :, None, None], fx.astype(x.dtype)[None, None, :, None]
top, bot = x[:, y0], x[:, y1]
top = top[:, :, x0] * (1 - fx) + top[:, :, x1] * fx
bot = bot[:, :, x0] * (1 - fx) + bot[:, :, x1] * fx
return top * (1 - fy) + bot * fy
def bicubic_matrix(n_in: int, n_out: int) -> np.ndarray:
"""torch F.interpolate(mode="bicubic", align_corners=False) along one axis as an (n_out, n_in) matrix
(A = -0.75, source index not clamped, taps clamped to the border)."""
A = -0.75
m = np.zeros((n_out, n_in), np.float64)
scale = n_in / n_out
for o in range(n_out):
s = (o + 0.5) * scale - 0.5
f = int(np.floor(s))
t = s - f
w = [((A * (t + 1) - 5 * A) * (t + 1) + 8 * A) * (t + 1) - 4 * A,
((A + 2) * t - (A + 3)) * t * t + 1,
((A + 2) * (1 - t) - (A + 3)) * (1 - t) * (1 - t) + 1,
((A * (2 - t) - 5 * A) * (2 - t) + 8 * A) * (2 - t) - 4 * A]
for j in range(4):
m[o, min(max(f - 1 + j, 0), n_in - 1)] += w[j]
return m.astype(np.float32)
def maxpool2(x):
b, h, w, c = x.shape
return x[:, :h // 2 * 2, :w // 2 * 2].reshape(b, h // 2, 2, w // 2, 2, c).max(axis=(2, 4))
def window_partition(x, ws):
b, h, w, c = x.shape
ph, pw = (-h) % ws, (-w) % ws
if ph or pw:
x = mx.pad(x, [(0, 0), (0, ph), (0, pw), (0, 0)])
hp, wp = h + ph, w + pw
x = x.reshape(b, hp // ws, ws, wp // ws, ws, c).transpose(0, 1, 3, 2, 4, 5).reshape(-1, ws, ws, c)
return x, (hp, wp)
def window_unpartition(x, ws, pad_hw, hw):
hp, wp = pad_hw
h, w = hw
b = x.shape[0] // ((hp // ws) * (wp // ws))
x = x.reshape(b, hp // ws, wp // ws, ws, ws, -1).transpose(0, 1, 3, 2, 4, 5).reshape(b, hp, wp, -1)
return x[:, :h, :w]
class MultiScaleAttention(nn.Module):
def __init__(self, dim, dim_out, heads, q_stride):
super().__init__()
self.heads, self.q_stride = heads, q_stride
self.scale = (dim_out // heads) ** -0.5
self.qkv = nn.Linear(dim, dim_out * 3)
self.proj = nn.Linear(dim_out, dim_out)
def __call__(self, x):
b, h, w, _ = x.shape
qkv = self.qkv(x).reshape(b, h * w, 3, self.heads, -1)
q, k, v = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2]
if self.q_stride:
q = maxpool2(q.reshape(b, h, w, -1))
h, w = q.shape[1:3]
q = q.reshape(b, h * w, self.heads, -1)
q, k, v = (t.transpose(0, 2, 1, 3) for t in (q, k, v))
o = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale)
return self.proj(o.transpose(0, 2, 1, 3).reshape(b, h, w, -1))
class Mlp(nn.Module):
"""Sam2FeedForward: proj_in, (layers), proj_out."""
def __init__(self, din, dh, dout, n, act=nn.relu, sigmoid=False):
super().__init__()
self.proj_in, self.proj_out = nn.Linear(din, dh), nn.Linear(dh, dout)
self.layers = [nn.Linear(dh, dh) for _ in range(n - 2)]
self.act, self.sigmoid = act, sigmoid
def __call__(self, x):
x = self.act(self.proj_in(x))
for l in self.layers:
x = self.act(l(x))
x = self.proj_out(x)
return mx.sigmoid(x) if self.sigmoid else x
class Block(nn.Module):
def __init__(self, c, stage, idx, total):
super().__init__()
first = stage > 0 and idx == 0
self.dim = c["embed_dim_per_stage"][stage - 1] if first else c["embed_dim_per_stage"][stage]
self.dim_out = c["embed_dim_per_stage"][stage]
ws = c["window_size_per_stage"][stage - 1] if first else c["window_size_per_stage"][stage]
self.ws = 0 if total in c["global_attention_blocks"] else ws
self.q_stride = 0 < stage <= 3 and idx == 0
self.layer_norm1 = nn.LayerNorm(self.dim, eps=1e-6)
self.attn = MultiScaleAttention(self.dim, self.dim_out, c["heads_per_stage"][stage], self.q_stride)
self.layer_norm2 = nn.LayerNorm(self.dim_out, eps=1e-6)
self.mlp = Mlp(self.dim_out, self.dim_out * 4, self.dim_out, 2, act=nn.gelu)
if self.dim != self.dim_out:
self.proj = nn.Linear(self.dim, self.dim_out)
def __call__(self, x):
res = x
x = self.layer_norm1(x)
if self.dim != self.dim_out:
res = self.proj(x)
if self.q_stride:
res = maxpool2(res)
ws = self.ws
if ws > 0:
hw = x.shape[1:3]
x, pad = window_partition(x, ws)
x = self.attn(x)
if self.q_stride:
ws = self.ws // 2
hw = res.shape[1:3]
pad = (hw[0] + (-hw[0]) % ws, hw[1] + (-hw[1]) % ws)
if self.ws > 0:
x = window_unpartition(x, ws, pad, hw)
x = res + x
return x + self.mlp(self.layer_norm2(x))
class Hiera(nn.Module):
def __init__(self, c):
super().__init__()
d = c["hidden_size"]
self.patch_embed = nn.Conv2d(3, d, 7, stride=4, padding=3)
self.pos_embed = mx.zeros((1, *c["pos_embed_bg"], d)) # NHWC
self.pos_embed_window = mx.zeros((1, c["window_size_per_stage"][0], c["window_size_per_stage"][0], d))
self.blocks, total = [], 0
for s, n in enumerate(c["blocks_per_stage"]):
for i in range(n):
self.blocks.append(Block(c, s, i, total))
total += 1
self.stage_ends = list(np.cumsum(c["blocks_per_stage"]) - 1)
def pos(self, h, w):
pe = self.pos_embed.astype(mx.float32)
mh = mx.array(bicubic_matrix(pe.shape[1], h))
mw = mx.array(bicubic_matrix(pe.shape[2], w))
pe = mx.einsum("oh,bhwc->bowc", mh, pe)
pe = mx.einsum("pw,bowc->bopc", mw, pe)
ws = self.pos_embed_window.shape[1]
win = mx.tile(self.pos_embed_window.astype(mx.float32), (1, h // ws, w // ws, 1))
return pe + win
def __call__(self, x):
x = self.patch_embed(x)
x = x + self.pos(x.shape[1], x.shape[2]).astype(x.dtype)
outs = []
for i, blk in enumerate(self.blocks):
x = blk(x)
if i in self.stage_ends:
outs.append(x)
return outs
class Neck(nn.Module):
def __init__(self, c):
super().__init__()
self.convs = [nn.Conv2d(d, c["fpn_hidden_size"], 1) for d in c["embed_dim_per_stage"][::-1]]
self.top_down = c["fpn_top_down_levels"]
def __call__(self, feats):
n = len(self.convs) - 1
out, prev = [], None
for i in range(n, -1, -1):
lat = self.convs[n - i](feats[i])
if i in self.top_down and i != n:
b, h, w, ch = prev.shape
up = mx.broadcast_to(prev[:, :, None, :, None, :], (b, h, 2, w, 2, ch)).reshape(b, 2 * h, 2 * w, ch)
prev = lat + up
else:
prev = lat
out.append(prev)
return out[::-1][:3] # 256, 128, 64 px feature maps
class LayerNorm2d(nn.Module):
def __init__(self, c, eps=1e-6):
super().__init__()
self.weight, self.bias, self.eps = mx.ones((c,)), mx.zeros((c,)), eps
def __call__(self, x):
y = x.astype(mx.float32)
u = y.mean(-1, keepdims=True)
s = ((y - u) ** 2).mean(-1, keepdims=True)
return self.weight * ((y - u) * mx.rsqrt(s + self.eps)).astype(x.dtype) + self.bias
class Attention(nn.Module):
def __init__(self, dim=256, heads=8, down=2):
super().__init__()
inner = dim // down
self.heads = heads
self.q_proj, self.k_proj, self.v_proj = nn.Linear(dim, inner), nn.Linear(dim, inner), nn.Linear(dim, inner)
self.o_proj = nn.Linear(inner, dim)
def __call__(self, q, k, v):
q, k, v = self.q_proj(q), self.k_proj(k), self.v_proj(v)
b, nq, c = q.shape
sp = lambda t: t.reshape(b, t.shape[1], self.heads, c // self.heads).transpose(0, 2, 1, 3)
o = mx.fast.scaled_dot_product_attention(sp(q), sp(k), sp(v), scale=(c // self.heads) ** -0.5)
return self.o_proj(o.transpose(0, 2, 1, 3).reshape(b, nq, c))
class TwoWayBlock(nn.Module):
def __init__(self, skip_pe):
super().__init__()
self.skip_pe = skip_pe
self.self_attn = Attention(down=1)
self.cross_attn_token_to_image, self.cross_attn_image_to_token = Attention(), Attention()
self.mlp = Mlp(256, 2048, 256, 2)
self.layer_norm1, self.layer_norm2 = nn.LayerNorm(256), nn.LayerNorm(256)
self.layer_norm3, self.layer_norm4 = nn.LayerNorm(256), nn.LayerNorm(256)
def __call__(self, q, k, qpe, kpe):
if self.skip_pe:
q = self.self_attn(q, q, q)
else:
t = q + qpe
q = q + self.self_attn(t, t, q)
q = self.layer_norm1(q)
q = self.layer_norm2(q + self.cross_attn_token_to_image(q + qpe, k + kpe, k))
q = self.layer_norm3(q + self.mlp(q))
k = self.layer_norm4(k + self.cross_attn_image_to_token(k + kpe, q + qpe, q))
return q, k
class TwoWayTransformer(nn.Module):
def __init__(self):
super().__init__()
self.layers = [TwoWayBlock(i == 0) for i in range(2)]
self.final_attn_token_to_image = Attention()
self.layer_norm_final_attn = nn.LayerNorm(256)
def __call__(self, img, pe, tokens):
b, h, w, c = img.shape
k, kpe, q = img.reshape(b, h * w, c), pe.reshape(b, h * w, c), tokens
for layer in self.layers:
q, k = layer(q, k, tokens, kpe)
q = self.layer_norm_final_attn(q + self.final_attn_token_to_image(q + tokens, k + kpe, k))
return q, k
class MaskDecoder(nn.Module):
def __init__(self):
super().__init__()
self.obj_score_token, self.iou_token, self.mask_tokens = mx.zeros((1, 256)), mx.zeros((1, 256)), mx.zeros((4, 256))
self.transformer = TwoWayTransformer()
self.upscale_conv1 = nn.ConvTranspose2d(256, 64, 2, stride=2)
self.upscale_layer_norm = LayerNorm2d(64)
self.upscale_conv2 = nn.ConvTranspose2d(64, 32, 2, stride=2)
self.output_hypernetworks_mlps = [Mlp(256, 256, 32, 3) for _ in range(4)]
self.iou_prediction_head = Mlp(256, 256, 4, 3, sigmoid=True)
self.conv_s0, self.conv_s1 = nn.Conv2d(256, 32, 1), nn.Conv2d(256, 64, 1)
self.pred_obj_score_head = Mlp(256, 256, 1, 3)
def __call__(self, img, pe, sparse, dense, s0, s1):
b = sparse.shape[0]
out = mx.concatenate([self.obj_score_token, self.iou_token, self.mask_tokens], 0)
tokens = mx.concatenate([mx.broadcast_to(out[None], (b,) + out.shape), sparse], 1)
src = mx.broadcast_to(img + dense, (b,) + img.shape[1:])
hs, src = self.transformer(src, mx.broadcast_to(pe, (b,) + pe.shape[1:]).astype(src.dtype), tokens)
_, h, w, c = img.shape
up = self.upscale_conv1(src.reshape(b, h, w, c)) + s1
up = nn.gelu(self.upscale_layer_norm(up))
up = nn.gelu(self.upscale_conv2(up) + s0)
hyper = mx.stack([self.output_hypernetworks_mlps[i](hs[:, 2 + i]) for i in range(4)], 1)
_, uh, uw, uc = up.shape
masks = (hyper @ up.reshape(b, uh * uw, uc).transpose(0, 2, 1)).reshape(b, 4, uh, uw)
return masks, self.iou_prediction_head(hs[:, 1]), self.pred_obj_score_head(hs[:, 0])
class PromptEncoder(nn.Module):
def __init__(self, size):
super().__init__()
self.size = size
self.positional_embedding = mx.zeros((2, 128))
self.point_embed = mx.zeros((4, 256))
self.not_a_point_embed = mx.zeros((1, 256))
self.no_mask_embed = mx.zeros((1, 256))
def pe(self, coords, gauss):
c = (2 * coords - 1) @ gauss.astype(mx.float32)
c = 2 * np.pi * c
return mx.concatenate([mx.sin(c), mx.cos(c)], -1)
def __call__(self, points, labels, box):
sparse = []
g = self.positional_embedding
if points is not None:
pts, lab = points + 0.5, labels
if box is None:
pts = mx.concatenate([pts, mx.zeros((1, 1, 2))], 1)
lab = mx.concatenate([lab, -mx.ones((1, 1))], 1)
e = self.pe(pts / self.size, g)
l = lab[..., None]
e = mx.where(l == -1, self.not_a_point_embed.astype(mx.float32), e)
idx = mx.maximum(lab, 0).astype(mx.int32)
e = e + self.point_embed.astype(mx.float32)[idx] * (l >= 0)
sparse.append(e)
if box is not None:
e = self.pe((box + 0.5).reshape(1, 2, 2) / self.size, g)
e = e + self.point_embed[2:4].astype(mx.float32)[None]
sparse.append(mx.concatenate([e, self.not_a_point_embed.astype(mx.float32)[None]], 1))
return mx.concatenate(sparse, 1).astype(self.no_mask_embed.dtype)
class Model(nn.Module):
def __init__(self, config: dict = CONFIG):
super().__init__()
self.config, self.size = config, config["image_size"]
self.backbone = Hiera(config)
self.neck = Neck(config)
self.no_memory_embedding = mx.zeros((1, 1, 256))
self.shared_image_embedding = mx.zeros((2, 128))
self.prompt_encoder = PromptEncoder(self.size)
self.mask_decoder = MaskDecoder()
def image_pe(self):
n = self.size // 16
g = (mx.arange(n, dtype=mx.float32) + 0.5) / n
grid = mx.stack([mx.broadcast_to(g[None, :], (n, n)), mx.broadcast_to(g[:, None], (n, n))], -1)
return self.prompt_encoder.pe(grid, self.shared_image_embedding)[None]
def load(folder) -> Model:
folder = Path(folder)
config = json.loads((folder / "config.json").read_text())
model = Model(config)
q = config.get("quantization")
if q:
nn.quantize(model, group_size=q["group_size"], bits=q["bits"], class_predicate=quantizable)
model.load_weights(str(folder / "model.safetensors"), strict=True)
mx.eval(model.parameters())
return model
def embed(model: Model, rgb: np.ndarray) -> dict:
"""Sam2ImageProcessor: the image resized to 1024x1024 (bilinear, no aspect kept), ImageNet-normalized."""
s = model.size
x = resize(mx.array(rgb.astype(np.float32))[None], (s, s))
x = (mx.clip(mx.round(x), 0, 255) / 255.0 - mx.array(MEAN)) / mx.array(STD)
return embed_pixels(model, x, rgb.shape[:2])
def embed_pixels(model: Model, x, original_size) -> dict:
"""The encoder on already preprocessed pixels (1, 1024, 1024, 3), e.g. Sam2Processor's pixel_values in NHWC
(its torchvision uint8 resize differs from `embed`'s by up to one level per pixel)."""
h, w = original_size
dt = model.no_memory_embedding.dtype
f0, f1, f2 = model.neck(model.backbone(mx.array(x).astype(dt)))
dec = model.mask_decoder
feats = {"s0": dec.conv_s0(f0), "s1": dec.conv_s1(f1), "img": f2 + model.no_memory_embedding.reshape(1, 1, 1, -1),
"original_size": (h, w)}
mx.eval(feats["s0"], feats["s1"], feats["img"])
return feats
def predict(model: Model, emb: dict, points=None, labels=None, box=None, multimask=True):
h, w = emb["original_size"]
s = model.size
sc = np.array([s / w, s / h], np.float32)
pts = lab = bx = None
if points is not None:
pts = mx.array(np.asarray(points, np.float32)[None] * sc)
lab = mx.array(np.asarray(labels, np.float32)[None])
if box is not None:
bx = mx.array(np.asarray(box, np.float32).reshape(1, 4) * np.tile(sc, 2))
sparse = model.prompt_encoder(pts, lab, bx)
dense = model.prompt_encoder.no_mask_embed.reshape(1, 1, 1, -1)
masks, iou, _ = model.mask_decoder(emb["img"], model.image_pe(), sparse, dense, emb["s0"], emb["s1"])
masks, iou = masks.astype(mx.float32)[0], iou.astype(mx.float32)[0]
if multimask:
masks, iou = masks[1:], iou[1:]
else:
single = masks[0]
stab = ((single > 0.05).sum() / mx.maximum((single > -0.05).sum(), 1)).item()
if (single > -0.05).sum().item() > 0 and stab < model.config["stability_thresh"]:
j = int(mx.argmax(iou[1:]).item()) + 1
else:
j = 0
masks, iou = masks[j:j + 1], iou[j:j + 1]
up = resize(masks[..., None].transpose(0, 1, 2, 3), (h, w))[..., 0]
return np.array(up > 0), np.array(iou), np.array(masks)
def convert(source: Path) -> dict:
"""runs/source/model.safetensors (transformers layout) -> this port's names; the video-memory parts dropped."""
from safetensors.numpy import load_file
sd = load_file(str(Path(source) / "model.safetensors"))
keep = ("vision_encoder.", "prompt_encoder.", "mask_decoder.", "shared_image_embedding.", "no_memory_embedding")
out = {}
for k, v in sd.items():
if not k.startswith(keep):
continue
n = k.replace("vision_encoder.backbone.", "backbone.").replace("vision_encoder.neck.", "neck.")
n = n.replace("backbone.patch_embed.projection.", "backbone.patch_embed.")
n = n.replace("shared_image_embedding.positional_embedding", "shared_image_embedding")
n = n.replace("prompt_encoder.shared_embedding.positional_embedding", "prompt_encoder.positional_embedding")
for t in ("point_embed", "not_a_point_embed", "no_mask_embed", "obj_score_token", "iou_token", "mask_tokens"):
if n.endswith(f".{t}.weight"):
n = n[:-len(".weight")]
if n.startswith("prompt_encoder.mask_embed"):
continue # mask prompts aren't exposed here
if n in ("backbone.pos_embed", "backbone.pos_embed_window"):
v = v.transpose(0, 2, 3, 1)
elif v.ndim == 4:
v = v.transpose(1, 2, 3, 0) if "upscale_conv" in n else v.transpose(0, 2, 3, 1)
out[n] = v.astype(np.float32)
return out