Instructions to use moonbox-dev/sam2.1-hiera-large-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use moonbox-dev/sam2.1-hiera-large-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download moonbox-dev/sam2.1-hiera-large-MLX --local-dir sam2.1-hiera-large-MLX
- sam2
How to use moonbox-dev/sam2.1-hiera-large-MLX with sam2:
# Use SAM2 with images import torch from sam2.sam2_image_predictor import SAM2ImagePredictor predictor = SAM2ImagePredictor.from_pretrained("moonbox-dev/sam2.1-hiera-large-MLX") with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): predictor.set_image(<your_image>) masks, _, _ = predictor.predict(<input_prompts>)# Use SAM2 with videos import torch from sam2.sam2_video_predictor import SAM2VideoPredictor predictor = SAM2VideoPredictor.from_pretrained("moonbox-dev/sam2.1-hiera-large-MLX") with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): state = predictor.init_state(<your_video>) # add new prompts and instantly get the output on the same frame frame_idx, object_ids, masks = predictor.add_new_points(state, <your_prompts>) # propagate the prompts to get masklets throughout the video for frame_idx, object_ids, masks in predictor.propagate_in_video(state): ... - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download 4bit/mlx_model.py from moonbox-dev/sam2.1-hiera-large-MLX: direct link, hf CLI and curl.
- Browser
- Download file 20 kB
-
https://huggingface.co/moonbox-dev/sam2.1-hiera-large-MLX/resolve/main/4bit/mlx_model.py
- Command line
-
hf download hf://moonbox-dev/sam2.1-hiera-large-MLX/4bit/mlx_model.py
-
curl -L -o mlx_model.py https://huggingface.co/moonbox-dev/sam2.1-hiera-large-MLX/resolve/main/4bit/mlx_model.py
20 kB
| """SAM 2.1 (facebook/sam2.1-hiera-large, Apache-2.0) in MLX: a port of transformers' `Sam2Model` for single images | |
| (Hiera-L image encoder + FPN neck, SAM 2 prompt encoder and mask decoder; no video memory). | |
| import mlx_model | |
| model = mlx_model.load("bf16") # a folder with config.json + model.safetensors | |
| emb = mlx_model.embed(model, rgb) # rgb HxWx3 uint8 -> image features (+ size) | |
| masks, scores, low_res = mlx_model.predict(model, emb, points=[[x, y]], labels=[1], box=None, multimask=True) | |
| Prompts in original-image pixels; masks at the original size (logits > 0). Single-mask output picks the most stable | |
| mask as SAM 2 does ("dynamic multimask via stability"). Runs on its own next to its weights (mlx, numpy). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| import numpy as np | |
| CONFIG = {"model_type": "sam2", "image_size": 1024, "hidden_size": 144, "blocks_per_stage": [2, 6, 36, 4], | |
| "embed_dim_per_stage": [144, 288, 576, 1152], "heads_per_stage": [2, 4, 8, 16], | |
| "window_size_per_stage": [8, 4, 16, 8], "global_attention_blocks": [23, 33, 43], "pos_embed_bg": [7, 7], | |
| "fpn_hidden_size": 256, "fpn_top_down_levels": [2, 3], "stability_delta": 0.05, "stability_thresh": 0.98} | |
| PRECISIONS = ["bf16", "fp16", "8bit", "6bit", "4bit"] | |
| MEAN, STD = np.array([0.485, 0.456, 0.406], np.float32), np.array([0.229, 0.224, 0.225], np.float32) | |
| GROUP = 64 | |
| def quantizable(path: str, module) -> bool: | |
| return isinstance(module, nn.Linear) and module.weight.shape[-1] % GROUP == 0 | |
| def resize(x, size): | |
| """torch F.interpolate(mode="bilinear", align_corners=False), NHWC.""" | |
| b, h, w, c = x.shape | |
| oh, ow = int(size[0]), int(size[1]) | |
| if (oh, ow) == (h, w): | |
| return x | |
| def axis(n_in, n_out): | |
| s = np.clip((np.arange(n_out, dtype=np.float64) + 0.5) * (n_in / n_out) - 0.5, 0, None) | |
| i0 = np.minimum(np.floor(s).astype(np.int32), n_in - 1) | |
| return mx.array(i0), mx.array(np.minimum(i0 + 1, n_in - 1)), mx.array((s - i0).astype(np.float32)) | |
| y0, y1, fy = axis(h, oh) | |
| x0, x1, fx = axis(w, ow) | |
| fy, fx = fy.astype(x.dtype)[None, :, None, None], fx.astype(x.dtype)[None, None, :, None] | |
| top, bot = x[:, y0], x[:, y1] | |
| top = top[:, :, x0] * (1 - fx) + top[:, :, x1] * fx | |
| bot = bot[:, :, x0] * (1 - fx) + bot[:, :, x1] * fx | |
| return top * (1 - fy) + bot * fy | |
| def bicubic_matrix(n_in: int, n_out: int) -> np.ndarray: | |
| """torch F.interpolate(mode="bicubic", align_corners=False) along one axis as an (n_out, n_in) matrix | |
| (A = -0.75, source index not clamped, taps clamped to the border).""" | |
| A = -0.75 | |
| m = np.zeros((n_out, n_in), np.float64) | |
| scale = n_in / n_out | |
| for o in range(n_out): | |
| s = (o + 0.5) * scale - 0.5 | |
| f = int(np.floor(s)) | |
| t = s - f | |
| w = [((A * (t + 1) - 5 * A) * (t + 1) + 8 * A) * (t + 1) - 4 * A, | |
| ((A + 2) * t - (A + 3)) * t * t + 1, | |
| ((A + 2) * (1 - t) - (A + 3)) * (1 - t) * (1 - t) + 1, | |
| ((A * (2 - t) - 5 * A) * (2 - t) + 8 * A) * (2 - t) - 4 * A] | |
| for j in range(4): | |
| m[o, min(max(f - 1 + j, 0), n_in - 1)] += w[j] | |
| return m.astype(np.float32) | |
| def maxpool2(x): | |
| b, h, w, c = x.shape | |
| return x[:, :h // 2 * 2, :w // 2 * 2].reshape(b, h // 2, 2, w // 2, 2, c).max(axis=(2, 4)) | |
| def window_partition(x, ws): | |
| b, h, w, c = x.shape | |
| ph, pw = (-h) % ws, (-w) % ws | |
| if ph or pw: | |
| x = mx.pad(x, [(0, 0), (0, ph), (0, pw), (0, 0)]) | |
| hp, wp = h + ph, w + pw | |
| x = x.reshape(b, hp // ws, ws, wp // ws, ws, c).transpose(0, 1, 3, 2, 4, 5).reshape(-1, ws, ws, c) | |
| return x, (hp, wp) | |
| def window_unpartition(x, ws, pad_hw, hw): | |
| hp, wp = pad_hw | |
| h, w = hw | |
| b = x.shape[0] // ((hp // ws) * (wp // ws)) | |
| x = x.reshape(b, hp // ws, wp // ws, ws, ws, -1).transpose(0, 1, 3, 2, 4, 5).reshape(b, hp, wp, -1) | |
| return x[:, :h, :w] | |
| class MultiScaleAttention(nn.Module): | |
| def __init__(self, dim, dim_out, heads, q_stride): | |
| super().__init__() | |
| self.heads, self.q_stride = heads, q_stride | |
| self.scale = (dim_out // heads) ** -0.5 | |
| self.qkv = nn.Linear(dim, dim_out * 3) | |
| self.proj = nn.Linear(dim_out, dim_out) | |
| def __call__(self, x): | |
| b, h, w, _ = x.shape | |
| qkv = self.qkv(x).reshape(b, h * w, 3, self.heads, -1) | |
| q, k, v = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2] | |
| if self.q_stride: | |
| q = maxpool2(q.reshape(b, h, w, -1)) | |
| h, w = q.shape[1:3] | |
| q = q.reshape(b, h * w, self.heads, -1) | |
| q, k, v = (t.transpose(0, 2, 1, 3) for t in (q, k, v)) | |
| o = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale) | |
| return self.proj(o.transpose(0, 2, 1, 3).reshape(b, h, w, -1)) | |
| class Mlp(nn.Module): | |
| """Sam2FeedForward: proj_in, (layers), proj_out.""" | |
| def __init__(self, din, dh, dout, n, act=nn.relu, sigmoid=False): | |
| super().__init__() | |
| self.proj_in, self.proj_out = nn.Linear(din, dh), nn.Linear(dh, dout) | |
| self.layers = [nn.Linear(dh, dh) for _ in range(n - 2)] | |
| self.act, self.sigmoid = act, sigmoid | |
| def __call__(self, x): | |
| x = self.act(self.proj_in(x)) | |
| for l in self.layers: | |
| x = self.act(l(x)) | |
| x = self.proj_out(x) | |
| return mx.sigmoid(x) if self.sigmoid else x | |
| class Block(nn.Module): | |
| def __init__(self, c, stage, idx, total): | |
| super().__init__() | |
| first = stage > 0 and idx == 0 | |
| self.dim = c["embed_dim_per_stage"][stage - 1] if first else c["embed_dim_per_stage"][stage] | |
| self.dim_out = c["embed_dim_per_stage"][stage] | |
| ws = c["window_size_per_stage"][stage - 1] if first else c["window_size_per_stage"][stage] | |
| self.ws = 0 if total in c["global_attention_blocks"] else ws | |
| self.q_stride = 0 < stage <= 3 and idx == 0 | |
| self.layer_norm1 = nn.LayerNorm(self.dim, eps=1e-6) | |
| self.attn = MultiScaleAttention(self.dim, self.dim_out, c["heads_per_stage"][stage], self.q_stride) | |
| self.layer_norm2 = nn.LayerNorm(self.dim_out, eps=1e-6) | |
| self.mlp = Mlp(self.dim_out, self.dim_out * 4, self.dim_out, 2, act=nn.gelu) | |
| if self.dim != self.dim_out: | |
| self.proj = nn.Linear(self.dim, self.dim_out) | |
| def __call__(self, x): | |
| res = x | |
| x = self.layer_norm1(x) | |
| if self.dim != self.dim_out: | |
| res = self.proj(x) | |
| if self.q_stride: | |
| res = maxpool2(res) | |
| ws = self.ws | |
| if ws > 0: | |
| hw = x.shape[1:3] | |
| x, pad = window_partition(x, ws) | |
| x = self.attn(x) | |
| if self.q_stride: | |
| ws = self.ws // 2 | |
| hw = res.shape[1:3] | |
| pad = (hw[0] + (-hw[0]) % ws, hw[1] + (-hw[1]) % ws) | |
| if self.ws > 0: | |
| x = window_unpartition(x, ws, pad, hw) | |
| x = res + x | |
| return x + self.mlp(self.layer_norm2(x)) | |
| class Hiera(nn.Module): | |
| def __init__(self, c): | |
| super().__init__() | |
| d = c["hidden_size"] | |
| self.patch_embed = nn.Conv2d(3, d, 7, stride=4, padding=3) | |
| self.pos_embed = mx.zeros((1, *c["pos_embed_bg"], d)) # NHWC | |
| self.pos_embed_window = mx.zeros((1, c["window_size_per_stage"][0], c["window_size_per_stage"][0], d)) | |
| self.blocks, total = [], 0 | |
| for s, n in enumerate(c["blocks_per_stage"]): | |
| for i in range(n): | |
| self.blocks.append(Block(c, s, i, total)) | |
| total += 1 | |
| self.stage_ends = list(np.cumsum(c["blocks_per_stage"]) - 1) | |
| def pos(self, h, w): | |
| pe = self.pos_embed.astype(mx.float32) | |
| mh = mx.array(bicubic_matrix(pe.shape[1], h)) | |
| mw = mx.array(bicubic_matrix(pe.shape[2], w)) | |
| pe = mx.einsum("oh,bhwc->bowc", mh, pe) | |
| pe = mx.einsum("pw,bowc->bopc", mw, pe) | |
| ws = self.pos_embed_window.shape[1] | |
| win = mx.tile(self.pos_embed_window.astype(mx.float32), (1, h // ws, w // ws, 1)) | |
| return pe + win | |
| def __call__(self, x): | |
| x = self.patch_embed(x) | |
| x = x + self.pos(x.shape[1], x.shape[2]).astype(x.dtype) | |
| outs = [] | |
| for i, blk in enumerate(self.blocks): | |
| x = blk(x) | |
| if i in self.stage_ends: | |
| outs.append(x) | |
| return outs | |
| class Neck(nn.Module): | |
| def __init__(self, c): | |
| super().__init__() | |
| self.convs = [nn.Conv2d(d, c["fpn_hidden_size"], 1) for d in c["embed_dim_per_stage"][::-1]] | |
| self.top_down = c["fpn_top_down_levels"] | |
| def __call__(self, feats): | |
| n = len(self.convs) - 1 | |
| out, prev = [], None | |
| for i in range(n, -1, -1): | |
| lat = self.convs[n - i](feats[i]) | |
| if i in self.top_down and i != n: | |
| b, h, w, ch = prev.shape | |
| up = mx.broadcast_to(prev[:, :, None, :, None, :], (b, h, 2, w, 2, ch)).reshape(b, 2 * h, 2 * w, ch) | |
| prev = lat + up | |
| else: | |
| prev = lat | |
| out.append(prev) | |
| return out[::-1][:3] # 256, 128, 64 px feature maps | |
| class LayerNorm2d(nn.Module): | |
| def __init__(self, c, eps=1e-6): | |
| super().__init__() | |
| self.weight, self.bias, self.eps = mx.ones((c,)), mx.zeros((c,)), eps | |
| def __call__(self, x): | |
| y = x.astype(mx.float32) | |
| u = y.mean(-1, keepdims=True) | |
| s = ((y - u) ** 2).mean(-1, keepdims=True) | |
| return self.weight * ((y - u) * mx.rsqrt(s + self.eps)).astype(x.dtype) + self.bias | |
| class Attention(nn.Module): | |
| def __init__(self, dim=256, heads=8, down=2): | |
| super().__init__() | |
| inner = dim // down | |
| self.heads = heads | |
| self.q_proj, self.k_proj, self.v_proj = nn.Linear(dim, inner), nn.Linear(dim, inner), nn.Linear(dim, inner) | |
| self.o_proj = nn.Linear(inner, dim) | |
| def __call__(self, q, k, v): | |
| q, k, v = self.q_proj(q), self.k_proj(k), self.v_proj(v) | |
| b, nq, c = q.shape | |
| sp = lambda t: t.reshape(b, t.shape[1], self.heads, c // self.heads).transpose(0, 2, 1, 3) | |
| o = mx.fast.scaled_dot_product_attention(sp(q), sp(k), sp(v), scale=(c // self.heads) ** -0.5) | |
| return self.o_proj(o.transpose(0, 2, 1, 3).reshape(b, nq, c)) | |
| class TwoWayBlock(nn.Module): | |
| def __init__(self, skip_pe): | |
| super().__init__() | |
| self.skip_pe = skip_pe | |
| self.self_attn = Attention(down=1) | |
| self.cross_attn_token_to_image, self.cross_attn_image_to_token = Attention(), Attention() | |
| self.mlp = Mlp(256, 2048, 256, 2) | |
| self.layer_norm1, self.layer_norm2 = nn.LayerNorm(256), nn.LayerNorm(256) | |
| self.layer_norm3, self.layer_norm4 = nn.LayerNorm(256), nn.LayerNorm(256) | |
| def __call__(self, q, k, qpe, kpe): | |
| if self.skip_pe: | |
| q = self.self_attn(q, q, q) | |
| else: | |
| t = q + qpe | |
| q = q + self.self_attn(t, t, q) | |
| q = self.layer_norm1(q) | |
| q = self.layer_norm2(q + self.cross_attn_token_to_image(q + qpe, k + kpe, k)) | |
| q = self.layer_norm3(q + self.mlp(q)) | |
| k = self.layer_norm4(k + self.cross_attn_image_to_token(k + kpe, q + qpe, q)) | |
| return q, k | |
| class TwoWayTransformer(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.layers = [TwoWayBlock(i == 0) for i in range(2)] | |
| self.final_attn_token_to_image = Attention() | |
| self.layer_norm_final_attn = nn.LayerNorm(256) | |
| def __call__(self, img, pe, tokens): | |
| b, h, w, c = img.shape | |
| k, kpe, q = img.reshape(b, h * w, c), pe.reshape(b, h * w, c), tokens | |
| for layer in self.layers: | |
| q, k = layer(q, k, tokens, kpe) | |
| q = self.layer_norm_final_attn(q + self.final_attn_token_to_image(q + tokens, k + kpe, k)) | |
| return q, k | |
| class MaskDecoder(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.obj_score_token, self.iou_token, self.mask_tokens = mx.zeros((1, 256)), mx.zeros((1, 256)), mx.zeros((4, 256)) | |
| self.transformer = TwoWayTransformer() | |
| self.upscale_conv1 = nn.ConvTranspose2d(256, 64, 2, stride=2) | |
| self.upscale_layer_norm = LayerNorm2d(64) | |
| self.upscale_conv2 = nn.ConvTranspose2d(64, 32, 2, stride=2) | |
| self.output_hypernetworks_mlps = [Mlp(256, 256, 32, 3) for _ in range(4)] | |
| self.iou_prediction_head = Mlp(256, 256, 4, 3, sigmoid=True) | |
| self.conv_s0, self.conv_s1 = nn.Conv2d(256, 32, 1), nn.Conv2d(256, 64, 1) | |
| self.pred_obj_score_head = Mlp(256, 256, 1, 3) | |
| def __call__(self, img, pe, sparse, dense, s0, s1): | |
| b = sparse.shape[0] | |
| out = mx.concatenate([self.obj_score_token, self.iou_token, self.mask_tokens], 0) | |
| tokens = mx.concatenate([mx.broadcast_to(out[None], (b,) + out.shape), sparse], 1) | |
| src = mx.broadcast_to(img + dense, (b,) + img.shape[1:]) | |
| hs, src = self.transformer(src, mx.broadcast_to(pe, (b,) + pe.shape[1:]).astype(src.dtype), tokens) | |
| _, h, w, c = img.shape | |
| up = self.upscale_conv1(src.reshape(b, h, w, c)) + s1 | |
| up = nn.gelu(self.upscale_layer_norm(up)) | |
| up = nn.gelu(self.upscale_conv2(up) + s0) | |
| hyper = mx.stack([self.output_hypernetworks_mlps[i](hs[:, 2 + i]) for i in range(4)], 1) | |
| _, uh, uw, uc = up.shape | |
| masks = (hyper @ up.reshape(b, uh * uw, uc).transpose(0, 2, 1)).reshape(b, 4, uh, uw) | |
| return masks, self.iou_prediction_head(hs[:, 1]), self.pred_obj_score_head(hs[:, 0]) | |
| class PromptEncoder(nn.Module): | |
| def __init__(self, size): | |
| super().__init__() | |
| self.size = size | |
| self.positional_embedding = mx.zeros((2, 128)) | |
| self.point_embed = mx.zeros((4, 256)) | |
| self.not_a_point_embed = mx.zeros((1, 256)) | |
| self.no_mask_embed = mx.zeros((1, 256)) | |
| def pe(self, coords, gauss): | |
| c = (2 * coords - 1) @ gauss.astype(mx.float32) | |
| c = 2 * np.pi * c | |
| return mx.concatenate([mx.sin(c), mx.cos(c)], -1) | |
| def __call__(self, points, labels, box): | |
| sparse = [] | |
| g = self.positional_embedding | |
| if points is not None: | |
| pts, lab = points + 0.5, labels | |
| if box is None: | |
| pts = mx.concatenate([pts, mx.zeros((1, 1, 2))], 1) | |
| lab = mx.concatenate([lab, -mx.ones((1, 1))], 1) | |
| e = self.pe(pts / self.size, g) | |
| l = lab[..., None] | |
| e = mx.where(l == -1, self.not_a_point_embed.astype(mx.float32), e) | |
| idx = mx.maximum(lab, 0).astype(mx.int32) | |
| e = e + self.point_embed.astype(mx.float32)[idx] * (l >= 0) | |
| sparse.append(e) | |
| if box is not None: | |
| e = self.pe((box + 0.5).reshape(1, 2, 2) / self.size, g) | |
| e = e + self.point_embed[2:4].astype(mx.float32)[None] | |
| sparse.append(mx.concatenate([e, self.not_a_point_embed.astype(mx.float32)[None]], 1)) | |
| return mx.concatenate(sparse, 1).astype(self.no_mask_embed.dtype) | |
| class Model(nn.Module): | |
| def __init__(self, config: dict = CONFIG): | |
| super().__init__() | |
| self.config, self.size = config, config["image_size"] | |
| self.backbone = Hiera(config) | |
| self.neck = Neck(config) | |
| self.no_memory_embedding = mx.zeros((1, 1, 256)) | |
| self.shared_image_embedding = mx.zeros((2, 128)) | |
| self.prompt_encoder = PromptEncoder(self.size) | |
| self.mask_decoder = MaskDecoder() | |
| def image_pe(self): | |
| n = self.size // 16 | |
| g = (mx.arange(n, dtype=mx.float32) + 0.5) / n | |
| grid = mx.stack([mx.broadcast_to(g[None, :], (n, n)), mx.broadcast_to(g[:, None], (n, n))], -1) | |
| return self.prompt_encoder.pe(grid, self.shared_image_embedding)[None] | |
| def load(folder) -> Model: | |
| folder = Path(folder) | |
| config = json.loads((folder / "config.json").read_text()) | |
| model = Model(config) | |
| q = config.get("quantization") | |
| if q: | |
| nn.quantize(model, group_size=q["group_size"], bits=q["bits"], class_predicate=quantizable) | |
| model.load_weights(str(folder / "model.safetensors"), strict=True) | |
| mx.eval(model.parameters()) | |
| return model | |
| def embed(model: Model, rgb: np.ndarray) -> dict: | |
| """Sam2ImageProcessor: the image resized to 1024x1024 (bilinear, no aspect kept), ImageNet-normalized.""" | |
| s = model.size | |
| x = resize(mx.array(rgb.astype(np.float32))[None], (s, s)) | |
| x = (mx.clip(mx.round(x), 0, 255) / 255.0 - mx.array(MEAN)) / mx.array(STD) | |
| return embed_pixels(model, x, rgb.shape[:2]) | |
| def embed_pixels(model: Model, x, original_size) -> dict: | |
| """The encoder on already preprocessed pixels (1, 1024, 1024, 3), e.g. Sam2Processor's pixel_values in NHWC | |
| (its torchvision uint8 resize differs from `embed`'s by up to one level per pixel).""" | |
| h, w = original_size | |
| dt = model.no_memory_embedding.dtype | |
| f0, f1, f2 = model.neck(model.backbone(mx.array(x).astype(dt))) | |
| dec = model.mask_decoder | |
| feats = {"s0": dec.conv_s0(f0), "s1": dec.conv_s1(f1), "img": f2 + model.no_memory_embedding.reshape(1, 1, 1, -1), | |
| "original_size": (h, w)} | |
| mx.eval(feats["s0"], feats["s1"], feats["img"]) | |
| return feats | |
| def predict(model: Model, emb: dict, points=None, labels=None, box=None, multimask=True): | |
| h, w = emb["original_size"] | |
| s = model.size | |
| sc = np.array([s / w, s / h], np.float32) | |
| pts = lab = bx = None | |
| if points is not None: | |
| pts = mx.array(np.asarray(points, np.float32)[None] * sc) | |
| lab = mx.array(np.asarray(labels, np.float32)[None]) | |
| if box is not None: | |
| bx = mx.array(np.asarray(box, np.float32).reshape(1, 4) * np.tile(sc, 2)) | |
| sparse = model.prompt_encoder(pts, lab, bx) | |
| dense = model.prompt_encoder.no_mask_embed.reshape(1, 1, 1, -1) | |
| masks, iou, _ = model.mask_decoder(emb["img"], model.image_pe(), sparse, dense, emb["s0"], emb["s1"]) | |
| masks, iou = masks.astype(mx.float32)[0], iou.astype(mx.float32)[0] | |
| if multimask: | |
| masks, iou = masks[1:], iou[1:] | |
| else: | |
| single = masks[0] | |
| stab = ((single > 0.05).sum() / mx.maximum((single > -0.05).sum(), 1)).item() | |
| if (single > -0.05).sum().item() > 0 and stab < model.config["stability_thresh"]: | |
| j = int(mx.argmax(iou[1:]).item()) + 1 | |
| else: | |
| j = 0 | |
| masks, iou = masks[j:j + 1], iou[j:j + 1] | |
| up = resize(masks[..., None].transpose(0, 1, 2, 3), (h, w))[..., 0] | |
| return np.array(up > 0), np.array(iou), np.array(masks) | |
| def convert(source: Path) -> dict: | |
| """runs/source/model.safetensors (transformers layout) -> this port's names; the video-memory parts dropped.""" | |
| from safetensors.numpy import load_file | |
| sd = load_file(str(Path(source) / "model.safetensors")) | |
| keep = ("vision_encoder.", "prompt_encoder.", "mask_decoder.", "shared_image_embedding.", "no_memory_embedding") | |
| out = {} | |
| for k, v in sd.items(): | |
| if not k.startswith(keep): | |
| continue | |
| n = k.replace("vision_encoder.backbone.", "backbone.").replace("vision_encoder.neck.", "neck.") | |
| n = n.replace("backbone.patch_embed.projection.", "backbone.patch_embed.") | |
| n = n.replace("shared_image_embedding.positional_embedding", "shared_image_embedding") | |
| n = n.replace("prompt_encoder.shared_embedding.positional_embedding", "prompt_encoder.positional_embedding") | |
| for t in ("point_embed", "not_a_point_embed", "no_mask_embed", "obj_score_token", "iou_token", "mask_tokens"): | |
| if n.endswith(f".{t}.weight"): | |
| n = n[:-len(".weight")] | |
| if n.startswith("prompt_encoder.mask_embed"): | |
| continue # mask prompts aren't exposed here | |
| if n in ("backbone.pos_embed", "backbone.pos_embed_window"): | |
| v = v.transpose(0, 2, 3, 1) | |
| elif v.ndim == 4: | |
| v = v.transpose(1, 2, 3, 0) if "upscale_conv" in n else v.transpose(0, 2, 3, 1) | |
| out[n] = v.astype(np.float32) | |
| return out | |