"""d1-2B's decision pass for our four questions as one fixed-shape PyTorch function (GPU layout): image [1, 3, 352, 640] in [-1, 1] -> logits [12, U] Re-authored from mlx-vlm's lfm3 / lfm3_vl / d1_lfm3_vl (itself a port of the LiquidAI PyTorch code); weights from LiquidAI/d1-2B safetensors. Prompt ids, set/sliding masks and RoPE tables are constants (prompt.json). """ import glob, json, math, os import torch import torch.nn as nn import torch.nn.functional as F from safetensors import safe_open CKPT = os.path.expanduser("~/.cache/huggingface/hub/models--LiquidAI--d1-2B/snapshots/*/") # hf download LiquidAI/d1-2B def rms(x, w, eps=1e-5): return F.rms_norm(x, (x.shape[-1],), w, eps) def rotate(x, cos, sin): a, b = x.chunk(2, dim=-1) return x * cos + torch.cat([-b, a], dim=-1) * sin def mlp(x, p): return F.linear(F.silu(F.linear(x, p["w1"])) * F.linear(x, p["w3"]), p["w2"]) class _W(nn.Module): """Holds one projection weight as `.weight`: coreai-opt's w8 spec quantizes parameters named `weight` only.""" def __init__(self, t): super().__init__() self.weight = nn.Parameter(t, requires_grad=False) def unwrap(module, n): v = getattr(module, n) return next(v.parameters()) if isinstance(v, _W) else v # its one parameter, whatever an export renamed it to class D1Fixed(nn.Module): def __init__(self, meta, dtype=torch.float16): super().__init__() path = glob.glob(CKPT)[0] cfg = json.load(open(path + "config.json")) self.tc, self.vc, self.meta, self.dtype = cfg["text_config"], cfg["vision_config"], meta, dtype f = safe_open(path + "model.safetensors", "pt") g = lambda k: f.get_tensor(k) self._n = 0 def P(t, q=None): # a weight constant; q: a projection weight (quantizable), by default any 2-D/4-D non-depthwise one name = "w%d" % self._n self._n += 1 t = t.to(dtype).contiguous() if q if q is not None else (t.dim() == 2 or t.dim() == 4 and t.shape[1] > 1): self.add_module(name, _W(t)) else: self._const(name, t) return name _, gh, gw = meta["grid_thw"] self.gh, self.gw = gh, gw vt = "model.vision_tower." # patchifier Linear(3*2*16*16 -> 1024), features (C, T, ph, pw): both temporal copies equal -> sum them self.patch = P(g(vt + "patchifier.weight").float().reshape(1024, 3, 2, 16, 16).sum(2)) self.v_norm = P(g(vt + "embedding_norm.weight")) self.vlayers = [] for i, kind in enumerate(self.vc["layer_types"]): pre = vt + "layers.%d." % i L = {"kind": kind, "operator_norm": P(g(pre + "operator_norm.weight")), "ffn_norm": P(g(pre + "ffn_norm.weight")), "mlp": {k: P(g(pre + "feed_forward.%s.weight" % k)) for k in ("w1", "w2", "w3")}} if kind == "conv": L.update(in_proj=P(g(pre + "conv.in_proj.weight")), out_proj=P(g(pre + "conv.out_proj.weight")), spatial=P(g(pre + "conv.spatial_conv.weight"))) else: for k in ("q_proj", "k_proj", "v_proj", "out_proj", "q_layernorm", "k_layernorm"): L[k] = P(g(pre + "self_attn.%s.weight" % k)) self.vlayers.append(L) # vision 2-D RoPE in raster order (row, col) hd = self.vc["hidden_size"] // self.vc["num_attention_heads"] inv = self.vc["rope_parameters"]["rope_theta"] ** (-torch.arange(0, hd // 2, 2, dtype=torch.float64) / (hd // 2)) rows, cols = torch.meshgrid(torch.arange(gh), torch.arange(gw), indexing="ij") coords = torch.stack([rows, cols], -1).reshape(-1, 2).double() ang = (coords[..., None] * inv).reshape(gh * gw, -1) ang = torch.cat([ang, ang], -1) self._const("v_cos", ang.cos().float()) self._const("v_sin", ang.sin().float()) mp = "model.multi_modal_projector." self.proj = {k: P(g(mp + k)) for k in ("layer_norm.weight", "layer_norm.bias", "linear_1.weight", "linear_1.bias", "linear_2.weight", "linear_2.bias")} # text: constant embeddings around the image, slot embeddings for the head lm = "model.language_model." emb = g(lm + "embed_tokens.weight") ids = torch.tensor(meta["ids"]) s, n = meta["image_start"], meta["image_len"] self.pre = P(emb[ids[:s]], q=False) self.post = P(emb[ids[s + n:]], q=False) self.head = P(emb[torch.tensor(meta["union"])]) self.final_norm = P(g(lm + "embedding_norm.weight")) self.layers = [] for i, kind in enumerate(self.tc["layer_types"]): pre = lm + "layers.%d." % i L = {"kind": kind, "mlp": {k: P(g(pre + "feed_forward.%s.weight" % k)) for k in ("w1", "w2", "w3")}} for k in ("operator_norm", "ffn_norm", "post_operator_norm", "post_ffn_norm"): L[k] = P(g(pre + k + ".weight")) if kind == "conv": L.update(in_proj=P(g(pre + "conv.in_proj.weight")), out_proj=P(g(pre + "conv.out_proj.weight")), conv=P(g(pre + "conv.conv.weight"))) else: for k in ("q_proj", "k_proj", "v_proj", "o_proj", "q_norm", "k_norm", "gate_proj"): L[k] = P(g(pre + "self_attn.%s.weight" % k)) self.layers.append(L) # text RoPE and masks T = len(meta["ids"]) self.T = T pos = torch.arange(T, dtype=torch.float64) for kind in ("full_attention", "sliding_attention"): rp = self.tc["rope_parameters"][kind] d = self.tc["head_dim"] angles = int(d * rp.get("partial_rotary_factor", 1.0)) // 2 inv = rp["rope_theta"] ** (-torch.arange(angles, dtype=torch.float64) * 2 / d) inv = F.pad(inv, (0, d // 2 - angles)) a = pos[:, None] * inv a = torch.cat([a, a], -1) self._const(kind + "_cos", a.cos().to(dtype)) self._const(kind + "_sin", a.sin().to(dtype)) q = torch.tensor(meta["question"]) p = torch.arange(T) sl = meta["state_len"] full = (p[:, None] < sl) | (p[None] < sl) | (q[:, None] == q[None]) slide = (p[:, None] >= p[None]) & (p[:, None] < p[None] + meta["sliding_window"]) neg = torch.finfo(dtype).min / 2 self._const("full_attention_mask", torch.zeros(T, T).masked_fill(~full, neg).to(dtype)) self._const("sliding_attention_mask", torch.zeros(T, T).masked_fill(~slide, neg).to(dtype)) self._const("slots", torch.tensor([s["pos"] for s in meta["slots"]])) def _const(self, name, t): # frozen parameter: torch.export lifts it as a weight, not as a graph input self.register_parameter(name, nn.Parameter(t, requires_grad=False)) def w(self, name): return unwrap(self, name) def W(self, d): return {k: self.w(v) for k, v in d.items()} # ---- vision ---- def vision(self, image): vc, gh, gw = self.vc, self.gh, self.gw D = vc["hidden_size"] x = F.conv2d(image.to(self.dtype), self.w(self.patch), stride=16) # [1, D, gh, gw] x = x.flatten(2).transpose(1, 2) # [1, N, D] raster x = rms(x, self.w(self.v_norm)) N = gh * gw H, Hk = vc["num_attention_heads"], vc["num_key_value_heads"] hd = D // H for L in self.vlayers: h = rms(x, self.w(L["operator_norm"])) if L["kind"] == "conv": b, c, v = F.linear(h, self.w(L["in_proj"])).chunk(3, -1) grid = (b * v).transpose(1, 2).reshape(1, D, gh, gw) grid = F.conv2d(grid, self.w(L["spatial"]), padding=1, groups=D) h = F.linear(c * grid.reshape(1, D, N).transpose(1, 2), self.w(L["out_proj"])) else: q = rms(F.linear(h, self.w(L["q_proj"])).reshape(1, N, H, hd), self.w(L["q_layernorm"])).transpose(1, 2) k = rms(F.linear(h, self.w(L["k_proj"])).reshape(1, N, Hk, hd), self.w(L["k_layernorm"])).transpose(1, 2) v = F.linear(h, self.w(L["v_proj"])).reshape(1, N, Hk, hd).transpose(1, 2) q = rotate(q.float(), self.v_cos, self.v_sin).to(self.dtype) k = rotate(k.float(), self.v_cos, self.v_sin).to(self.dtype) k, v = k.repeat_interleave(H // Hk, 1), v.repeat_interleave(H // Hk, 1) o = F.scaled_dot_product_attention(q, k, v, scale=hd ** -0.5) h = F.linear(o.transpose(1, 2).reshape(1, N, D), self.w(L["out_proj"])) x = x + h x = x + mlp(rms(x, self.w(L["ffn_norm"])), self.W(L["mlp"])) # projector: LayerNorm per token, then 2x2 merge blocks in raster order of blocks pj = self.W(self.proj) x = F.layer_norm(x, (D,), pj["layer_norm.weight"], pj["layer_norm.bias"], 1e-6) x = x.reshape(gh // 2, 2, gw // 2, 2, D).permute(0, 2, 1, 3, 4).reshape(gh * gw // 4, 4 * D) x = F.gelu(F.linear(x, pj["linear_1.weight"], pj["linear_1.bias"])) return F.linear(x, pj["linear_2.weight"], pj["linear_2.bias"]) # [220, 2048] # ---- language ---- def forward(self, image): tc, T = self.tc, self.T D, H, Hk, hd = tc["hidden_size"], tc["num_attention_heads"], tc["num_key_value_heads"], tc["head_dim"] x = torch.cat([self.w(self.pre), self.vision(image), self.w(self.post)], 0)[None] # [1, T, D] for L in self.layers: h = rms(x, self.w(L["operator_norm"])) if L["kind"] == "conv": b, c, v = F.linear(h, self.w(L["in_proj"])).chunk(3, -1) bx = F.pad((b * v).transpose(1, 2), (6, 0)) h = F.linear(c * F.conv1d(bx, self.w(L["conv"]), groups=D).transpose(1, 2), self.w(L["out_proj"])) else: kind = L["kind"] cos, sin = self.w(kind + "_cos"), self.w(kind + "_sin") q = rms(F.linear(h, self.w(L["q_proj"])).reshape(1, T, H, hd), self.w(L["q_norm"])).transpose(1, 2) k = rms(F.linear(h, self.w(L["k_proj"])).reshape(1, T, Hk, hd), self.w(L["k_norm"])).transpose(1, 2) v = F.linear(h, self.w(L["v_proj"])).reshape(1, T, Hk, hd).transpose(1, 2) q, k = rotate(q, cos, sin), rotate(k, cos, sin) k, v = k.repeat_interleave(H // Hk, 1), v.repeat_interleave(H // Hk, 1) o = F.scaled_dot_product_attention(q, k, v, attn_mask=self.w(kind + "_mask"), scale=hd ** -0.5) o = o.transpose(1, 2) * torch.sigmoid(F.linear(h, self.w(L["gate_proj"])))[..., None] h = F.linear(o.reshape(1, T, D), self.w(L["o_proj"])) x = x + rms(h, self.w(L["post_operator_norm"])) x = x + rms(mlp(rms(x, self.w(L["ffn_norm"])), self.W(L["mlp"])), self.w(L["post_ffn_norm"])) x = rms(x, self.w(self.final_norm))[0] return (x[self.slots] @ self.w(self.head).T).float() # [12, U]