Download code/d1_torch.py from ivanfioravanti/d1-2B-gestures-CoreAI: direct link, hf CLI and curl.
- Browser
- Download file 11.1 kB
-
https://huggingface.co/ivanfioravanti/d1-2B-gestures-CoreAI/resolve/main/code/d1_torch.py
- Command line
-
hf download hf://ivanfioravanti/d1-2B-gestures-CoreAI/code/d1_torch.py
-
curl -L -o d1_torch.py https://huggingface.co/ivanfioravanti/d1-2B-gestures-CoreAI/resolve/main/code/d1_torch.py
11.1 kB
| """d1-2B's decision pass for our four questions as one fixed-shape PyTorch function (GPU layout): | |
| image [1, 3, 352, 640] in [-1, 1] -> logits [12, U] | |
| Re-authored from mlx-vlm's lfm3 / lfm3_vl / d1_lfm3_vl (itself a port of the LiquidAI PyTorch code); weights from | |
| LiquidAI/d1-2B safetensors. Prompt ids, set/sliding masks and RoPE tables are constants (prompt.json). | |
| """ | |
| import glob, json, math, os | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from safetensors import safe_open | |
| CKPT = os.path.expanduser("~/.cache/huggingface/hub/models--LiquidAI--d1-2B/snapshots/*/") # hf download LiquidAI/d1-2B | |
| def rms(x, w, eps=1e-5): | |
| return F.rms_norm(x, (x.shape[-1],), w, eps) | |
| def rotate(x, cos, sin): | |
| a, b = x.chunk(2, dim=-1) | |
| return x * cos + torch.cat([-b, a], dim=-1) * sin | |
| def mlp(x, p): | |
| return F.linear(F.silu(F.linear(x, p["w1"])) * F.linear(x, p["w3"]), p["w2"]) | |
| class _W(nn.Module): | |
| """Holds one projection weight as `.weight`: coreai-opt's w8 spec quantizes parameters named `weight` only.""" | |
| def __init__(self, t): | |
| super().__init__() | |
| self.weight = nn.Parameter(t, requires_grad=False) | |
| def unwrap(module, n): | |
| v = getattr(module, n) | |
| return next(v.parameters()) if isinstance(v, _W) else v # its one parameter, whatever an export renamed it to | |
| class D1Fixed(nn.Module): | |
| def __init__(self, meta, dtype=torch.float16): | |
| super().__init__() | |
| path = glob.glob(CKPT)[0] | |
| cfg = json.load(open(path + "config.json")) | |
| self.tc, self.vc, self.meta, self.dtype = cfg["text_config"], cfg["vision_config"], meta, dtype | |
| f = safe_open(path + "model.safetensors", "pt") | |
| g = lambda k: f.get_tensor(k) | |
| self._n = 0 | |
| def P(t, q=None): # a weight constant; q: a projection weight (quantizable), by default any 2-D/4-D non-depthwise one | |
| name = "w%d" % self._n | |
| self._n += 1 | |
| t = t.to(dtype).contiguous() | |
| if q if q is not None else (t.dim() == 2 or t.dim() == 4 and t.shape[1] > 1): | |
| self.add_module(name, _W(t)) | |
| else: | |
| self._const(name, t) | |
| return name | |
| _, gh, gw = meta["grid_thw"] | |
| self.gh, self.gw = gh, gw | |
| vt = "model.vision_tower." | |
| # patchifier Linear(3*2*16*16 -> 1024), features (C, T, ph, pw): both temporal copies equal -> sum them | |
| self.patch = P(g(vt + "patchifier.weight").float().reshape(1024, 3, 2, 16, 16).sum(2)) | |
| self.v_norm = P(g(vt + "embedding_norm.weight")) | |
| self.vlayers = [] | |
| for i, kind in enumerate(self.vc["layer_types"]): | |
| pre = vt + "layers.%d." % i | |
| L = {"kind": kind, "operator_norm": P(g(pre + "operator_norm.weight")), "ffn_norm": P(g(pre + "ffn_norm.weight")), | |
| "mlp": {k: P(g(pre + "feed_forward.%s.weight" % k)) for k in ("w1", "w2", "w3")}} | |
| if kind == "conv": | |
| L.update(in_proj=P(g(pre + "conv.in_proj.weight")), out_proj=P(g(pre + "conv.out_proj.weight")), | |
| spatial=P(g(pre + "conv.spatial_conv.weight"))) | |
| else: | |
| for k in ("q_proj", "k_proj", "v_proj", "out_proj", "q_layernorm", "k_layernorm"): | |
| L[k] = P(g(pre + "self_attn.%s.weight" % k)) | |
| self.vlayers.append(L) | |
| # vision 2-D RoPE in raster order (row, col) | |
| hd = self.vc["hidden_size"] // self.vc["num_attention_heads"] | |
| inv = self.vc["rope_parameters"]["rope_theta"] ** (-torch.arange(0, hd // 2, 2, dtype=torch.float64) / (hd // 2)) | |
| rows, cols = torch.meshgrid(torch.arange(gh), torch.arange(gw), indexing="ij") | |
| coords = torch.stack([rows, cols], -1).reshape(-1, 2).double() | |
| ang = (coords[..., None] * inv).reshape(gh * gw, -1) | |
| ang = torch.cat([ang, ang], -1) | |
| self._const("v_cos", ang.cos().float()) | |
| self._const("v_sin", ang.sin().float()) | |
| mp = "model.multi_modal_projector." | |
| self.proj = {k: P(g(mp + k)) for k in ("layer_norm.weight", "layer_norm.bias", "linear_1.weight", "linear_1.bias", | |
| "linear_2.weight", "linear_2.bias")} | |
| # text: constant embeddings around the image, slot embeddings for the head | |
| lm = "model.language_model." | |
| emb = g(lm + "embed_tokens.weight") | |
| ids = torch.tensor(meta["ids"]) | |
| s, n = meta["image_start"], meta["image_len"] | |
| self.pre = P(emb[ids[:s]], q=False) | |
| self.post = P(emb[ids[s + n:]], q=False) | |
| self.head = P(emb[torch.tensor(meta["union"])]) | |
| self.final_norm = P(g(lm + "embedding_norm.weight")) | |
| self.layers = [] | |
| for i, kind in enumerate(self.tc["layer_types"]): | |
| pre = lm + "layers.%d." % i | |
| L = {"kind": kind, "mlp": {k: P(g(pre + "feed_forward.%s.weight" % k)) for k in ("w1", "w2", "w3")}} | |
| for k in ("operator_norm", "ffn_norm", "post_operator_norm", "post_ffn_norm"): | |
| L[k] = P(g(pre + k + ".weight")) | |
| if kind == "conv": | |
| L.update(in_proj=P(g(pre + "conv.in_proj.weight")), out_proj=P(g(pre + "conv.out_proj.weight")), | |
| conv=P(g(pre + "conv.conv.weight"))) | |
| else: | |
| for k in ("q_proj", "k_proj", "v_proj", "o_proj", "q_norm", "k_norm", "gate_proj"): | |
| L[k] = P(g(pre + "self_attn.%s.weight" % k)) | |
| self.layers.append(L) | |
| # text RoPE and masks | |
| T = len(meta["ids"]) | |
| self.T = T | |
| pos = torch.arange(T, dtype=torch.float64) | |
| for kind in ("full_attention", "sliding_attention"): | |
| rp = self.tc["rope_parameters"][kind] | |
| d = self.tc["head_dim"] | |
| angles = int(d * rp.get("partial_rotary_factor", 1.0)) // 2 | |
| inv = rp["rope_theta"] ** (-torch.arange(angles, dtype=torch.float64) * 2 / d) | |
| inv = F.pad(inv, (0, d // 2 - angles)) | |
| a = pos[:, None] * inv | |
| a = torch.cat([a, a], -1) | |
| self._const(kind + "_cos", a.cos().to(dtype)) | |
| self._const(kind + "_sin", a.sin().to(dtype)) | |
| q = torch.tensor(meta["question"]) | |
| p = torch.arange(T) | |
| sl = meta["state_len"] | |
| full = (p[:, None] < sl) | (p[None] < sl) | (q[:, None] == q[None]) | |
| slide = (p[:, None] >= p[None]) & (p[:, None] < p[None] + meta["sliding_window"]) | |
| neg = torch.finfo(dtype).min / 2 | |
| self._const("full_attention_mask", torch.zeros(T, T).masked_fill(~full, neg).to(dtype)) | |
| self._const("sliding_attention_mask", torch.zeros(T, T).masked_fill(~slide, neg).to(dtype)) | |
| self._const("slots", torch.tensor([s["pos"] for s in meta["slots"]])) | |
| def _const(self, name, t): # frozen parameter: torch.export lifts it as a weight, not as a graph input | |
| self.register_parameter(name, nn.Parameter(t, requires_grad=False)) | |
| def w(self, name): | |
| return unwrap(self, name) | |
| def W(self, d): | |
| return {k: self.w(v) for k, v in d.items()} | |
| # ---- vision ---- | |
| def vision(self, image): | |
| vc, gh, gw = self.vc, self.gh, self.gw | |
| D = vc["hidden_size"] | |
| x = F.conv2d(image.to(self.dtype), self.w(self.patch), stride=16) # [1, D, gh, gw] | |
| x = x.flatten(2).transpose(1, 2) # [1, N, D] raster | |
| x = rms(x, self.w(self.v_norm)) | |
| N = gh * gw | |
| H, Hk = vc["num_attention_heads"], vc["num_key_value_heads"] | |
| hd = D // H | |
| for L in self.vlayers: | |
| h = rms(x, self.w(L["operator_norm"])) | |
| if L["kind"] == "conv": | |
| b, c, v = F.linear(h, self.w(L["in_proj"])).chunk(3, -1) | |
| grid = (b * v).transpose(1, 2).reshape(1, D, gh, gw) | |
| grid = F.conv2d(grid, self.w(L["spatial"]), padding=1, groups=D) | |
| h = F.linear(c * grid.reshape(1, D, N).transpose(1, 2), self.w(L["out_proj"])) | |
| else: | |
| q = rms(F.linear(h, self.w(L["q_proj"])).reshape(1, N, H, hd), self.w(L["q_layernorm"])).transpose(1, 2) | |
| k = rms(F.linear(h, self.w(L["k_proj"])).reshape(1, N, Hk, hd), self.w(L["k_layernorm"])).transpose(1, 2) | |
| v = F.linear(h, self.w(L["v_proj"])).reshape(1, N, Hk, hd).transpose(1, 2) | |
| q = rotate(q.float(), self.v_cos, self.v_sin).to(self.dtype) | |
| k = rotate(k.float(), self.v_cos, self.v_sin).to(self.dtype) | |
| k, v = k.repeat_interleave(H // Hk, 1), v.repeat_interleave(H // Hk, 1) | |
| o = F.scaled_dot_product_attention(q, k, v, scale=hd ** -0.5) | |
| h = F.linear(o.transpose(1, 2).reshape(1, N, D), self.w(L["out_proj"])) | |
| x = x + h | |
| x = x + mlp(rms(x, self.w(L["ffn_norm"])), self.W(L["mlp"])) | |
| # projector: LayerNorm per token, then 2x2 merge blocks in raster order of blocks | |
| pj = self.W(self.proj) | |
| x = F.layer_norm(x, (D,), pj["layer_norm.weight"], pj["layer_norm.bias"], 1e-6) | |
| x = x.reshape(gh // 2, 2, gw // 2, 2, D).permute(0, 2, 1, 3, 4).reshape(gh * gw // 4, 4 * D) | |
| x = F.gelu(F.linear(x, pj["linear_1.weight"], pj["linear_1.bias"])) | |
| return F.linear(x, pj["linear_2.weight"], pj["linear_2.bias"]) # [220, 2048] | |
| # ---- language ---- | |
| def forward(self, image): | |
| tc, T = self.tc, self.T | |
| D, H, Hk, hd = tc["hidden_size"], tc["num_attention_heads"], tc["num_key_value_heads"], tc["head_dim"] | |
| x = torch.cat([self.w(self.pre), self.vision(image), self.w(self.post)], 0)[None] # [1, T, D] | |
| for L in self.layers: | |
| h = rms(x, self.w(L["operator_norm"])) | |
| if L["kind"] == "conv": | |
| b, c, v = F.linear(h, self.w(L["in_proj"])).chunk(3, -1) | |
| bx = F.pad((b * v).transpose(1, 2), (6, 0)) | |
| h = F.linear(c * F.conv1d(bx, self.w(L["conv"]), groups=D).transpose(1, 2), self.w(L["out_proj"])) | |
| else: | |
| kind = L["kind"] | |
| cos, sin = self.w(kind + "_cos"), self.w(kind + "_sin") | |
| q = rms(F.linear(h, self.w(L["q_proj"])).reshape(1, T, H, hd), self.w(L["q_norm"])).transpose(1, 2) | |
| k = rms(F.linear(h, self.w(L["k_proj"])).reshape(1, T, Hk, hd), self.w(L["k_norm"])).transpose(1, 2) | |
| v = F.linear(h, self.w(L["v_proj"])).reshape(1, T, Hk, hd).transpose(1, 2) | |
| q, k = rotate(q, cos, sin), rotate(k, cos, sin) | |
| k, v = k.repeat_interleave(H // Hk, 1), v.repeat_interleave(H // Hk, 1) | |
| o = F.scaled_dot_product_attention(q, k, v, attn_mask=self.w(kind + "_mask"), scale=hd ** -0.5) | |
| o = o.transpose(1, 2) * torch.sigmoid(F.linear(h, self.w(L["gate_proj"])))[..., None] | |
| h = F.linear(o.reshape(1, T, D), self.w(L["o_proj"])) | |
| x = x + rms(h, self.w(L["post_operator_norm"])) | |
| x = x + rms(mlp(rms(x, self.w(L["ffn_norm"])), self.W(L["mlp"])), self.w(L["post_ffn_norm"])) | |
| x = rms(x, self.w(self.final_norm))[0] | |
| return (x[self.slots] @ self.w(self.head).T).float() # [12, U] | |