ivanfioravanti's picture
Card: new Neural Engine speed, energy and parity; code: MLP split, power.py
8a7b200 verified
Raw History Blame Contribute Delete
17 kB
"""The same fixed d1 function re-authored for the Neural Engine (coreai-models `model-authoring` rules):
BC1S layout [1, C, 1, S] (vision NCHW [1, C, 22, 40]), 1x1 Conv2d projections, per-head attention with the
`bchq,bkhc->bkhq` einsum, softmax over the key (channel) axis, additive masks (1, key, 1, query) at -40000,
fp16 everywhere (no fp32 literals: every scalar is an fp16 constant), the projector's 2x2 merge as a stride-2 conv.
image [1, 3, 352, 640] fp32 in [-1, 1] -> logits [1, U, 1, 12] fp16
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from d1_torch import D1Fixed, _W, unwrap
NEG = -40000.0
class D1ANE(nn.Module):
def __init__(self, src: D1Fixed):
super().__init__()
assert src.dtype == torch.float16
self.tc, self.vc, self.gh, self.gw, self.T = src.tc, src.vc, src.gh, src.gw, src.T
h = torch.float16
w = src.w
conv = lambda t: t.reshape(*t.shape, 1, 1) if t.dim() == 2 else t # Linear [O, I] -> Conv2d [O, I, 1, 1]
col = lambda t: t.reshape(1, -1, 1, 1) # norm / bias weight over channels
def C(name, t): # a constant; projection / conv weights (O>1, I>1) in a `.weight` holder, quantizable
name = "p_" + name
t = t.to(h).contiguous()
if t.dim() == 4 and t.shape[0] > 1 and t.shape[1] > 1:
self.add_module(name, _W(t))
else:
self.register_parameter(name, nn.Parameter(t, requires_grad=False))
return name
self.c = {}
for k, v in {"one": 1.0, "half": 0.5, "eps": 1e-5, "eps_ln": 1e-6, "v_scale": (self.vc["hidden_size"] // self.vc["num_attention_heads"]) ** -0.5,
"t_scale": self.tc["head_dim"] ** -0.5, "pre": 8.0, "eps_pre": 1e-5 * 64.0, "eps_v": 1e-5 * self.VS ** 2, "inv_vs": 1 / self.VS, "inv_ts": 1 / self.TS, "inv_ps": 1 / self.PATCH, "eps_pconv": 1e-5 * self.PS["conv"] ** 2, "eps_pattn": 1e-5 * self.PS["attn"] ** 2, "eps_pmlp": 1e-5 * self.PS["mlp"] ** 2}.items():
self.c[k] = C("c_" + k, torch.tensor([v]))
self.patch = C("patch", w(src.patch))
self.v_norm = C("v_norm", col(w(src.v_norm)))
self.vl = []
for i, L in enumerate(src.vlayers):
d = {"kind": L["kind"], "operator_norm": C("v%d_on" % i, col(w(L["operator_norm"]))),
"ffn_norm": C("v%d_fn" % i, col(w(L["ffn_norm"]))),
"mlp": {k: C("v%d_%s" % (i, k), conv(w(n))) for k, n in L["mlp"].items()}}
if L["kind"] == "conv":
d.update(in_proj=C("v%d_in" % i, conv(w(L["in_proj"]))), out_proj=C("v%d_out" % i, conv(w(L["out_proj"]))),
spatial=C("v%d_sp" % i, w(L["spatial"])))
else:
for k in ("q_proj", "k_proj", "v_proj", "out_proj"):
d[k] = C("v%d_%s" % (i, k), conv(w(L[k])))
for k in ("q_layernorm", "k_layernorm"):
d[k] = C("v%d_%s" % (i, k), w(L[k]).reshape(1, 1, -1, 1))
self.vl.append(d)
N, hd = self.gh * self.gw, self.vc["hidden_size"] // self.vc["num_attention_heads"]
self.v_cos = C("v_cos", src.v_cos.T.reshape(1, 1, hd, N))
self.v_sin = C("v_sin", src.v_sin.T.reshape(1, 1, hd, N))
pj = {k: w(v) for k, v in src.proj.items()}
D = self.vc["hidden_size"]
self.ln_w, self.ln_b = C("ln_w", col(pj["layer_norm.weight"])), C("ln_b", col(pj["layer_norm.bias"]))
# merged feature index (mr*2+mc)*D + c -> conv weight [O, c, mr, mc], kernel 2 stride 2
W1 = pj["linear_1.weight"]
self.l1 = C("l1", W1.reshape(W1.shape[0], 2, 2, D).permute(0, 3, 1, 2))
self.l1b = C("l1b", pj["linear_1.bias"])
self.l2 = C("l2", conv(pj["linear_2.weight"]))
self.l2b = C("l2b", pj["linear_2.bias"])
bc1s = lambda t: t.T.reshape(1, t.shape[1], 1, t.shape[0]) # [S, D] -> [1, D, 1, S]
self.pre_e, self.post_e = C("pre_e", bc1s(w(src.pre))), C("post_e", bc1s(w(src.post)))
self.head = C("head", conv(w(src.head)))
self.final_norm = C("final_norm", col(w(src.final_norm)))
self.ll = []
for i, L in enumerate(src.layers):
d = {"kind": L["kind"], "mlp": {k: C("t%d_%s" % (i, k), conv(w(n))) for k, n in L["mlp"].items()}}
for k in ("operator_norm", "ffn_norm", "post_operator_norm", "post_ffn_norm"):
d[k] = C("t%d_%s" % (i, k), col(w(L[k])))
if L["kind"] == "conv":
d.update(in_proj=C("t%d_in" % i, conv(w(L["in_proj"]))), out_proj=C("t%d_out" % i, conv(w(L["out_proj"]))),
conv=C("t%d_conv" % i, w(L["conv"]).reshape(-1, 1, 1, w(L["conv"]).shape[-1])))
else:
for k in ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj"):
d[k] = C("t%d_%s" % (i, k), conv(w(L[k])))
for k in ("q_norm", "k_norm"):
d[k] = C("t%d_%s" % (i, k), w(L[k]).reshape(1, 1, -1, 1))
self.ll.append(d)
T, thd = self.T, self.tc["head_dim"]
for kind in ("full_attention", "sliding_attention"):
C(kind + "_cos", w(kind + "_cos").T.reshape(1, 1, thd, T))
C(kind + "_sin", w(kind + "_sin").T.reshape(1, 1, thd, T))
m = (w(kind + "_mask") < 0).T # [key, query]
C(kind + "_mask", torch.zeros(T, T).masked_fill(m, NEG).reshape(1, T, 1, T))
self.slots = [int(s) for s in src.slots.tolist()]
self.taps = None # debug: a list collects intermediates, forward then returns (logits, *taps)
self.rescale()
self.split_mlps()
# Scales on the 1x1 conv inputs (exact algebra): the ANE loses precision on small conv inputs (out_proj on its real
# input 43.8 dB, the same input x64 77 dB, measured with single-op exports), so (1) every norm weight is folded into the convs it feeds,
# whose input is then the unit-RMS normalised value, and (2) the rows that make the small products feeding out_proj /
# o_proj / w2 (the gate c, the values v, w3) are scaled up by S and the consuming conv down by S. S keeps the largest
# measured input (out/conv_in_probe.py) far below fp16's max.
S = {"vconv": 64.0, "vattn": 32.0, "vmlp": 32.0, "tconv": 64.0, "tattn": 32.0, "tmlp": 16.0}
# (3) the vision residual stream itself is ~0.03 and the ANE loses precision on small outputs, so the whole stream is
# carried at VS x (v_norm weight and every branch's last conv x VS, vision eps x VS^2), divided back before the
# projector LayerNorm. Largest stream value before the last layer 1.34 (x64 = 86: fp16 squares safe).
VS = 64.0
TS = 32.0 # same for the LM stream (max 28 -> 900)
PATCH = 256.0 # patch embed output x256: a mid-gray frame is 0.0039 after normalisation (blank_gray failed at 28.9 dB)
# (4) LM branch outputs only feed the (scale-invariant) post norms but are small (conv branch median token-max
# 0.0225, out/post_probe.py): their last conv is scaled up, the post norm's eps by the square.
PS = {"conv": 64.0, "attn": 16.0, "mlp": 4.0}
def rescale(self):
g = lambda n: self.g(n).data
def setp(n, t): g(n).copy_(t.to(torch.float16))
def fold(norm, convs): # norm weight [1, C, 1, 1] into conv input channels
wn = g(norm).float().reshape(1, -1, 1, 1)
for c in convs:
setp(c, g(c).float() * wn)
def rows(n, lo, hi, f):
t = g(n).float(); t[lo:hi] *= f; setp(n, t)
def scale(n, f): setp(n, g(n).float() * f)
for p, layers in (("v", self.vl), ("t", self.ll)):
for L in layers:
m = L["mlp"]
if L["kind"] == "conv":
fold(L["operator_norm"], [L["in_proj"]])
D = g(L["in_proj"]).shape[0] // 3
rows(L["in_proj"], D, 2 * D, self.S[p + "conv"]); scale(L["out_proj"], 1 / self.S[p + "conv"])
else:
fold(L["operator_norm"], [L["q_proj"], L["k_proj"], L["v_proj"]] + ([L["gate_proj"]] if p == "t" else []))
rows(L["v_proj"], 0, None, self.S[p + "attn"]); scale(L["out_proj" if p == "v" else "o_proj"], 1 / self.S[p + "attn"])
L["operator_norm"] = None
fold(L["ffn_norm"], [m["w1"], m["w3"]]); L["ffn_norm"] = None
rows(m["w3"], 0, None, self.S[p + "mlp"]); scale(m["w2"], 1 / self.S[p + "mlp"])
fold(self.final_norm, [self.head]); self.final_norm = None
scale(self.v_norm, self.VS)
scale(self.patch, self.PATCH)
for L in self.vl:
scale(L["out_proj"], self.VS); scale(L["mlp"]["w2"], self.VS)
for n in (self.pre_e, self.post_e, self.l2, self.l2b):
scale(n, self.TS)
for L in self.ll:
scale(L["post_operator_norm"], self.TS); scale(L["post_ffn_norm"], self.TS)
scale(L["out_proj"] if L["kind"] == "conv" else L["o_proj"], self.PS["conv" if L["kind"] == "conv" else "attn"])
scale(L["mlp"]["w2"], self.PS["mlp"])
# projector LayerNorm affine into the merge conv: W(x*w + b) = (W*w) x + W b
W, w_, b_ = g(self.l1).float(), g(self.ln_w).float().reshape(1, -1, 1, 1), g(self.ln_b).float().reshape(-1)
setp(self.l1b, g(self.l1b).float() + torch.einsum("ochw,c->o", W, b_)); setp(self.l1, W * w_)
self.ln_w = self.ln_b = None
def g(self, n):
return unwrap(self, n)
def _t(self, x):
if self.taps is not None:
self.taps.append(x)
return x
def k(self, name):
return self.g(self.c[name]) if name in self.c else self.g(name)
# ---- primitives (BC1S) ----
def norm(self, x, w, eps="eps"): # RMSNorm over channels (dim 1); "eps_v": the vision stream, carried at VS x
if eps in ("eps_v", "eps_t", "eps_p"):
x = x * self.k({"eps_v": "inv_vs", "eps_t": "inv_ts", "eps_p": "inv_ps"}[eps]) # squares at the true scale
eps = "eps"
y = x * torch.rsqrt((x * x).mean(1, keepdim=True) + self.k(eps))
return y if w is None else y * self.g(w)
def head_norm(self, x, w): # per-head RMSNorm, x [1, H, hd, S]; pre-scaled by 8 (eps by 64): fp16 squares stay normal
y = x * self.k("pre")
return y * torch.rsqrt((y * y).mean(2, keepdim=True) + self.k("eps_pre")) * self.g(w)
def proj(self, x, w, b=None):
return F.conv2d(x, self.g(w), None if b is None else self.g(b))
def sigmoid(self, x): # the ANE's own sigmoid is coarse (SiLU 52 dB vs 84 dB this way, single-op exports); tanh is not
return (self.k("one") + torch.tanh(x * self.k("half"))) * self.k("half")
def silu(self, x):
return x * self.sigmoid(x)
# The MLP's hidden channels in chunks, w2's partial products summed: 1.4x faster on the ANE for the whole model
# (MLP 8.0 -> 5.5 ms; dev/mlp_variants.py, speed.py). Each chunk is its own weight, so a quantizer sees plain convs.
MLP_SPLIT = 8
def split_mlps(self):
for L in self.vl + self.ll:
p = L["mlp"]
n = self.g(p["w1"]).shape[0] // self.MLP_SPLIT
for k, cut in (("w1", lambda w, s: w[s]), ("w3", lambda w, s: w[s]), ("w2", lambda w, s: w[:, s])):
w = self.g(p[k])
p[k + "s"] = []
for i in range(self.MLP_SPLIT):
name = "%s_%d" % (p[k], i)
self.add_module(name, _W(cut(w, slice(i * n, (i + 1) * n)).clone().contiguous()))
p[k + "s"].append(name)
delattr(self, p[k])
def mlp(self, x, p):
out = None
for w1, w3, w2 in zip(p["w1s"], p["w3s"], p["w2s"]):
y = self.proj(self.silu(self.proj(x, w1)) * self.proj(x, w3), w2)
out = y if out is None else out + y
return out
@staticmethod
def rope(x, cos, sin): # x [1, H, hd, S], cos/sin [1, 1, hd, S]
half = x.shape[2] // 2
a, b = x[:, :, :half], x[:, :, half:]
return x * cos + torch.cat([-b, a], dim=2) * sin
@staticmethod
def attention(q, k, v, H, Hk, hd, mask=None):
"""q [1, H*hd, 1, Sq] (already scaled), k/v [1, Hk*hd, 1, Sk]; mask [1, Sk, 1, Sq] additive."""
qs = q.split(hd, dim=1)
ks = k.transpose(1, 3).split(hd, dim=3) # [1, Sk, 1, hd]
vs = v.transpose(1, 3).split(hd, dim=3)
out = []
for h in range(H):
j = h // (H // Hk)
s = torch.einsum("bchq,bkhc->bkhq", qs[h], ks[j]) # [1, Sk, 1, Sq]
if mask is not None:
s = s + mask
s = torch.softmax(s, dim=1)
out.append(torch.einsum("bkhq,bkhc->bchq", s, vs[j])) # [1, hd, 1, Sq]
return torch.cat(out, dim=1)
# ---- model ----
def vision(self, image):
vc, gh, gw = self.vc, self.gh, self.gw
D, H, Hk = vc["hidden_size"], vc["num_attention_heads"], vc["num_key_value_heads"]
hd, N = D // H, gh * gw
x = F.conv2d(image.to(self.g(self.patch).dtype), self.g(self.patch), stride=16) # [1, D, gh, gw]
x = self._t(self.norm(x, self.v_norm, "eps_p"))
for L in self.vl:
h = self.norm(x, L["operator_norm"], "eps_v")
if L["kind"] == "conv":
b, c, v = self.proj(h, L["in_proj"]).chunk(3, dim=1)
h = self.proj(c * F.conv2d(b * v, self.g(L["spatial"]), padding=1, groups=D), L["out_proj"])
else:
f = h.reshape(1, D, 1, N)
q = self.head_norm(self.proj(f, L["q_proj"]).reshape(1, H, hd, N), L["q_layernorm"])
k = self.head_norm(self.proj(f, L["k_proj"]).reshape(1, Hk, hd, N), L["k_layernorm"])
q = self.rope(q, self.g(self.v_cos), self.g(self.v_sin)) * self.k("v_scale")
k = self.rope(k, self.g(self.v_cos), self.g(self.v_sin))
o = self.attention(q.reshape(1, H * hd, 1, N), k.reshape(1, Hk * hd, 1, N), self.proj(f, L["v_proj"]), H, Hk, hd)
h = self.proj(o, L["out_proj"]).reshape(1, D, gh, gw)
x = x + h
x = self._t(x + self.mlp(self.norm(x, L["ffn_norm"], "eps_v"), L["mlp"]))
x = x * self.k("inv_vs")
xc = x - x.mean(1, keepdim=True)
x = xc * torch.rsqrt((xc * xc).mean(1, keepdim=True) + self.k("eps_ln")) # LayerNorm, affine folded into l1
x = F.gelu(F.conv2d(x, self.g(self.l1), self.g(self.l1b), stride=2)) # [1, 4096, gh/2, gw/2]
x = F.conv2d(x, self.g(self.l2), self.g(self.l2b)) # [1, 2048, gh/2, gw/2]
return self._t(x.reshape(1, x.shape[1], 1, -1)) # [1, 2048, 1, 220], blocks in raster order
def forward(self, image):
return self.language(self.vision(image))
def layer(self, L, x):
tc, T = self.tc, self.T
D, H, Hk, hd = tc["hidden_size"], tc["num_attention_heads"], tc["num_key_value_heads"], tc["head_dim"]
h = self.norm(x, L["operator_norm"], "eps_t")
if L["kind"] == "conv":
b, c, v = self.proj(h, L["in_proj"]).chunk(3, dim=1)
h = self.proj(c * F.conv2d(F.pad(b * v, (6, 0)), self.g(L["conv"]), groups=D), L["out_proj"])
else:
kind = L["kind"]
cos, sin = self.g("p_" + kind + "_cos"), self.g("p_" + kind + "_sin")
q = self.head_norm(self.proj(h, L["q_proj"]).reshape(1, H, hd, T), L["q_norm"])
k = self.head_norm(self.proj(h, L["k_proj"]).reshape(1, Hk, hd, T), L["k_norm"])
q = self.rope(q, cos, sin) * self.k("t_scale")
k = self.rope(k, cos, sin)
o = self.attention(q.reshape(1, H * hd, 1, T), k.reshape(1, Hk * hd, 1, T), self.proj(h, L["v_proj"]), H, Hk, hd,
self.g("p_" + kind + "_mask"))
g = self.sigmoid(self.proj(h, L["gate_proj"])) # [1, H, 1, T]
h = self.proj((o.reshape(1, H, hd, T) * g.reshape(1, H, 1, T)).reshape(1, D, 1, T), L["o_proj"])
x = x + self.norm(h, L["post_operator_norm"], "eps_pconv" if L["kind"] == "conv" else "eps_pattn")
return self._t(x + self.norm(self.mlp(self.norm(x, L["ffn_norm"], "eps_t"), L["mlp"]), L["post_ffn_norm"], "eps_pmlp"))
def language(self, feats):
"""feats [1, 2048, 1, 220] (the projector's output) -> logits [1, U, 1, 12]"""
tc, T = self.tc, self.T
D, H, Hk, hd = tc["hidden_size"], tc["num_attention_heads"], tc["num_key_value_heads"], tc["head_dim"]
x = torch.cat([self.g(self.pre_e), feats.to(torch.float16) if feats.dtype != self.g(self.pre_e).dtype else feats,
self.g(self.post_e)], dim=3) # [1, D, 1, T]
for L in self.ll:
x = self.layer(L, x)
x = torch.cat([x[:, :, :, s:s + 1] for s in self.slots], dim=3) # [1, D, 1, 12]
out = self.proj(self.norm(x, self.final_norm, "eps_t"), self.head) # [1, U, 1, 12]
return out if self.taps is None else (out, *self.taps)