tt-hous's picture
tt-model push embeddinggemma-2-p150 (container)
714c70a verified
Raw History Blame Contribute Delete
32.6 kB
# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc.
# SPDX-License-Identifier: Apache-2.0
import dataclasses
import math
from collections import Counter, OrderedDict
from dataclasses import dataclass
from pathlib import Path
import numpy as np
import torch
from safetensors import safe_open
import ttnn
from models.autoports.google_embeddinggemma_2.tt import optimized_encoder_layer as O
from models.autoports.google_embeddinggemma_2.tt.functional_encoder_layer import MASK_NEG
from models.autoports.google_embeddinggemma_2.tt.fused_encoder_layer import (
FULL,
SLIDING,
TILE,
AttentionPlan,
_to_device,
cu_window_seqlens,
interleaved_rope_tables,
)
from models.common.lightweightmodule import LightweightModule
MRL_DIMS = (768, 512, 256, 128)
SDPA_HEADS = 4
SDPA_WINDOW_LONG_Q = 2048
FOLD_UNIFORM_LENGTHS = (256, 513)
FOLD_UNIFORM_MIN_BATCH = 8
STATIC_MASK_MIN_LEN = 8192
HEADFOLD_MIN_ROWS = 4096
PADDED_HEADFOLD_MIN_LEN = 4096
FOLDED_Q128_MIN_LEN = 2048
MASK_ROW_BLOCK = 1024
ROPE_CACHE_BYTES = 1 << 30
MASK_CACHE_BYTES = 1 << 30
PREFIX = "language_model."
EMBED_DTYPE = torch.bfloat16
L2_EPS = 1e-12
TEXT_MM_CONFIGS = {
("qkv", 512, 1024): (
(128, (2, 4, 4, 2, 2, 8, 4)),
(512, (2, 2, 4, 2, 1, 8, 8)),
(2048, (6, 2, 4, 1, 4, 11, 10)),
(4096, (4, 2, 6, 4, 1, 11, 10)),
(8192, (12, 8, 4, 1, 4, 11, 10)),
(O.UNBOUNDED, (16, 8, 4, 1, 4, 11, 10)),
),
("qkv", 512, 2048): (
(128, (1, 4, 8, 1, 2, 8, 4)),
(512, (2, 2, 6, 1, 1, 11, 10)),
(2048, (2, 2, 8, 2, 2, 11, 10)),
(4096, (6, 4, 4, 3, 1, 11, 10)),
(8192, (12, 8, 4, 4, 1, 11, 10)),
(O.UNBOUNDED, (16, 8, 4, 1, 4, 11, 10)),
),
("qkv", 512, 3072): (
(128, (1, 4, 8, 1, 2, 8, 4)),
(512, (2, 2, 3, 1, 3, 11, 10)),
(2048, (2, 4, 6, 1, 3, 11, 10)),
(4096, (16, 8, 4, 4, 1, 11, 10)),
(8192, (12, 4, 6, 3, 1, 11, 10)),
(O.UNBOUNDED, (16, 4, 6, 1, 2, 11, 10)),
),
("o", 1024, 512): (
(128, (1, 16, 2, 1, 2, 8, 4)),
(512, (3, 4, 3, 3, 1, 8, 8)),
(2048, (3, 4, 2, 3, 1, 11, 10)),
(4096, (4, 4, 2, 1, 2, 11, 10)),
(8192, (4, 4, 2, 4, 1, 11, 10)),
(O.UNBOUNDED, (4, 4, 2, 4, 1, 11, 10)),
),
("o", 2048, 512): (
(128, (2, 8, 4, 1, 4, 8, 4)),
(512, (3, 8, 3, 3, 1, 8, 8)),
(2048, (6, 4, 2, 2, 2, 11, 10)),
(4096, (6, 4, 2, 2, 2, 11, 10)),
(8192, (6, 4, 2, 2, 2, 11, 10)),
(O.UNBOUNDED, (4, 8, 3, 4, 1, 11, 10)),
),
("gate_up", 512, 4096): (
(128, (1, 4, 8, 1, 4, 8, 4)),
(512, (2, 2, 4, 1, 2, 11, 10)),
(2048, (4, 4, 12, 4, 1, 11, 10)),
(4096, (3, 4, 12, 1, 4, 11, 10)),
(8192, (24, 4, 4, 3, 1, 11, 10)),
(O.UNBOUNDED, (24, 4, 4, 3, 1, 11, 10)),
),
("down", 2048, 512): (
(128, (2, 8, 3, 1, 3, 8, 4)),
(512, (2, 8, 2, 2, 1, 8, 8)),
(2048, (3, 4, 2, 3, 1, 11, 10)),
(4096, (4, 4, 3, 4, 1, 11, 10)),
(8192, (6, 4, 2, 2, 2, 11, 10)),
(O.UNBOUNDED, (4, 8, 3, 4, 1, 11, 10)),
),
("ple_gate", 512, 512): (
(128, (1, 8, 3, 1, 1, 8, 4)),
(512, (2, 8, 2, 2, 1, 8, 8)),
(2048, (6, 4, 2, 3, 1, 11, 10)),
(4096, (4, 4, 3, 4, 1, 11, 10)),
(8192, (3, 4, 2, 1, 2, 11, 10)),
(O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)),
),
("ple_proj", 512, 512): (
(128, (1, 8, 3, 1, 1, 8, 4)),
(512, (2, 8, 2, 2, 1, 8, 8)),
(2048, (6, 4, 2, 3, 1, 11, 10)),
(4096, (4, 4, 3, 4, 1, 11, 10)),
(8192, (3, 4, 2, 1, 2, 11, 10)),
(O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)),
),
("ple_in", 512, 512): (
(128, (1, 8, 3, 1, 1, 8, 4)),
(512, (2, 8, 2, 2, 1, 8, 8)),
(2048, (6, 4, 2, 3, 1, 11, 10)),
(4096, (4, 4, 3, 4, 1, 11, 10)),
(8192, (3, 4, 2, 1, 2, 11, 10)),
(O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)),
),
}
TEXT_MM_CONFIGS_HIFI2 = {
("qkv", 512, 1024): (
(128, (2, 4, 4, 2, 2, 8, 4)),
(512, (2, 2, 4, 2, 1, 8, 8)),
(2048, (6, 2, 4, 1, 4, 11, 10)),
(4096, (12, 8, 4, 4, 1, 11, 10)),
(8192, (12, 4, 4, 3, 1, 11, 10)),
(O.UNBOUNDED, (24, 4, 4, 1, 4, 11, 10)),
),
("qkv", 512, 2048): (
(128, (1, 4, 8, 1, 2, 8, 4)),
(512, (2, 2, 6, 1, 1, 11, 10)),
(2048, (2, 2, 8, 2, 2, 11, 10)),
(4096, (12, 4, 4, 1, 2, 11, 10)),
(8192, (24, 4, 4, 2, 1, 11, 10)),
(O.UNBOUNDED, (24, 4, 4, 1, 1, 11, 10)),
),
("qkv", 512, 3072): (
(128, (1, 4, 8, 1, 2, 8, 4)),
(512, (2, 2, 3, 1, 3, 11, 10)),
(2048, (2, 4, 6, 1, 3, 11, 10)),
(4096, (12, 4, 6, 4, 1, 11, 10)),
(8192, (24, 4, 4, 4, 1, 11, 10)),
(O.UNBOUNDED, (24, 4, 4, 1, 4, 11, 10)),
),
("o", 1024, 512): (
(128, (1, 16, 2, 1, 2, 8, 4)),
(512, (3, 4, 3, 3, 1, 8, 8)),
(2048, (3, 4, 2, 3, 1, 11, 10)),
(4096, (4, 4, 2, 4, 1, 11, 10)),
(8192, (8, 8, 3, 4, 1, 11, 10)),
(O.UNBOUNDED, (8, 8, 3, 4, 1, 11, 10)),
),
("o", 2048, 512): (
(128, (2, 8, 4, 1, 4, 8, 4)),
(512, (3, 8, 3, 3, 1, 8, 8)),
(2048, (6, 4, 2, 2, 2, 11, 10)),
(4096, (12, 4, 2, 4, 1, 11, 10)),
(8192, None),
(O.UNBOUNDED, (6, 16, 3, 1, 3, 11, 10)),
),
}
TEXT_POLICY = dataclasses.replace(
O.OPTIMIZED_POLICY,
weight_dtype=O._uniform(O.WEIGHT_GROUPS, "bf16"),
mm_fidelity={**O._uniform(O.WEIGHT_GROUPS, ("hifi4", True)), "qkv": ("hifi2", True), "o": ("hifi2", True)},
out_dtype={},
norm_in_dtype=None,
heads_dtype=None,
sdpa_fidelity=("hifi2", True),
sdpa_chunk={},
mm_config={**TEXT_MM_CONFIGS, **TEXT_MM_CONFIGS_HIFI2},
l1_rows=2048,
mlp_l1_rows=2048,
norm_l1_rows=16384,
residual="fp32",
geglu_fused_min_rows=TILE,
full_attn_l1_rows=1024,
sdpa_impl="custom",
stock_window_shapes=(
((1024, 2), (64, 128)),
((1024, 4), (64, 64)),
((1024, 8), (128, 128)),
((1024, 16), (128, 128)),
((1024, 32), (128, 128)),
),
)
def bf16_embed_scale(hidden_size):
return float(torch.tensor(hidden_size**0.5).to(EMBED_DTYPE))
@dataclass(frozen=True)
class HeadPolicy:
ple_weight_dtype: str = "bf16"
ple_fidelity: tuple = ("hifi4", True)
head_fidelity: tuple = ("hifi4", True)
proj_weight_dtype: str = "bf16"
DEFAULT_HEAD_POLICY = HeadPolicy()
@dataclass(frozen=True)
class TextPlan(AttentionPlan):
k_chunk: int = 0
headfold: bool = False
@dataclass
class TextInputs:
input_ids: ttnn.Tensor
pool: ttnn.Tensor
keypad: ttnn.Tensor
keyrows: ttnn.Tensor
cu_window: ttnn.Tensor
band: ttnn.Tensor
kinds: dict
valid_lengths: tuple
seq_len: int
physical_len: int
exact: bool
owned: list
cache_keys: list
refreshable: bool = True
def host_refreshable(self):
return {
"ids": self.input_ids,
"pool": self.pool,
"keypad": self.keypad,
"keyrows": self.keyrows,
"cu": self.cu_window,
}
class DeviceCache:
def __init__(self, max_bytes):
self.max_bytes = max_bytes
self.items = OrderedDict()
self.pins = Counter()
self.bytes = 0
self.frozen = False
def acquire(self, key, nbytes, build):
if key in self.items:
self.items.move_to_end(key)
else:
if self.frozen or nbytes > self.max_bytes:
return None
self._evict(nbytes)
self.items[key] = (build(), nbytes)
self.bytes += nbytes
self.pins[key] += 1
return self.items[key][0]
def release(self, key):
self.pins[key] -= 1
if self.pins[key] <= 0:
del self.pins[key]
def _evict(self, need):
if self.frozen:
return
for key in list(self.items):
if self.bytes + need <= self.max_bytes:
return
if self.pins[key]:
continue
tensors, nbytes = self.items.pop(key)
for t in tensors:
ttnn.deallocate(t)
self.bytes -= nbytes
def key_padding_host(valid_lengths, phys):
cols = torch.arange(phys)
keep = cols[None, :] < torch.tensor(valid_lengths)[:, None]
return torch.where(keep, 0.0, MASK_NEG).to(torch.bfloat16)[:, None, None, :]
def band_mask_host(config, kind, seq_len, phys):
cols = torch.arange(phys)
mask = torch.empty(phys, phys, dtype=torch.bfloat16)
for r0 in range(0, phys, MASK_ROW_BLOCK):
rows = torch.arange(r0, min(r0 + MASK_ROW_BLOCK, phys))
vis = (cols[None, :] < seq_len).expand(len(rows), phys)
if kind == SLIDING:
vis = vis & ((rows[:, None] - cols[None, :]).abs() <= config.sliding_window)
mask[rows] = torch.where(vis, 0.0, MASK_NEG).to(torch.bfloat16)
return mask[None, None]
def load_text_state(snapshot):
out = {}
with safe_open(str(Path(snapshot) / "model.safetensors"), "pt") as f:
for k in f.keys():
if k.startswith(PREFIX):
out[k[len(PREFIX) :]] = f.get_tensor(k)
return out
class EmbeddingGemma2TextModel(LightweightModule):
def __init__(
self,
mesh_device,
state_dict,
config,
policy=TEXT_POLICY,
head_policy=DEFAULT_HEAD_POLICY,
layer_indices=None,
layer_policies=None,
):
super().__init__()
self.mesh_device = mesh_device
self.config = config
self.policy = policy
self.head_policy = head_policy
self.layer_policies = dict(layer_policies or {})
self.layer_indices = tuple(range(config.num_hidden_layers) if layer_indices is None else layer_indices)
self.hidden = config.hidden_size
self.ple_dim = config.hidden_size_per_layer_input
self.embed_dim = state_dict["embedding_projection.weight"].shape[0]
self.eps = config.rms_norm_eps
arch = mesh_device.arch()
def ckc(fid, fp32):
return ttnn.init_device_compute_kernel_config(
arch,
math_fidelity=O.FIDELITY[fid],
math_approx_mode=False,
fp32_dest_acc_en=fp32,
packer_l1_acc=True,
)
self.ple_cfg = ckc(*head_policy.ple_fidelity)
self.head_cfg = ckc(*head_policy.head_fidelity)
def put(t, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT):
return ttnn.from_torch(
t.contiguous(), dtype=dtype, layout=layout, device=mesh_device, memory_config=ttnn.DRAM_MEMORY_CONFIG
)
scale = bf16_embed_scale(self.hidden)
table = (state_dict["embed_tokens.weight"].to(EMBED_DTYPE) * torch.tensor(scale, dtype=EMBED_DTYPE)).float()
self.media_id = table.shape[0]
table = torch.cat([table, torch.zeros(1, table.shape[1])])
self.embed_scale = scale
self.embed_table = put(table.reshape(1, 1, *table.shape), layout=ttnn.ROW_MAJOR_LAYOUT)
ple_w = state_dict["ple.per_layer_model_projection.weight"].float() * self.hidden**-0.5
ple_w = ple_w.reshape(config.num_hidden_layers, self.ple_dim, self.hidden)
ple_dtype = O.DTYPES[head_policy.ple_weight_dtype]
self.ple_w = {i: put(ple_w[i].T, ple_dtype) for i in self.layer_indices}
self.ple_norm = put(state_dict["ple.per_layer_projection_norm.weight"].float().reshape(1, 1, 1, -1))
self.final_norm = put(state_dict["norm.weight"].float().reshape(1, 1, 1, -1))
self.proj = put(state_dict["embedding_projection.weight"].float().T, O.DTYPES[head_policy.proj_weight_dtype])
self.layers = []
for i in self.layer_indices:
layer_state = {k[len(f"layers.{i}.") :]: v for k, v in state_dict.items() if k.startswith(f"layers.{i}.")}
self.layers.append(
O.OptimizedEncoderLayer.from_state_dict(
layer_state,
hf_config=config,
layer_idx=i,
mesh_device=mesh_device,
prefix="",
policy=self.layer_policies.get(i, policy),
)
)
self._rope = DeviceCache(ROPE_CACHE_BYTES)
self._masks = DeviceCache(MASK_CACHE_BYTES)
@classmethod
def from_pretrained(cls, snapshot, mesh_device, precision_policy=None, text_base=None, **kwargs):
from models.autoports.google_embeddinggemma_2.tt import precision_policy as PP
config = O.TextConfig.from_hf(Path(snapshot) / "config.json")
precision = PP.resolve(precision_policy)
derived = precision.model_kwargs(text_base or TEXT_POLICY)
if "policy" in kwargs:
derived.pop("layer_policies")
kwargs = {**derived, **kwargs}
model = cls(mesh_device, load_text_state(snapshot), config, **kwargs)
model.precision = precision
return model
def kinds(self):
return sorted({self.config.kind(i) for i in self.layer_indices})
def plans(self, physical_len, exact, batch=1):
sw = self.config.sliding_window
cores = self.mesh_device.compute_with_storage_grid_size()
cores = cores.x * cores.y
lo, hi = FOLD_UNIFORM_LENGTHS
fold_exact = batch >= FOLD_UNIFORM_MIN_BATCH and lo <= physical_len <= hi
out = {}
for kind in (SLIDING, FULL):
windowed = kind == SLIDING and physical_len > sw + 1
q, k = sdpa_chunks(physical_len, batch, windowed, cores)
window = 2 * sw if windowed else None
stock_chunks = dict(self.policy.stock_window_shapes).get((physical_len, batch))
if windowed and not exact and stock_chunks:
q, k = stock_chunks
out[kind] = TextPlan(physical_len, physical_len, q, True, None, False, k)
elif windowed:
out[kind] = TextPlan(physical_len, physical_len, q, not exact, window, False, k)
elif exact and kind == FULL and not fold_exact and batch * physical_len >= HEADFOLD_MIN_ROWS:
q, k = headfold_chunks(physical_len, batch)
out[kind] = TextPlan(physical_len, physical_len, q, False, None, False, k, True)
elif (
not exact
and kind == FULL
and self.policy.sdpa_impl == "custom"
and physical_len >= PADDED_HEADFOLD_MIN_LEN
and batch * physical_len >= HEADFOLD_MIN_ROWS
):
q, k = headfold_chunks(physical_len, batch)
out[kind] = TextPlan(physical_len, physical_len, q, True, None, False, k, True)
elif exact and not fold_exact:
out[kind] = TextPlan(physical_len, physical_len, q, False, None, False, k)
else:
q, k = folded_chunks(physical_len, batch, cores)
out[kind] = TextPlan(physical_len, physical_len, q, False, None, True, k)
return out
def _band(self, valid, phys):
key = ("band", valid, phys)
def build():
return (_to_device(band_mask_host(self.config, SLIDING, valid, phys), self.mesh_device),)
cached = self._masks.acquire(key, 2 * phys * phys, build)
if cached is None:
return build()[0], None
return cached[0], key
def _rope_tables(self, kind, phys, repeat):
key = ("rope", kind, phys, repeat)
head_dim = self.config.head_dims[self.config.layer_types.index(kind)]
def build():
cos, sin = interleaved_rope_tables(self.config, kind, phys)
if repeat > 1:
cos, sin = cos.repeat(repeat, 1), sin.repeat(repeat, 1)
return (_to_device(cos[None, None], self.mesh_device), _to_device(sin[None, None], self.mesh_device))
cached = self._rope.acquire(key, 4 * phys * repeat * head_dim, build)
if cached is None:
return build(), None
return cached, key
def host_inputs(self, input_ids, valid_lengths, seq_len=None, exact=None, refreshable=False):
valid_lengths = tuple(int(v) for v in valid_lengths)
b = len(valid_lengths)
if seq_len is None:
seq_len = max(valid_lengths)
if seq_len < max(valid_lengths):
raise ValueError(f"seq_len {seq_len} < longest input {max(valid_lengths)}")
if min(valid_lengths) < 1:
raise ValueError("every input needs at least one token")
phys = uniform_physical_length(seq_len)
is_exact = all(v == phys for v in valid_lengths)
if exact is None:
exact = is_exact
elif exact and not is_exact:
raise ValueError(f"exact plan needs every length equal to {phys}, got {valid_lengths}")
plans = self.plans(phys, exact, b)
ids = torch.zeros(b, phys, dtype=torch.int32)
pool = torch.zeros(1, 1, pool_rows(b), b * phys)
for r, n in enumerate(valid_lengths):
ids[r, :n] = torch.as_tensor(np.asarray(input_ids[r][:n]), dtype=torch.int32)
pool[0, 0, r, r * phys : r * phys + n] = 1.0
used = [plans[k] for k in self.kinds()]
masked = SLIDING in self.kinds() and plans[SLIDING].masked
rows_masked = FULL in self.kinds() and plans[FULL].masked
static_mask = masked and b == 1 and not refreshable and phys > STATIC_MASK_MIN_LEN
keypad = key_padding_host(valid_lengths, phys) if masked or rows_masked else None
return {
"ids": ids,
"pool": pool,
"keypad": keypad if masked and not static_mask else None,
"keyrows": keypad.expand(b, 1, TILE, phys).contiguous() if rows_masked else None,
"mask_valid": valid_lengths[0] if static_mask else phys,
"cu": cu_window_seqlens(valid_lengths, phys) if any(p.folded for p in used) else None,
"plans": plans,
"valid_lengths": valid_lengths,
"seq_len": seq_len,
"physical_len": phys,
"exact": exact,
}
@staticmethod
def host_tensors(host):
out = {
"ids": ttnn.from_torch(host["ids"], dtype=ttnn.uint32, layout=ttnn.ROW_MAJOR_LAYOUT),
"pool": ttnn.from_torch(host["pool"], dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT),
}
for name in ("keypad", "keyrows"):
if host[name] is not None:
out[name] = ttnn.from_torch(host[name], dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT)
if host["cu"] is not None:
out["cu"] = ttnn.from_torch(host["cu"], dtype=ttnn.uint32, layout=ttnn.ROW_MAJOR_LAYOUT)
return out
def to_device_inputs(self, host):
dev = self.mesh_device
phys = host["physical_len"]
b = len(host["valid_lengths"])
owned, keys = [], []
try:
tensors = {
k: ttnn.to_device(t, dev, memory_config=ttnn.DRAM_MEMORY_CONFIG)
for k, t in self.host_tensors(host).items()
}
owned += list(tensors.values())
band = None
if SLIDING in self.kinds() and host["plans"][SLIDING].masked:
band, key = self._band(host["mask_valid"], phys)
if key:
keys.append(key)
else:
owned.append(band)
kinds = {}
for kind in self.kinds():
plan = host["plans"][kind]
(cos, sin), key = self._rope_tables(kind, phys, b if plan.folded else 1)
if key:
keys.append(key)
else:
owned.extend((cos, sin))
kinds[kind] = {"plan": plan, "cos": cos, "sin": sin}
except Exception:
self._release(owned, keys)
raise
return TextInputs(
input_ids=tensors["ids"],
pool=tensors["pool"],
keypad=tensors.get("keypad"),
keyrows=tensors.get("keyrows"),
cu_window=tensors.get("cu"),
band=band,
kinds=kinds,
valid_lengths=host["valid_lengths"],
seq_len=host["seq_len"],
physical_len=phys,
exact=host["exact"],
owned=owned,
cache_keys=keys,
refreshable=host["keypad"] is not None or band is None,
)
def prepare_inputs(self, input_ids, valid_lengths, seq_len=None, exact=None, refreshable=False):
return self.to_device_inputs(self.host_inputs(input_ids, valid_lengths, seq_len, exact, refreshable))
def refresh_inputs(self, inputs, host):
if not inputs.refreshable:
raise ValueError("inputs hold a mask built for fixed lengths; prepare them with refreshable=True")
if (host["physical_len"], len(host["valid_lengths"]), host["exact"]) != (
inputs.physical_len,
len(inputs.valid_lengths),
inputs.exact,
):
raise ValueError("host inputs do not match the persistent input signature")
dst = inputs.host_refreshable()
for name, t in self.host_tensors(host).items():
ttnn.copy_host_to_device_tensor(t, dst[name])
inputs.valid_lengths = host["valid_lengths"]
inputs.seq_len = host["seq_len"]
def freeze_caches(self, frozen=True):
for cache in (self._rope, self._masks):
cache.frozen = frozen
def _release(self, owned, keys):
for t in owned:
ttnn.deallocate(t)
for key in keys:
(self._rope if key[0] == "rope" else self._masks).release(key)
def release(self, inputs):
self._release(inputs.owned, inputs.cache_keys)
inputs.owned, inputs.cache_keys = [], []
def embed(self, input_ids):
b, phys = input_ids.shape
x = ttnn.embedding(input_ids, self.embed_table, layout=ttnn.TILE_LAYOUT, memory_config=ttnn.DRAM_MEMORY_CONFIG)
return ttnn.reshape(x, [b, 1, phys, self.hidden])
def per_layer_input(self, embeds, layer_idx):
b, one, phys, d = embeds.shape
kw = {}
cfg = mm_config_for(self.policy.mm_config, ("ple_in", d, self.ple_dim), b * phys)
if cfg is not None:
kw["config"] = cfg
p = ttnn.experimental.minimal_matmul(
ttnn.reshape(embeds, [1, 1, b * phys, d]),
self.ple_w[layer_idx],
compute_kernel_config=self.ple_cfg,
memory_config=ttnn.DRAM_MEMORY_CONFIG,
dtype=ttnn.bfloat16,
**kw,
)
out = ttnn.rms_norm(
p,
epsilon=self.eps,
weight=self.ple_norm,
compute_kernel_config=self.ple_cfg,
memory_config=ttnn.DRAM_MEMORY_CONFIG,
)
ttnn.deallocate(p)
return ttnn.reshape(out, [b, one, phys, self.ple_dim])
def attention_mask(self, inputs):
if inputs.keypad is None:
return inputs.band
return ttnn.add(inputs.band, inputs.keypad, memory_config=ttnn.DRAM_MEMORY_CONFIG)
def merge_media(self, embeds, media):
merged = ttnn.add(embeds, ttnn.reshape(media, embeds.shape), memory_config=ttnn.DRAM_MEMORY_CONFIG)
ttnn.deallocate(embeds)
return merged
def encode_hidden(self, inputs, media=None):
mask = self.attention_mask(inputs)
embeds = self.embed(inputs.input_ids)
if media is not None:
embeds = self.merge_media(embeds, media)
x = embeds
for layer in self.layers:
k = inputs.kinds[layer.kind]
plan = k["plan"]
ple = self.per_layer_input(embeds, layer.layer_idx)
layer_mask = None
if plan.masked:
layer_mask = inputs.keyrows if layer.kind == FULL else mask
y = layer.forward(
x,
ple,
layer_mask,
k["cos"],
k["sin"],
plan,
inputs.cu_window if plan.folded else None,
)
ttnn.deallocate(ple)
if x is not embeds:
ttnn.deallocate(x)
x = y
ttnn.deallocate(embeds)
if mask is not None and mask is not inputs.band:
ttnn.deallocate(mask)
return x
def pooled(self, hidden, inputs):
b, one, phys, d = hidden.shape
normed = ttnn.rms_norm(
ttnn.reshape(hidden, [1, 1, b * phys, d]),
epsilon=self.eps,
weight=self.final_norm,
compute_kernel_config=self.head_cfg,
memory_config=ttnn.DRAM_MEMORY_CONFIG,
)
summed = ttnn.matmul(
inputs.pool,
normed,
compute_kernel_config=self.head_cfg,
memory_config=ttnn.DRAM_MEMORY_CONFIG,
dtype=ttnn.float32,
)
ttnn.deallocate(normed)
out = ttnn.matmul(
summed,
self.proj,
compute_kernel_config=self.head_cfg,
memory_config=ttnn.DRAM_MEMORY_CONFIG,
dtype=ttnn.float32,
)
ttnn.deallocate(summed)
return out
def normalize(self, projected, dim):
x = projected
if dim != self.embed_dim:
x = ttnn.slice(projected, [0, 0, 0, 0], [1, 1, projected.shape[2], dim])
sq = ttnn.multiply(x, x, dtype=ttnn.float32, memory_config=ttnn.DRAM_MEMORY_CONFIG)
ss = ttnn.sum(sq, dim=-1, keepdim=True, compute_kernel_config=self.head_cfg)
ttnn.deallocate(sq)
ss_eps = ttnn.add(ss, L2_EPS * L2_EPS, dtype=ttnn.float32)
ttnn.deallocate(ss)
inv = ttnn.rsqrt(ss_eps, fast_and_approximate_mode=False)
ttnn.deallocate(ss_eps)
out = ttnn.multiply(x, inv, dtype=ttnn.float32, memory_config=ttnn.DRAM_MEMORY_CONFIG)
ttnn.deallocate(inv)
if x is not projected:
ttnn.deallocate(x)
return out
def forward(self, inputs, dims=(768,), outputs=None, media=None):
hidden = self.encode_hidden(inputs, media)
projected = self.pooled(hidden, inputs)
ttnn.deallocate(hidden)
out = {}
for d in dims:
y = self.normalize(projected, d)
if outputs is None:
out[d] = y
else:
ttnn.copy(y, outputs[d])
ttnn.deallocate(y)
out[d] = outputs[d]
ttnn.deallocate(projected)
return out
def allocate_outputs(self, batch, dims=(768,)):
return {
d: ttnn.from_torch(
torch.zeros(1, 1, pool_rows(batch), d),
dtype=ttnn.float32,
layout=ttnn.TILE_LAYOUT,
device=self.mesh_device,
memory_config=ttnn.DRAM_MEMORY_CONFIG,
)
for d in dims
}
@staticmethod
def to_numpy(t, batch):
x = ttnn.to_torch(ttnn.get_device_tensors(t)[0]).float()
return x.reshape(-1, x.shape[-1])[:batch].numpy()
class TraceSlot:
def __init__(self, model, batch, physical_len, exact=False, dims=(768,)):
self.model = model
self.batch = batch
self.physical_len = physical_len
self.exact = exact
self.dims = tuple(dims)
n = physical_len if exact else 1
self.inputs = model.prepare_inputs([[2] * n] * batch, [n] * batch, physical_len, exact=exact, refreshable=True)
self.outputs = model.allocate_outputs(batch, self.dims)
self.trace_id = None
self.replays = 0
self.refreshes = 0
def key(self):
return (self.batch, self.physical_len, self.exact)
def run_eager(self):
return self.model.forward(self.inputs, self.dims, outputs=self.outputs)
def capture(self):
mesh = self.model.mesh_device
if self.trace_id is not None:
raise RuntimeError(f"slot {self.key()} already captured")
self.trace_id = ttnn.begin_trace_capture(mesh, cq_id=0)
try:
self.model.forward(self.inputs, self.dims, outputs=self.outputs)
finally:
ttnn.end_trace_capture(mesh, self.trace_id, cq_id=0)
def refresh(self, host):
self.model.refresh_inputs(self.inputs, host)
self.refreshes += 1
def replay(self, blocking=False):
ttnn.execute_trace(self.model.mesh_device, self.trace_id, cq_id=0, blocking=blocking)
self.replays += 1
def read(self, rows, dims=None):
ttnn.synchronize_device(self.model.mesh_device)
return {d: self.model.to_numpy(self.outputs[d], rows) for d in (dims or self.dims)}
def release(self):
if self.trace_id is not None:
ttnn.release_trace(self.model.mesh_device, self.trace_id)
self.trace_id = None
for t in self.outputs.values():
ttnn.deallocate(t)
self.outputs = {}
self.model.release(self.inputs)
class TraceSet:
def __init__(self, model, shapes, dims=(768,), extra_slots=()):
self.model = model
self.slots = {}
self.extra = list(extra_slots)
for batch, phys, exact in shapes:
key = (batch, phys, exact)
if key not in self.slots:
self.slots[key] = TraceSlot(model, batch, phys, exact, dims)
everything = list(self.slots.values()) + self.extra
for slot in everything:
slot.run_eager()
ttnn.synchronize_device(model.mesh_device)
for slot in everything:
slot.capture()
self.captures = len(everything)
model.freeze_caches()
def slot_for(self, rows, phys, exact=False):
fits = [s for (b, p, e), s in self.slots.items() if p == phys and e == exact and b >= rows]
return min(fits, key=lambda s: s.batch) if fits else None
def run(self, slot, input_ids, valid_lengths, dims=None):
rows = len(valid_lengths)
ids = list(input_ids) + [[2]] * (slot.batch - rows)
lengths = list(valid_lengths) + [1] * (slot.batch - rows)
host = self.model.host_inputs(ids, lengths, slot.physical_len, exact=slot.exact, refreshable=True)
slot.refresh(host)
slot.replay()
return slot.read(rows, dims)
def release(self):
for slot in list(self.slots.values()) + self.extra:
slot.release()
self.slots = {}
self.extra = []
self.model.freeze_caches(False)
def mm_config_for(table, key, rows):
blocks = next((b for max_rows, b in table.get(key, ()) if rows <= max_rows), None)
if blocks is None:
return None
mb, kb, nb, sh, sw, gx, gy = blocks
return ttnn.MinimalMatmulConfig(
M_block_size=mb,
K_block_size=kb,
N_block_size=nb,
subblock_h=sh,
subblock_w=sw,
compute_with_storage_grid_size=ttnn.CoreCoord(gx, gy),
)
def headfold_chunks(physical_len, batch):
if batch == 1:
prefs = ((64, 128), (64, 64), (32, 32))
elif batch >= 32 and physical_len <= 2048:
prefs = ((128, 64), (64, 64), (32, 32))
else:
prefs = ((128, 128), (64, 64), (32, 32))
return next(
(q, k) for q, k in prefs if physical_len % q == 0 and physical_len % k == 0 and (physical_len // k) % 2 == 0
)
def folded_chunks(physical_len, batch, cores):
q, k = sdpa_chunks(physical_len, batch, False, cores)
if physical_len >= FOLDED_Q128_MIN_LEN and physical_len % 128 == 0 and (physical_len // 128) % 2 == 0:
q = 128
return q, k
def sdpa_chunks(physical_len, batch, windowed, cores):
def fits(c):
return physical_len % c == 0 and (physical_len // c) % 2 == 0
if windowed:
q = 128 if physical_len >= SDPA_WINDOW_LONG_Q and fits(128) else 64 if fits(64) else 32
return q, q
q = next((c for c in (128, 64, 32) if fits(c) and batch * SDPA_HEADS * physical_len // c >= cores), None)
if q is None:
q = next(c for c in (64, 32) if fits(c))
k = next(c for c in (128, 64, 32) if fits(c))
return q, k
def pool_rows(batch):
return TILE * math.ceil(batch / TILE)
def uniform_physical_length(seq_len):
align = 256 if seq_len >= 2048 else 128 if seq_len >= 512 else 64
return align * math.ceil(seq_len / align)
def mrl_dims(embed_dim):
return tuple(d for d in MRL_DIMS if d <= embed_dim)
def bucket_length(n, buckets):
return next((b for b in buckets if n <= b), n)