# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. # SPDX-License-Identifier: Apache-2.0 import dataclasses import math from collections import Counter, OrderedDict from dataclasses import dataclass from pathlib import Path import numpy as np import torch from safetensors import safe_open import ttnn from models.autoports.google_embeddinggemma_2.tt import optimized_encoder_layer as O from models.autoports.google_embeddinggemma_2.tt.functional_encoder_layer import MASK_NEG from models.autoports.google_embeddinggemma_2.tt.fused_encoder_layer import ( FULL, SLIDING, TILE, AttentionPlan, _to_device, cu_window_seqlens, interleaved_rope_tables, ) from models.common.lightweightmodule import LightweightModule MRL_DIMS = (768, 512, 256, 128) SDPA_HEADS = 4 SDPA_WINDOW_LONG_Q = 2048 FOLD_UNIFORM_LENGTHS = (256, 513) FOLD_UNIFORM_MIN_BATCH = 8 STATIC_MASK_MIN_LEN = 8192 HEADFOLD_MIN_ROWS = 4096 PADDED_HEADFOLD_MIN_LEN = 4096 FOLDED_Q128_MIN_LEN = 2048 MASK_ROW_BLOCK = 1024 ROPE_CACHE_BYTES = 1 << 30 MASK_CACHE_BYTES = 1 << 30 PREFIX = "language_model." EMBED_DTYPE = torch.bfloat16 L2_EPS = 1e-12 TEXT_MM_CONFIGS = { ("qkv", 512, 1024): ( (128, (2, 4, 4, 2, 2, 8, 4)), (512, (2, 2, 4, 2, 1, 8, 8)), (2048, (6, 2, 4, 1, 4, 11, 10)), (4096, (4, 2, 6, 4, 1, 11, 10)), (8192, (12, 8, 4, 1, 4, 11, 10)), (O.UNBOUNDED, (16, 8, 4, 1, 4, 11, 10)), ), ("qkv", 512, 2048): ( (128, (1, 4, 8, 1, 2, 8, 4)), (512, (2, 2, 6, 1, 1, 11, 10)), (2048, (2, 2, 8, 2, 2, 11, 10)), (4096, (6, 4, 4, 3, 1, 11, 10)), (8192, (12, 8, 4, 4, 1, 11, 10)), (O.UNBOUNDED, (16, 8, 4, 1, 4, 11, 10)), ), ("qkv", 512, 3072): ( (128, (1, 4, 8, 1, 2, 8, 4)), (512, (2, 2, 3, 1, 3, 11, 10)), (2048, (2, 4, 6, 1, 3, 11, 10)), (4096, (16, 8, 4, 4, 1, 11, 10)), (8192, (12, 4, 6, 3, 1, 11, 10)), (O.UNBOUNDED, (16, 4, 6, 1, 2, 11, 10)), ), ("o", 1024, 512): ( (128, (1, 16, 2, 1, 2, 8, 4)), (512, (3, 4, 3, 3, 1, 8, 8)), (2048, (3, 4, 2, 3, 1, 11, 10)), (4096, (4, 4, 2, 1, 2, 11, 10)), (8192, (4, 4, 2, 4, 1, 11, 10)), (O.UNBOUNDED, (4, 4, 2, 4, 1, 11, 10)), ), ("o", 2048, 512): ( (128, (2, 8, 4, 1, 4, 8, 4)), (512, (3, 8, 3, 3, 1, 8, 8)), (2048, (6, 4, 2, 2, 2, 11, 10)), (4096, (6, 4, 2, 2, 2, 11, 10)), (8192, (6, 4, 2, 2, 2, 11, 10)), (O.UNBOUNDED, (4, 8, 3, 4, 1, 11, 10)), ), ("gate_up", 512, 4096): ( (128, (1, 4, 8, 1, 4, 8, 4)), (512, (2, 2, 4, 1, 2, 11, 10)), (2048, (4, 4, 12, 4, 1, 11, 10)), (4096, (3, 4, 12, 1, 4, 11, 10)), (8192, (24, 4, 4, 3, 1, 11, 10)), (O.UNBOUNDED, (24, 4, 4, 3, 1, 11, 10)), ), ("down", 2048, 512): ( (128, (2, 8, 3, 1, 3, 8, 4)), (512, (2, 8, 2, 2, 1, 8, 8)), (2048, (3, 4, 2, 3, 1, 11, 10)), (4096, (4, 4, 3, 4, 1, 11, 10)), (8192, (6, 4, 2, 2, 2, 11, 10)), (O.UNBOUNDED, (4, 8, 3, 4, 1, 11, 10)), ), ("ple_gate", 512, 512): ( (128, (1, 8, 3, 1, 1, 8, 4)), (512, (2, 8, 2, 2, 1, 8, 8)), (2048, (6, 4, 2, 3, 1, 11, 10)), (4096, (4, 4, 3, 4, 1, 11, 10)), (8192, (3, 4, 2, 1, 2, 11, 10)), (O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)), ), ("ple_proj", 512, 512): ( (128, (1, 8, 3, 1, 1, 8, 4)), (512, (2, 8, 2, 2, 1, 8, 8)), (2048, (6, 4, 2, 3, 1, 11, 10)), (4096, (4, 4, 3, 4, 1, 11, 10)), (8192, (3, 4, 2, 1, 2, 11, 10)), (O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)), ), ("ple_in", 512, 512): ( (128, (1, 8, 3, 1, 1, 8, 4)), (512, (2, 8, 2, 2, 1, 8, 8)), (2048, (6, 4, 2, 3, 1, 11, 10)), (4096, (4, 4, 3, 4, 1, 11, 10)), (8192, (3, 4, 2, 1, 2, 11, 10)), (O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)), ), } TEXT_MM_CONFIGS_HIFI2 = { ("qkv", 512, 1024): ( (128, (2, 4, 4, 2, 2, 8, 4)), (512, (2, 2, 4, 2, 1, 8, 8)), (2048, (6, 2, 4, 1, 4, 11, 10)), (4096, (12, 8, 4, 4, 1, 11, 10)), (8192, (12, 4, 4, 3, 1, 11, 10)), (O.UNBOUNDED, (24, 4, 4, 1, 4, 11, 10)), ), ("qkv", 512, 2048): ( (128, (1, 4, 8, 1, 2, 8, 4)), (512, (2, 2, 6, 1, 1, 11, 10)), (2048, (2, 2, 8, 2, 2, 11, 10)), (4096, (12, 4, 4, 1, 2, 11, 10)), (8192, (24, 4, 4, 2, 1, 11, 10)), (O.UNBOUNDED, (24, 4, 4, 1, 1, 11, 10)), ), ("qkv", 512, 3072): ( (128, (1, 4, 8, 1, 2, 8, 4)), (512, (2, 2, 3, 1, 3, 11, 10)), (2048, (2, 4, 6, 1, 3, 11, 10)), (4096, (12, 4, 6, 4, 1, 11, 10)), (8192, (24, 4, 4, 4, 1, 11, 10)), (O.UNBOUNDED, (24, 4, 4, 1, 4, 11, 10)), ), ("o", 1024, 512): ( (128, (1, 16, 2, 1, 2, 8, 4)), (512, (3, 4, 3, 3, 1, 8, 8)), (2048, (3, 4, 2, 3, 1, 11, 10)), (4096, (4, 4, 2, 4, 1, 11, 10)), (8192, (8, 8, 3, 4, 1, 11, 10)), (O.UNBOUNDED, (8, 8, 3, 4, 1, 11, 10)), ), ("o", 2048, 512): ( (128, (2, 8, 4, 1, 4, 8, 4)), (512, (3, 8, 3, 3, 1, 8, 8)), (2048, (6, 4, 2, 2, 2, 11, 10)), (4096, (12, 4, 2, 4, 1, 11, 10)), (8192, None), (O.UNBOUNDED, (6, 16, 3, 1, 3, 11, 10)), ), } TEXT_POLICY = dataclasses.replace( O.OPTIMIZED_POLICY, weight_dtype=O._uniform(O.WEIGHT_GROUPS, "bf16"), mm_fidelity={**O._uniform(O.WEIGHT_GROUPS, ("hifi4", True)), "qkv": ("hifi2", True), "o": ("hifi2", True)}, out_dtype={}, norm_in_dtype=None, heads_dtype=None, sdpa_fidelity=("hifi2", True), sdpa_chunk={}, mm_config={**TEXT_MM_CONFIGS, **TEXT_MM_CONFIGS_HIFI2}, l1_rows=2048, mlp_l1_rows=2048, norm_l1_rows=16384, residual="fp32", geglu_fused_min_rows=TILE, full_attn_l1_rows=1024, sdpa_impl="custom", stock_window_shapes=( ((1024, 2), (64, 128)), ((1024, 4), (64, 64)), ((1024, 8), (128, 128)), ((1024, 16), (128, 128)), ((1024, 32), (128, 128)), ), ) def bf16_embed_scale(hidden_size): return float(torch.tensor(hidden_size**0.5).to(EMBED_DTYPE)) @dataclass(frozen=True) class HeadPolicy: ple_weight_dtype: str = "bf16" ple_fidelity: tuple = ("hifi4", True) head_fidelity: tuple = ("hifi4", True) proj_weight_dtype: str = "bf16" DEFAULT_HEAD_POLICY = HeadPolicy() @dataclass(frozen=True) class TextPlan(AttentionPlan): k_chunk: int = 0 headfold: bool = False @dataclass class TextInputs: input_ids: ttnn.Tensor pool: ttnn.Tensor keypad: ttnn.Tensor keyrows: ttnn.Tensor cu_window: ttnn.Tensor band: ttnn.Tensor kinds: dict valid_lengths: tuple seq_len: int physical_len: int exact: bool owned: list cache_keys: list refreshable: bool = True def host_refreshable(self): return { "ids": self.input_ids, "pool": self.pool, "keypad": self.keypad, "keyrows": self.keyrows, "cu": self.cu_window, } class DeviceCache: def __init__(self, max_bytes): self.max_bytes = max_bytes self.items = OrderedDict() self.pins = Counter() self.bytes = 0 self.frozen = False def acquire(self, key, nbytes, build): if key in self.items: self.items.move_to_end(key) else: if self.frozen or nbytes > self.max_bytes: return None self._evict(nbytes) self.items[key] = (build(), nbytes) self.bytes += nbytes self.pins[key] += 1 return self.items[key][0] def release(self, key): self.pins[key] -= 1 if self.pins[key] <= 0: del self.pins[key] def _evict(self, need): if self.frozen: return for key in list(self.items): if self.bytes + need <= self.max_bytes: return if self.pins[key]: continue tensors, nbytes = self.items.pop(key) for t in tensors: ttnn.deallocate(t) self.bytes -= nbytes def key_padding_host(valid_lengths, phys): cols = torch.arange(phys) keep = cols[None, :] < torch.tensor(valid_lengths)[:, None] return torch.where(keep, 0.0, MASK_NEG).to(torch.bfloat16)[:, None, None, :] def band_mask_host(config, kind, seq_len, phys): cols = torch.arange(phys) mask = torch.empty(phys, phys, dtype=torch.bfloat16) for r0 in range(0, phys, MASK_ROW_BLOCK): rows = torch.arange(r0, min(r0 + MASK_ROW_BLOCK, phys)) vis = (cols[None, :] < seq_len).expand(len(rows), phys) if kind == SLIDING: vis = vis & ((rows[:, None] - cols[None, :]).abs() <= config.sliding_window) mask[rows] = torch.where(vis, 0.0, MASK_NEG).to(torch.bfloat16) return mask[None, None] def load_text_state(snapshot): out = {} with safe_open(str(Path(snapshot) / "model.safetensors"), "pt") as f: for k in f.keys(): if k.startswith(PREFIX): out[k[len(PREFIX) :]] = f.get_tensor(k) return out class EmbeddingGemma2TextModel(LightweightModule): def __init__( self, mesh_device, state_dict, config, policy=TEXT_POLICY, head_policy=DEFAULT_HEAD_POLICY, layer_indices=None, layer_policies=None, ): super().__init__() self.mesh_device = mesh_device self.config = config self.policy = policy self.head_policy = head_policy self.layer_policies = dict(layer_policies or {}) self.layer_indices = tuple(range(config.num_hidden_layers) if layer_indices is None else layer_indices) self.hidden = config.hidden_size self.ple_dim = config.hidden_size_per_layer_input self.embed_dim = state_dict["embedding_projection.weight"].shape[0] self.eps = config.rms_norm_eps arch = mesh_device.arch() def ckc(fid, fp32): return ttnn.init_device_compute_kernel_config( arch, math_fidelity=O.FIDELITY[fid], math_approx_mode=False, fp32_dest_acc_en=fp32, packer_l1_acc=True, ) self.ple_cfg = ckc(*head_policy.ple_fidelity) self.head_cfg = ckc(*head_policy.head_fidelity) def put(t, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT): return ttnn.from_torch( t.contiguous(), dtype=dtype, layout=layout, device=mesh_device, memory_config=ttnn.DRAM_MEMORY_CONFIG ) scale = bf16_embed_scale(self.hidden) table = (state_dict["embed_tokens.weight"].to(EMBED_DTYPE) * torch.tensor(scale, dtype=EMBED_DTYPE)).float() self.media_id = table.shape[0] table = torch.cat([table, torch.zeros(1, table.shape[1])]) self.embed_scale = scale self.embed_table = put(table.reshape(1, 1, *table.shape), layout=ttnn.ROW_MAJOR_LAYOUT) ple_w = state_dict["ple.per_layer_model_projection.weight"].float() * self.hidden**-0.5 ple_w = ple_w.reshape(config.num_hidden_layers, self.ple_dim, self.hidden) ple_dtype = O.DTYPES[head_policy.ple_weight_dtype] self.ple_w = {i: put(ple_w[i].T, ple_dtype) for i in self.layer_indices} self.ple_norm = put(state_dict["ple.per_layer_projection_norm.weight"].float().reshape(1, 1, 1, -1)) self.final_norm = put(state_dict["norm.weight"].float().reshape(1, 1, 1, -1)) self.proj = put(state_dict["embedding_projection.weight"].float().T, O.DTYPES[head_policy.proj_weight_dtype]) self.layers = [] for i in self.layer_indices: layer_state = {k[len(f"layers.{i}.") :]: v for k, v in state_dict.items() if k.startswith(f"layers.{i}.")} self.layers.append( O.OptimizedEncoderLayer.from_state_dict( layer_state, hf_config=config, layer_idx=i, mesh_device=mesh_device, prefix="", policy=self.layer_policies.get(i, policy), ) ) self._rope = DeviceCache(ROPE_CACHE_BYTES) self._masks = DeviceCache(MASK_CACHE_BYTES) @classmethod def from_pretrained(cls, snapshot, mesh_device, precision_policy=None, text_base=None, **kwargs): from models.autoports.google_embeddinggemma_2.tt import precision_policy as PP config = O.TextConfig.from_hf(Path(snapshot) / "config.json") precision = PP.resolve(precision_policy) derived = precision.model_kwargs(text_base or TEXT_POLICY) if "policy" in kwargs: derived.pop("layer_policies") kwargs = {**derived, **kwargs} model = cls(mesh_device, load_text_state(snapshot), config, **kwargs) model.precision = precision return model def kinds(self): return sorted({self.config.kind(i) for i in self.layer_indices}) def plans(self, physical_len, exact, batch=1): sw = self.config.sliding_window cores = self.mesh_device.compute_with_storage_grid_size() cores = cores.x * cores.y lo, hi = FOLD_UNIFORM_LENGTHS fold_exact = batch >= FOLD_UNIFORM_MIN_BATCH and lo <= physical_len <= hi out = {} for kind in (SLIDING, FULL): windowed = kind == SLIDING and physical_len > sw + 1 q, k = sdpa_chunks(physical_len, batch, windowed, cores) window = 2 * sw if windowed else None stock_chunks = dict(self.policy.stock_window_shapes).get((physical_len, batch)) if windowed and not exact and stock_chunks: q, k = stock_chunks out[kind] = TextPlan(physical_len, physical_len, q, True, None, False, k) elif windowed: out[kind] = TextPlan(physical_len, physical_len, q, not exact, window, False, k) elif exact and kind == FULL and not fold_exact and batch * physical_len >= HEADFOLD_MIN_ROWS: q, k = headfold_chunks(physical_len, batch) out[kind] = TextPlan(physical_len, physical_len, q, False, None, False, k, True) elif ( not exact and kind == FULL and self.policy.sdpa_impl == "custom" and physical_len >= PADDED_HEADFOLD_MIN_LEN and batch * physical_len >= HEADFOLD_MIN_ROWS ): q, k = headfold_chunks(physical_len, batch) out[kind] = TextPlan(physical_len, physical_len, q, True, None, False, k, True) elif exact and not fold_exact: out[kind] = TextPlan(physical_len, physical_len, q, False, None, False, k) else: q, k = folded_chunks(physical_len, batch, cores) out[kind] = TextPlan(physical_len, physical_len, q, False, None, True, k) return out def _band(self, valid, phys): key = ("band", valid, phys) def build(): return (_to_device(band_mask_host(self.config, SLIDING, valid, phys), self.mesh_device),) cached = self._masks.acquire(key, 2 * phys * phys, build) if cached is None: return build()[0], None return cached[0], key def _rope_tables(self, kind, phys, repeat): key = ("rope", kind, phys, repeat) head_dim = self.config.head_dims[self.config.layer_types.index(kind)] def build(): cos, sin = interleaved_rope_tables(self.config, kind, phys) if repeat > 1: cos, sin = cos.repeat(repeat, 1), sin.repeat(repeat, 1) return (_to_device(cos[None, None], self.mesh_device), _to_device(sin[None, None], self.mesh_device)) cached = self._rope.acquire(key, 4 * phys * repeat * head_dim, build) if cached is None: return build(), None return cached, key def host_inputs(self, input_ids, valid_lengths, seq_len=None, exact=None, refreshable=False): valid_lengths = tuple(int(v) for v in valid_lengths) b = len(valid_lengths) if seq_len is None: seq_len = max(valid_lengths) if seq_len < max(valid_lengths): raise ValueError(f"seq_len {seq_len} < longest input {max(valid_lengths)}") if min(valid_lengths) < 1: raise ValueError("every input needs at least one token") phys = uniform_physical_length(seq_len) is_exact = all(v == phys for v in valid_lengths) if exact is None: exact = is_exact elif exact and not is_exact: raise ValueError(f"exact plan needs every length equal to {phys}, got {valid_lengths}") plans = self.plans(phys, exact, b) ids = torch.zeros(b, phys, dtype=torch.int32) pool = torch.zeros(1, 1, pool_rows(b), b * phys) for r, n in enumerate(valid_lengths): ids[r, :n] = torch.as_tensor(np.asarray(input_ids[r][:n]), dtype=torch.int32) pool[0, 0, r, r * phys : r * phys + n] = 1.0 used = [plans[k] for k in self.kinds()] masked = SLIDING in self.kinds() and plans[SLIDING].masked rows_masked = FULL in self.kinds() and plans[FULL].masked static_mask = masked and b == 1 and not refreshable and phys > STATIC_MASK_MIN_LEN keypad = key_padding_host(valid_lengths, phys) if masked or rows_masked else None return { "ids": ids, "pool": pool, "keypad": keypad if masked and not static_mask else None, "keyrows": keypad.expand(b, 1, TILE, phys).contiguous() if rows_masked else None, "mask_valid": valid_lengths[0] if static_mask else phys, "cu": cu_window_seqlens(valid_lengths, phys) if any(p.folded for p in used) else None, "plans": plans, "valid_lengths": valid_lengths, "seq_len": seq_len, "physical_len": phys, "exact": exact, } @staticmethod def host_tensors(host): out = { "ids": ttnn.from_torch(host["ids"], dtype=ttnn.uint32, layout=ttnn.ROW_MAJOR_LAYOUT), "pool": ttnn.from_torch(host["pool"], dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT), } for name in ("keypad", "keyrows"): if host[name] is not None: out[name] = ttnn.from_torch(host[name], dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT) if host["cu"] is not None: out["cu"] = ttnn.from_torch(host["cu"], dtype=ttnn.uint32, layout=ttnn.ROW_MAJOR_LAYOUT) return out def to_device_inputs(self, host): dev = self.mesh_device phys = host["physical_len"] b = len(host["valid_lengths"]) owned, keys = [], [] try: tensors = { k: ttnn.to_device(t, dev, memory_config=ttnn.DRAM_MEMORY_CONFIG) for k, t in self.host_tensors(host).items() } owned += list(tensors.values()) band = None if SLIDING in self.kinds() and host["plans"][SLIDING].masked: band, key = self._band(host["mask_valid"], phys) if key: keys.append(key) else: owned.append(band) kinds = {} for kind in self.kinds(): plan = host["plans"][kind] (cos, sin), key = self._rope_tables(kind, phys, b if plan.folded else 1) if key: keys.append(key) else: owned.extend((cos, sin)) kinds[kind] = {"plan": plan, "cos": cos, "sin": sin} except Exception: self._release(owned, keys) raise return TextInputs( input_ids=tensors["ids"], pool=tensors["pool"], keypad=tensors.get("keypad"), keyrows=tensors.get("keyrows"), cu_window=tensors.get("cu"), band=band, kinds=kinds, valid_lengths=host["valid_lengths"], seq_len=host["seq_len"], physical_len=phys, exact=host["exact"], owned=owned, cache_keys=keys, refreshable=host["keypad"] is not None or band is None, ) def prepare_inputs(self, input_ids, valid_lengths, seq_len=None, exact=None, refreshable=False): return self.to_device_inputs(self.host_inputs(input_ids, valid_lengths, seq_len, exact, refreshable)) def refresh_inputs(self, inputs, host): if not inputs.refreshable: raise ValueError("inputs hold a mask built for fixed lengths; prepare them with refreshable=True") if (host["physical_len"], len(host["valid_lengths"]), host["exact"]) != ( inputs.physical_len, len(inputs.valid_lengths), inputs.exact, ): raise ValueError("host inputs do not match the persistent input signature") dst = inputs.host_refreshable() for name, t in self.host_tensors(host).items(): ttnn.copy_host_to_device_tensor(t, dst[name]) inputs.valid_lengths = host["valid_lengths"] inputs.seq_len = host["seq_len"] def freeze_caches(self, frozen=True): for cache in (self._rope, self._masks): cache.frozen = frozen def _release(self, owned, keys): for t in owned: ttnn.deallocate(t) for key in keys: (self._rope if key[0] == "rope" else self._masks).release(key) def release(self, inputs): self._release(inputs.owned, inputs.cache_keys) inputs.owned, inputs.cache_keys = [], [] def embed(self, input_ids): b, phys = input_ids.shape x = ttnn.embedding(input_ids, self.embed_table, layout=ttnn.TILE_LAYOUT, memory_config=ttnn.DRAM_MEMORY_CONFIG) return ttnn.reshape(x, [b, 1, phys, self.hidden]) def per_layer_input(self, embeds, layer_idx): b, one, phys, d = embeds.shape kw = {} cfg = mm_config_for(self.policy.mm_config, ("ple_in", d, self.ple_dim), b * phys) if cfg is not None: kw["config"] = cfg p = ttnn.experimental.minimal_matmul( ttnn.reshape(embeds, [1, 1, b * phys, d]), self.ple_w[layer_idx], compute_kernel_config=self.ple_cfg, memory_config=ttnn.DRAM_MEMORY_CONFIG, dtype=ttnn.bfloat16, **kw, ) out = ttnn.rms_norm( p, epsilon=self.eps, weight=self.ple_norm, compute_kernel_config=self.ple_cfg, memory_config=ttnn.DRAM_MEMORY_CONFIG, ) ttnn.deallocate(p) return ttnn.reshape(out, [b, one, phys, self.ple_dim]) def attention_mask(self, inputs): if inputs.keypad is None: return inputs.band return ttnn.add(inputs.band, inputs.keypad, memory_config=ttnn.DRAM_MEMORY_CONFIG) def merge_media(self, embeds, media): merged = ttnn.add(embeds, ttnn.reshape(media, embeds.shape), memory_config=ttnn.DRAM_MEMORY_CONFIG) ttnn.deallocate(embeds) return merged def encode_hidden(self, inputs, media=None): mask = self.attention_mask(inputs) embeds = self.embed(inputs.input_ids) if media is not None: embeds = self.merge_media(embeds, media) x = embeds for layer in self.layers: k = inputs.kinds[layer.kind] plan = k["plan"] ple = self.per_layer_input(embeds, layer.layer_idx) layer_mask = None if plan.masked: layer_mask = inputs.keyrows if layer.kind == FULL else mask y = layer.forward( x, ple, layer_mask, k["cos"], k["sin"], plan, inputs.cu_window if plan.folded else None, ) ttnn.deallocate(ple) if x is not embeds: ttnn.deallocate(x) x = y ttnn.deallocate(embeds) if mask is not None and mask is not inputs.band: ttnn.deallocate(mask) return x def pooled(self, hidden, inputs): b, one, phys, d = hidden.shape normed = ttnn.rms_norm( ttnn.reshape(hidden, [1, 1, b * phys, d]), epsilon=self.eps, weight=self.final_norm, compute_kernel_config=self.head_cfg, memory_config=ttnn.DRAM_MEMORY_CONFIG, ) summed = ttnn.matmul( inputs.pool, normed, compute_kernel_config=self.head_cfg, memory_config=ttnn.DRAM_MEMORY_CONFIG, dtype=ttnn.float32, ) ttnn.deallocate(normed) out = ttnn.matmul( summed, self.proj, compute_kernel_config=self.head_cfg, memory_config=ttnn.DRAM_MEMORY_CONFIG, dtype=ttnn.float32, ) ttnn.deallocate(summed) return out def normalize(self, projected, dim): x = projected if dim != self.embed_dim: x = ttnn.slice(projected, [0, 0, 0, 0], [1, 1, projected.shape[2], dim]) sq = ttnn.multiply(x, x, dtype=ttnn.float32, memory_config=ttnn.DRAM_MEMORY_CONFIG) ss = ttnn.sum(sq, dim=-1, keepdim=True, compute_kernel_config=self.head_cfg) ttnn.deallocate(sq) ss_eps = ttnn.add(ss, L2_EPS * L2_EPS, dtype=ttnn.float32) ttnn.deallocate(ss) inv = ttnn.rsqrt(ss_eps, fast_and_approximate_mode=False) ttnn.deallocate(ss_eps) out = ttnn.multiply(x, inv, dtype=ttnn.float32, memory_config=ttnn.DRAM_MEMORY_CONFIG) ttnn.deallocate(inv) if x is not projected: ttnn.deallocate(x) return out def forward(self, inputs, dims=(768,), outputs=None, media=None): hidden = self.encode_hidden(inputs, media) projected = self.pooled(hidden, inputs) ttnn.deallocate(hidden) out = {} for d in dims: y = self.normalize(projected, d) if outputs is None: out[d] = y else: ttnn.copy(y, outputs[d]) ttnn.deallocate(y) out[d] = outputs[d] ttnn.deallocate(projected) return out def allocate_outputs(self, batch, dims=(768,)): return { d: ttnn.from_torch( torch.zeros(1, 1, pool_rows(batch), d), dtype=ttnn.float32, layout=ttnn.TILE_LAYOUT, device=self.mesh_device, memory_config=ttnn.DRAM_MEMORY_CONFIG, ) for d in dims } @staticmethod def to_numpy(t, batch): x = ttnn.to_torch(ttnn.get_device_tensors(t)[0]).float() return x.reshape(-1, x.shape[-1])[:batch].numpy() class TraceSlot: def __init__(self, model, batch, physical_len, exact=False, dims=(768,)): self.model = model self.batch = batch self.physical_len = physical_len self.exact = exact self.dims = tuple(dims) n = physical_len if exact else 1 self.inputs = model.prepare_inputs([[2] * n] * batch, [n] * batch, physical_len, exact=exact, refreshable=True) self.outputs = model.allocate_outputs(batch, self.dims) self.trace_id = None self.replays = 0 self.refreshes = 0 def key(self): return (self.batch, self.physical_len, self.exact) def run_eager(self): return self.model.forward(self.inputs, self.dims, outputs=self.outputs) def capture(self): mesh = self.model.mesh_device if self.trace_id is not None: raise RuntimeError(f"slot {self.key()} already captured") self.trace_id = ttnn.begin_trace_capture(mesh, cq_id=0) try: self.model.forward(self.inputs, self.dims, outputs=self.outputs) finally: ttnn.end_trace_capture(mesh, self.trace_id, cq_id=0) def refresh(self, host): self.model.refresh_inputs(self.inputs, host) self.refreshes += 1 def replay(self, blocking=False): ttnn.execute_trace(self.model.mesh_device, self.trace_id, cq_id=0, blocking=blocking) self.replays += 1 def read(self, rows, dims=None): ttnn.synchronize_device(self.model.mesh_device) return {d: self.model.to_numpy(self.outputs[d], rows) for d in (dims or self.dims)} def release(self): if self.trace_id is not None: ttnn.release_trace(self.model.mesh_device, self.trace_id) self.trace_id = None for t in self.outputs.values(): ttnn.deallocate(t) self.outputs = {} self.model.release(self.inputs) class TraceSet: def __init__(self, model, shapes, dims=(768,), extra_slots=()): self.model = model self.slots = {} self.extra = list(extra_slots) for batch, phys, exact in shapes: key = (batch, phys, exact) if key not in self.slots: self.slots[key] = TraceSlot(model, batch, phys, exact, dims) everything = list(self.slots.values()) + self.extra for slot in everything: slot.run_eager() ttnn.synchronize_device(model.mesh_device) for slot in everything: slot.capture() self.captures = len(everything) model.freeze_caches() def slot_for(self, rows, phys, exact=False): fits = [s for (b, p, e), s in self.slots.items() if p == phys and e == exact and b >= rows] return min(fits, key=lambda s: s.batch) if fits else None def run(self, slot, input_ids, valid_lengths, dims=None): rows = len(valid_lengths) ids = list(input_ids) + [[2]] * (slot.batch - rows) lengths = list(valid_lengths) + [1] * (slot.batch - rows) host = self.model.host_inputs(ids, lengths, slot.physical_len, exact=slot.exact, refreshable=True) slot.refresh(host) slot.replay() return slot.read(rows, dims) def release(self): for slot in list(self.slots.values()) + self.extra: slot.release() self.slots = {} self.extra = [] self.model.freeze_caches(False) def mm_config_for(table, key, rows): blocks = next((b for max_rows, b in table.get(key, ()) if rows <= max_rows), None) if blocks is None: return None mb, kb, nb, sh, sw, gx, gy = blocks return ttnn.MinimalMatmulConfig( M_block_size=mb, K_block_size=kb, N_block_size=nb, subblock_h=sh, subblock_w=sw, compute_with_storage_grid_size=ttnn.CoreCoord(gx, gy), ) def headfold_chunks(physical_len, batch): if batch == 1: prefs = ((64, 128), (64, 64), (32, 32)) elif batch >= 32 and physical_len <= 2048: prefs = ((128, 64), (64, 64), (32, 32)) else: prefs = ((128, 128), (64, 64), (32, 32)) return next( (q, k) for q, k in prefs if physical_len % q == 0 and physical_len % k == 0 and (physical_len // k) % 2 == 0 ) def folded_chunks(physical_len, batch, cores): q, k = sdpa_chunks(physical_len, batch, False, cores) if physical_len >= FOLDED_Q128_MIN_LEN and physical_len % 128 == 0 and (physical_len // 128) % 2 == 0: q = 128 return q, k def sdpa_chunks(physical_len, batch, windowed, cores): def fits(c): return physical_len % c == 0 and (physical_len // c) % 2 == 0 if windowed: q = 128 if physical_len >= SDPA_WINDOW_LONG_Q and fits(128) else 64 if fits(64) else 32 return q, q q = next((c for c in (128, 64, 32) if fits(c) and batch * SDPA_HEADS * physical_len // c >= cores), None) if q is None: q = next(c for c in (64, 32) if fits(c)) k = next(c for c in (128, 64, 32) if fits(c)) return q, k def pool_rows(batch): return TILE * math.ceil(batch / TILE) def uniform_physical_length(seq_len): align = 256 if seq_len >= 2048 else 128 if seq_len >= 512 else 64 return align * math.ceil(seq_len / align) def mrl_dims(embed_dim): return tuple(d for d in MRL_DIMS if d <= embed_dim) def bucket_length(n, buckets): return next((b for b in buckets if n <= b), n)