Download code/models/autoports/google_embeddinggemma_2/tt/model.py from tt-hous/embeddinggemma-2: direct link, hf CLI and curl.
- Browser
- Download file 32.6 kB
-
https://huggingface.co/tt-hous/embeddinggemma-2/resolve/main/code/models/autoports/google_embeddinggemma_2/tt/model.py
- Command line
-
hf download hf://tt-hous/embeddinggemma-2/code/models/autoports/google_embeddinggemma_2/tt/model.py
-
curl -L -o model.py https://huggingface.co/tt-hous/embeddinggemma-2/resolve/main/code/models/autoports/google_embeddinggemma_2/tt/model.py
32.6 kB
| # SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| import dataclasses | |
| import math | |
| from collections import Counter, OrderedDict | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from safetensors import safe_open | |
| import ttnn | |
| from models.autoports.google_embeddinggemma_2.tt import optimized_encoder_layer as O | |
| from models.autoports.google_embeddinggemma_2.tt.functional_encoder_layer import MASK_NEG | |
| from models.autoports.google_embeddinggemma_2.tt.fused_encoder_layer import ( | |
| FULL, | |
| SLIDING, | |
| TILE, | |
| AttentionPlan, | |
| _to_device, | |
| cu_window_seqlens, | |
| interleaved_rope_tables, | |
| ) | |
| from models.common.lightweightmodule import LightweightModule | |
| MRL_DIMS = (768, 512, 256, 128) | |
| SDPA_HEADS = 4 | |
| SDPA_WINDOW_LONG_Q = 2048 | |
| FOLD_UNIFORM_LENGTHS = (256, 513) | |
| FOLD_UNIFORM_MIN_BATCH = 8 | |
| STATIC_MASK_MIN_LEN = 8192 | |
| HEADFOLD_MIN_ROWS = 4096 | |
| PADDED_HEADFOLD_MIN_LEN = 4096 | |
| FOLDED_Q128_MIN_LEN = 2048 | |
| MASK_ROW_BLOCK = 1024 | |
| ROPE_CACHE_BYTES = 1 << 30 | |
| MASK_CACHE_BYTES = 1 << 30 | |
| PREFIX = "language_model." | |
| EMBED_DTYPE = torch.bfloat16 | |
| L2_EPS = 1e-12 | |
| TEXT_MM_CONFIGS = { | |
| ("qkv", 512, 1024): ( | |
| (128, (2, 4, 4, 2, 2, 8, 4)), | |
| (512, (2, 2, 4, 2, 1, 8, 8)), | |
| (2048, (6, 2, 4, 1, 4, 11, 10)), | |
| (4096, (4, 2, 6, 4, 1, 11, 10)), | |
| (8192, (12, 8, 4, 1, 4, 11, 10)), | |
| (O.UNBOUNDED, (16, 8, 4, 1, 4, 11, 10)), | |
| ), | |
| ("qkv", 512, 2048): ( | |
| (128, (1, 4, 8, 1, 2, 8, 4)), | |
| (512, (2, 2, 6, 1, 1, 11, 10)), | |
| (2048, (2, 2, 8, 2, 2, 11, 10)), | |
| (4096, (6, 4, 4, 3, 1, 11, 10)), | |
| (8192, (12, 8, 4, 4, 1, 11, 10)), | |
| (O.UNBOUNDED, (16, 8, 4, 1, 4, 11, 10)), | |
| ), | |
| ("qkv", 512, 3072): ( | |
| (128, (1, 4, 8, 1, 2, 8, 4)), | |
| (512, (2, 2, 3, 1, 3, 11, 10)), | |
| (2048, (2, 4, 6, 1, 3, 11, 10)), | |
| (4096, (16, 8, 4, 4, 1, 11, 10)), | |
| (8192, (12, 4, 6, 3, 1, 11, 10)), | |
| (O.UNBOUNDED, (16, 4, 6, 1, 2, 11, 10)), | |
| ), | |
| ("o", 1024, 512): ( | |
| (128, (1, 16, 2, 1, 2, 8, 4)), | |
| (512, (3, 4, 3, 3, 1, 8, 8)), | |
| (2048, (3, 4, 2, 3, 1, 11, 10)), | |
| (4096, (4, 4, 2, 1, 2, 11, 10)), | |
| (8192, (4, 4, 2, 4, 1, 11, 10)), | |
| (O.UNBOUNDED, (4, 4, 2, 4, 1, 11, 10)), | |
| ), | |
| ("o", 2048, 512): ( | |
| (128, (2, 8, 4, 1, 4, 8, 4)), | |
| (512, (3, 8, 3, 3, 1, 8, 8)), | |
| (2048, (6, 4, 2, 2, 2, 11, 10)), | |
| (4096, (6, 4, 2, 2, 2, 11, 10)), | |
| (8192, (6, 4, 2, 2, 2, 11, 10)), | |
| (O.UNBOUNDED, (4, 8, 3, 4, 1, 11, 10)), | |
| ), | |
| ("gate_up", 512, 4096): ( | |
| (128, (1, 4, 8, 1, 4, 8, 4)), | |
| (512, (2, 2, 4, 1, 2, 11, 10)), | |
| (2048, (4, 4, 12, 4, 1, 11, 10)), | |
| (4096, (3, 4, 12, 1, 4, 11, 10)), | |
| (8192, (24, 4, 4, 3, 1, 11, 10)), | |
| (O.UNBOUNDED, (24, 4, 4, 3, 1, 11, 10)), | |
| ), | |
| ("down", 2048, 512): ( | |
| (128, (2, 8, 3, 1, 3, 8, 4)), | |
| (512, (2, 8, 2, 2, 1, 8, 8)), | |
| (2048, (3, 4, 2, 3, 1, 11, 10)), | |
| (4096, (4, 4, 3, 4, 1, 11, 10)), | |
| (8192, (6, 4, 2, 2, 2, 11, 10)), | |
| (O.UNBOUNDED, (4, 8, 3, 4, 1, 11, 10)), | |
| ), | |
| ("ple_gate", 512, 512): ( | |
| (128, (1, 8, 3, 1, 1, 8, 4)), | |
| (512, (2, 8, 2, 2, 1, 8, 8)), | |
| (2048, (6, 4, 2, 3, 1, 11, 10)), | |
| (4096, (4, 4, 3, 4, 1, 11, 10)), | |
| (8192, (3, 4, 2, 1, 2, 11, 10)), | |
| (O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)), | |
| ), | |
| ("ple_proj", 512, 512): ( | |
| (128, (1, 8, 3, 1, 1, 8, 4)), | |
| (512, (2, 8, 2, 2, 1, 8, 8)), | |
| (2048, (6, 4, 2, 3, 1, 11, 10)), | |
| (4096, (4, 4, 3, 4, 1, 11, 10)), | |
| (8192, (3, 4, 2, 1, 2, 11, 10)), | |
| (O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)), | |
| ), | |
| ("ple_in", 512, 512): ( | |
| (128, (1, 8, 3, 1, 1, 8, 4)), | |
| (512, (2, 8, 2, 2, 1, 8, 8)), | |
| (2048, (6, 4, 2, 3, 1, 11, 10)), | |
| (4096, (4, 4, 3, 4, 1, 11, 10)), | |
| (8192, (3, 4, 2, 1, 2, 11, 10)), | |
| (O.UNBOUNDED, (3, 4, 2, 3, 1, 11, 10)), | |
| ), | |
| } | |
| TEXT_MM_CONFIGS_HIFI2 = { | |
| ("qkv", 512, 1024): ( | |
| (128, (2, 4, 4, 2, 2, 8, 4)), | |
| (512, (2, 2, 4, 2, 1, 8, 8)), | |
| (2048, (6, 2, 4, 1, 4, 11, 10)), | |
| (4096, (12, 8, 4, 4, 1, 11, 10)), | |
| (8192, (12, 4, 4, 3, 1, 11, 10)), | |
| (O.UNBOUNDED, (24, 4, 4, 1, 4, 11, 10)), | |
| ), | |
| ("qkv", 512, 2048): ( | |
| (128, (1, 4, 8, 1, 2, 8, 4)), | |
| (512, (2, 2, 6, 1, 1, 11, 10)), | |
| (2048, (2, 2, 8, 2, 2, 11, 10)), | |
| (4096, (12, 4, 4, 1, 2, 11, 10)), | |
| (8192, (24, 4, 4, 2, 1, 11, 10)), | |
| (O.UNBOUNDED, (24, 4, 4, 1, 1, 11, 10)), | |
| ), | |
| ("qkv", 512, 3072): ( | |
| (128, (1, 4, 8, 1, 2, 8, 4)), | |
| (512, (2, 2, 3, 1, 3, 11, 10)), | |
| (2048, (2, 4, 6, 1, 3, 11, 10)), | |
| (4096, (12, 4, 6, 4, 1, 11, 10)), | |
| (8192, (24, 4, 4, 4, 1, 11, 10)), | |
| (O.UNBOUNDED, (24, 4, 4, 1, 4, 11, 10)), | |
| ), | |
| ("o", 1024, 512): ( | |
| (128, (1, 16, 2, 1, 2, 8, 4)), | |
| (512, (3, 4, 3, 3, 1, 8, 8)), | |
| (2048, (3, 4, 2, 3, 1, 11, 10)), | |
| (4096, (4, 4, 2, 4, 1, 11, 10)), | |
| (8192, (8, 8, 3, 4, 1, 11, 10)), | |
| (O.UNBOUNDED, (8, 8, 3, 4, 1, 11, 10)), | |
| ), | |
| ("o", 2048, 512): ( | |
| (128, (2, 8, 4, 1, 4, 8, 4)), | |
| (512, (3, 8, 3, 3, 1, 8, 8)), | |
| (2048, (6, 4, 2, 2, 2, 11, 10)), | |
| (4096, (12, 4, 2, 4, 1, 11, 10)), | |
| (8192, None), | |
| (O.UNBOUNDED, (6, 16, 3, 1, 3, 11, 10)), | |
| ), | |
| } | |
| TEXT_POLICY = dataclasses.replace( | |
| O.OPTIMIZED_POLICY, | |
| weight_dtype=O._uniform(O.WEIGHT_GROUPS, "bf16"), | |
| mm_fidelity={**O._uniform(O.WEIGHT_GROUPS, ("hifi4", True)), "qkv": ("hifi2", True), "o": ("hifi2", True)}, | |
| out_dtype={}, | |
| norm_in_dtype=None, | |
| heads_dtype=None, | |
| sdpa_fidelity=("hifi2", True), | |
| sdpa_chunk={}, | |
| mm_config={**TEXT_MM_CONFIGS, **TEXT_MM_CONFIGS_HIFI2}, | |
| l1_rows=2048, | |
| mlp_l1_rows=2048, | |
| norm_l1_rows=16384, | |
| residual="fp32", | |
| geglu_fused_min_rows=TILE, | |
| full_attn_l1_rows=1024, | |
| sdpa_impl="custom", | |
| stock_window_shapes=( | |
| ((1024, 2), (64, 128)), | |
| ((1024, 4), (64, 64)), | |
| ((1024, 8), (128, 128)), | |
| ((1024, 16), (128, 128)), | |
| ((1024, 32), (128, 128)), | |
| ), | |
| ) | |
| def bf16_embed_scale(hidden_size): | |
| return float(torch.tensor(hidden_size**0.5).to(EMBED_DTYPE)) | |
| class HeadPolicy: | |
| ple_weight_dtype: str = "bf16" | |
| ple_fidelity: tuple = ("hifi4", True) | |
| head_fidelity: tuple = ("hifi4", True) | |
| proj_weight_dtype: str = "bf16" | |
| DEFAULT_HEAD_POLICY = HeadPolicy() | |
| class TextPlan(AttentionPlan): | |
| k_chunk: int = 0 | |
| headfold: bool = False | |
| class TextInputs: | |
| input_ids: ttnn.Tensor | |
| pool: ttnn.Tensor | |
| keypad: ttnn.Tensor | |
| keyrows: ttnn.Tensor | |
| cu_window: ttnn.Tensor | |
| band: ttnn.Tensor | |
| kinds: dict | |
| valid_lengths: tuple | |
| seq_len: int | |
| physical_len: int | |
| exact: bool | |
| owned: list | |
| cache_keys: list | |
| refreshable: bool = True | |
| def host_refreshable(self): | |
| return { | |
| "ids": self.input_ids, | |
| "pool": self.pool, | |
| "keypad": self.keypad, | |
| "keyrows": self.keyrows, | |
| "cu": self.cu_window, | |
| } | |
| class DeviceCache: | |
| def __init__(self, max_bytes): | |
| self.max_bytes = max_bytes | |
| self.items = OrderedDict() | |
| self.pins = Counter() | |
| self.bytes = 0 | |
| self.frozen = False | |
| def acquire(self, key, nbytes, build): | |
| if key in self.items: | |
| self.items.move_to_end(key) | |
| else: | |
| if self.frozen or nbytes > self.max_bytes: | |
| return None | |
| self._evict(nbytes) | |
| self.items[key] = (build(), nbytes) | |
| self.bytes += nbytes | |
| self.pins[key] += 1 | |
| return self.items[key][0] | |
| def release(self, key): | |
| self.pins[key] -= 1 | |
| if self.pins[key] <= 0: | |
| del self.pins[key] | |
| def _evict(self, need): | |
| if self.frozen: | |
| return | |
| for key in list(self.items): | |
| if self.bytes + need <= self.max_bytes: | |
| return | |
| if self.pins[key]: | |
| continue | |
| tensors, nbytes = self.items.pop(key) | |
| for t in tensors: | |
| ttnn.deallocate(t) | |
| self.bytes -= nbytes | |
| def key_padding_host(valid_lengths, phys): | |
| cols = torch.arange(phys) | |
| keep = cols[None, :] < torch.tensor(valid_lengths)[:, None] | |
| return torch.where(keep, 0.0, MASK_NEG).to(torch.bfloat16)[:, None, None, :] | |
| def band_mask_host(config, kind, seq_len, phys): | |
| cols = torch.arange(phys) | |
| mask = torch.empty(phys, phys, dtype=torch.bfloat16) | |
| for r0 in range(0, phys, MASK_ROW_BLOCK): | |
| rows = torch.arange(r0, min(r0 + MASK_ROW_BLOCK, phys)) | |
| vis = (cols[None, :] < seq_len).expand(len(rows), phys) | |
| if kind == SLIDING: | |
| vis = vis & ((rows[:, None] - cols[None, :]).abs() <= config.sliding_window) | |
| mask[rows] = torch.where(vis, 0.0, MASK_NEG).to(torch.bfloat16) | |
| return mask[None, None] | |
| def load_text_state(snapshot): | |
| out = {} | |
| with safe_open(str(Path(snapshot) / "model.safetensors"), "pt") as f: | |
| for k in f.keys(): | |
| if k.startswith(PREFIX): | |
| out[k[len(PREFIX) :]] = f.get_tensor(k) | |
| return out | |
| class EmbeddingGemma2TextModel(LightweightModule): | |
| def __init__( | |
| self, | |
| mesh_device, | |
| state_dict, | |
| config, | |
| policy=TEXT_POLICY, | |
| head_policy=DEFAULT_HEAD_POLICY, | |
| layer_indices=None, | |
| layer_policies=None, | |
| ): | |
| super().__init__() | |
| self.mesh_device = mesh_device | |
| self.config = config | |
| self.policy = policy | |
| self.head_policy = head_policy | |
| self.layer_policies = dict(layer_policies or {}) | |
| self.layer_indices = tuple(range(config.num_hidden_layers) if layer_indices is None else layer_indices) | |
| self.hidden = config.hidden_size | |
| self.ple_dim = config.hidden_size_per_layer_input | |
| self.embed_dim = state_dict["embedding_projection.weight"].shape[0] | |
| self.eps = config.rms_norm_eps | |
| arch = mesh_device.arch() | |
| def ckc(fid, fp32): | |
| return ttnn.init_device_compute_kernel_config( | |
| arch, | |
| math_fidelity=O.FIDELITY[fid], | |
| math_approx_mode=False, | |
| fp32_dest_acc_en=fp32, | |
| packer_l1_acc=True, | |
| ) | |
| self.ple_cfg = ckc(*head_policy.ple_fidelity) | |
| self.head_cfg = ckc(*head_policy.head_fidelity) | |
| def put(t, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT): | |
| return ttnn.from_torch( | |
| t.contiguous(), dtype=dtype, layout=layout, device=mesh_device, memory_config=ttnn.DRAM_MEMORY_CONFIG | |
| ) | |
| scale = bf16_embed_scale(self.hidden) | |
| table = (state_dict["embed_tokens.weight"].to(EMBED_DTYPE) * torch.tensor(scale, dtype=EMBED_DTYPE)).float() | |
| self.media_id = table.shape[0] | |
| table = torch.cat([table, torch.zeros(1, table.shape[1])]) | |
| self.embed_scale = scale | |
| self.embed_table = put(table.reshape(1, 1, *table.shape), layout=ttnn.ROW_MAJOR_LAYOUT) | |
| ple_w = state_dict["ple.per_layer_model_projection.weight"].float() * self.hidden**-0.5 | |
| ple_w = ple_w.reshape(config.num_hidden_layers, self.ple_dim, self.hidden) | |
| ple_dtype = O.DTYPES[head_policy.ple_weight_dtype] | |
| self.ple_w = {i: put(ple_w[i].T, ple_dtype) for i in self.layer_indices} | |
| self.ple_norm = put(state_dict["ple.per_layer_projection_norm.weight"].float().reshape(1, 1, 1, -1)) | |
| self.final_norm = put(state_dict["norm.weight"].float().reshape(1, 1, 1, -1)) | |
| self.proj = put(state_dict["embedding_projection.weight"].float().T, O.DTYPES[head_policy.proj_weight_dtype]) | |
| self.layers = [] | |
| for i in self.layer_indices: | |
| layer_state = {k[len(f"layers.{i}.") :]: v for k, v in state_dict.items() if k.startswith(f"layers.{i}.")} | |
| self.layers.append( | |
| O.OptimizedEncoderLayer.from_state_dict( | |
| layer_state, | |
| hf_config=config, | |
| layer_idx=i, | |
| mesh_device=mesh_device, | |
| prefix="", | |
| policy=self.layer_policies.get(i, policy), | |
| ) | |
| ) | |
| self._rope = DeviceCache(ROPE_CACHE_BYTES) | |
| self._masks = DeviceCache(MASK_CACHE_BYTES) | |
| def from_pretrained(cls, snapshot, mesh_device, precision_policy=None, text_base=None, **kwargs): | |
| from models.autoports.google_embeddinggemma_2.tt import precision_policy as PP | |
| config = O.TextConfig.from_hf(Path(snapshot) / "config.json") | |
| precision = PP.resolve(precision_policy) | |
| derived = precision.model_kwargs(text_base or TEXT_POLICY) | |
| if "policy" in kwargs: | |
| derived.pop("layer_policies") | |
| kwargs = {**derived, **kwargs} | |
| model = cls(mesh_device, load_text_state(snapshot), config, **kwargs) | |
| model.precision = precision | |
| return model | |
| def kinds(self): | |
| return sorted({self.config.kind(i) for i in self.layer_indices}) | |
| def plans(self, physical_len, exact, batch=1): | |
| sw = self.config.sliding_window | |
| cores = self.mesh_device.compute_with_storage_grid_size() | |
| cores = cores.x * cores.y | |
| lo, hi = FOLD_UNIFORM_LENGTHS | |
| fold_exact = batch >= FOLD_UNIFORM_MIN_BATCH and lo <= physical_len <= hi | |
| out = {} | |
| for kind in (SLIDING, FULL): | |
| windowed = kind == SLIDING and physical_len > sw + 1 | |
| q, k = sdpa_chunks(physical_len, batch, windowed, cores) | |
| window = 2 * sw if windowed else None | |
| stock_chunks = dict(self.policy.stock_window_shapes).get((physical_len, batch)) | |
| if windowed and not exact and stock_chunks: | |
| q, k = stock_chunks | |
| out[kind] = TextPlan(physical_len, physical_len, q, True, None, False, k) | |
| elif windowed: | |
| out[kind] = TextPlan(physical_len, physical_len, q, not exact, window, False, k) | |
| elif exact and kind == FULL and not fold_exact and batch * physical_len >= HEADFOLD_MIN_ROWS: | |
| q, k = headfold_chunks(physical_len, batch) | |
| out[kind] = TextPlan(physical_len, physical_len, q, False, None, False, k, True) | |
| elif ( | |
| not exact | |
| and kind == FULL | |
| and self.policy.sdpa_impl == "custom" | |
| and physical_len >= PADDED_HEADFOLD_MIN_LEN | |
| and batch * physical_len >= HEADFOLD_MIN_ROWS | |
| ): | |
| q, k = headfold_chunks(physical_len, batch) | |
| out[kind] = TextPlan(physical_len, physical_len, q, True, None, False, k, True) | |
| elif exact and not fold_exact: | |
| out[kind] = TextPlan(physical_len, physical_len, q, False, None, False, k) | |
| else: | |
| q, k = folded_chunks(physical_len, batch, cores) | |
| out[kind] = TextPlan(physical_len, physical_len, q, False, None, True, k) | |
| return out | |
| def _band(self, valid, phys): | |
| key = ("band", valid, phys) | |
| def build(): | |
| return (_to_device(band_mask_host(self.config, SLIDING, valid, phys), self.mesh_device),) | |
| cached = self._masks.acquire(key, 2 * phys * phys, build) | |
| if cached is None: | |
| return build()[0], None | |
| return cached[0], key | |
| def _rope_tables(self, kind, phys, repeat): | |
| key = ("rope", kind, phys, repeat) | |
| head_dim = self.config.head_dims[self.config.layer_types.index(kind)] | |
| def build(): | |
| cos, sin = interleaved_rope_tables(self.config, kind, phys) | |
| if repeat > 1: | |
| cos, sin = cos.repeat(repeat, 1), sin.repeat(repeat, 1) | |
| return (_to_device(cos[None, None], self.mesh_device), _to_device(sin[None, None], self.mesh_device)) | |
| cached = self._rope.acquire(key, 4 * phys * repeat * head_dim, build) | |
| if cached is None: | |
| return build(), None | |
| return cached, key | |
| def host_inputs(self, input_ids, valid_lengths, seq_len=None, exact=None, refreshable=False): | |
| valid_lengths = tuple(int(v) for v in valid_lengths) | |
| b = len(valid_lengths) | |
| if seq_len is None: | |
| seq_len = max(valid_lengths) | |
| if seq_len < max(valid_lengths): | |
| raise ValueError(f"seq_len {seq_len} < longest input {max(valid_lengths)}") | |
| if min(valid_lengths) < 1: | |
| raise ValueError("every input needs at least one token") | |
| phys = uniform_physical_length(seq_len) | |
| is_exact = all(v == phys for v in valid_lengths) | |
| if exact is None: | |
| exact = is_exact | |
| elif exact and not is_exact: | |
| raise ValueError(f"exact plan needs every length equal to {phys}, got {valid_lengths}") | |
| plans = self.plans(phys, exact, b) | |
| ids = torch.zeros(b, phys, dtype=torch.int32) | |
| pool = torch.zeros(1, 1, pool_rows(b), b * phys) | |
| for r, n in enumerate(valid_lengths): | |
| ids[r, :n] = torch.as_tensor(np.asarray(input_ids[r][:n]), dtype=torch.int32) | |
| pool[0, 0, r, r * phys : r * phys + n] = 1.0 | |
| used = [plans[k] for k in self.kinds()] | |
| masked = SLIDING in self.kinds() and plans[SLIDING].masked | |
| rows_masked = FULL in self.kinds() and plans[FULL].masked | |
| static_mask = masked and b == 1 and not refreshable and phys > STATIC_MASK_MIN_LEN | |
| keypad = key_padding_host(valid_lengths, phys) if masked or rows_masked else None | |
| return { | |
| "ids": ids, | |
| "pool": pool, | |
| "keypad": keypad if masked and not static_mask else None, | |
| "keyrows": keypad.expand(b, 1, TILE, phys).contiguous() if rows_masked else None, | |
| "mask_valid": valid_lengths[0] if static_mask else phys, | |
| "cu": cu_window_seqlens(valid_lengths, phys) if any(p.folded for p in used) else None, | |
| "plans": plans, | |
| "valid_lengths": valid_lengths, | |
| "seq_len": seq_len, | |
| "physical_len": phys, | |
| "exact": exact, | |
| } | |
| def host_tensors(host): | |
| out = { | |
| "ids": ttnn.from_torch(host["ids"], dtype=ttnn.uint32, layout=ttnn.ROW_MAJOR_LAYOUT), | |
| "pool": ttnn.from_torch(host["pool"], dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT), | |
| } | |
| for name in ("keypad", "keyrows"): | |
| if host[name] is not None: | |
| out[name] = ttnn.from_torch(host[name], dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT) | |
| if host["cu"] is not None: | |
| out["cu"] = ttnn.from_torch(host["cu"], dtype=ttnn.uint32, layout=ttnn.ROW_MAJOR_LAYOUT) | |
| return out | |
| def to_device_inputs(self, host): | |
| dev = self.mesh_device | |
| phys = host["physical_len"] | |
| b = len(host["valid_lengths"]) | |
| owned, keys = [], [] | |
| try: | |
| tensors = { | |
| k: ttnn.to_device(t, dev, memory_config=ttnn.DRAM_MEMORY_CONFIG) | |
| for k, t in self.host_tensors(host).items() | |
| } | |
| owned += list(tensors.values()) | |
| band = None | |
| if SLIDING in self.kinds() and host["plans"][SLIDING].masked: | |
| band, key = self._band(host["mask_valid"], phys) | |
| if key: | |
| keys.append(key) | |
| else: | |
| owned.append(band) | |
| kinds = {} | |
| for kind in self.kinds(): | |
| plan = host["plans"][kind] | |
| (cos, sin), key = self._rope_tables(kind, phys, b if plan.folded else 1) | |
| if key: | |
| keys.append(key) | |
| else: | |
| owned.extend((cos, sin)) | |
| kinds[kind] = {"plan": plan, "cos": cos, "sin": sin} | |
| except Exception: | |
| self._release(owned, keys) | |
| raise | |
| return TextInputs( | |
| input_ids=tensors["ids"], | |
| pool=tensors["pool"], | |
| keypad=tensors.get("keypad"), | |
| keyrows=tensors.get("keyrows"), | |
| cu_window=tensors.get("cu"), | |
| band=band, | |
| kinds=kinds, | |
| valid_lengths=host["valid_lengths"], | |
| seq_len=host["seq_len"], | |
| physical_len=phys, | |
| exact=host["exact"], | |
| owned=owned, | |
| cache_keys=keys, | |
| refreshable=host["keypad"] is not None or band is None, | |
| ) | |
| def prepare_inputs(self, input_ids, valid_lengths, seq_len=None, exact=None, refreshable=False): | |
| return self.to_device_inputs(self.host_inputs(input_ids, valid_lengths, seq_len, exact, refreshable)) | |
| def refresh_inputs(self, inputs, host): | |
| if not inputs.refreshable: | |
| raise ValueError("inputs hold a mask built for fixed lengths; prepare them with refreshable=True") | |
| if (host["physical_len"], len(host["valid_lengths"]), host["exact"]) != ( | |
| inputs.physical_len, | |
| len(inputs.valid_lengths), | |
| inputs.exact, | |
| ): | |
| raise ValueError("host inputs do not match the persistent input signature") | |
| dst = inputs.host_refreshable() | |
| for name, t in self.host_tensors(host).items(): | |
| ttnn.copy_host_to_device_tensor(t, dst[name]) | |
| inputs.valid_lengths = host["valid_lengths"] | |
| inputs.seq_len = host["seq_len"] | |
| def freeze_caches(self, frozen=True): | |
| for cache in (self._rope, self._masks): | |
| cache.frozen = frozen | |
| def _release(self, owned, keys): | |
| for t in owned: | |
| ttnn.deallocate(t) | |
| for key in keys: | |
| (self._rope if key[0] == "rope" else self._masks).release(key) | |
| def release(self, inputs): | |
| self._release(inputs.owned, inputs.cache_keys) | |
| inputs.owned, inputs.cache_keys = [], [] | |
| def embed(self, input_ids): | |
| b, phys = input_ids.shape | |
| x = ttnn.embedding(input_ids, self.embed_table, layout=ttnn.TILE_LAYOUT, memory_config=ttnn.DRAM_MEMORY_CONFIG) | |
| return ttnn.reshape(x, [b, 1, phys, self.hidden]) | |
| def per_layer_input(self, embeds, layer_idx): | |
| b, one, phys, d = embeds.shape | |
| kw = {} | |
| cfg = mm_config_for(self.policy.mm_config, ("ple_in", d, self.ple_dim), b * phys) | |
| if cfg is not None: | |
| kw["config"] = cfg | |
| p = ttnn.experimental.minimal_matmul( | |
| ttnn.reshape(embeds, [1, 1, b * phys, d]), | |
| self.ple_w[layer_idx], | |
| compute_kernel_config=self.ple_cfg, | |
| memory_config=ttnn.DRAM_MEMORY_CONFIG, | |
| dtype=ttnn.bfloat16, | |
| **kw, | |
| ) | |
| out = ttnn.rms_norm( | |
| p, | |
| epsilon=self.eps, | |
| weight=self.ple_norm, | |
| compute_kernel_config=self.ple_cfg, | |
| memory_config=ttnn.DRAM_MEMORY_CONFIG, | |
| ) | |
| ttnn.deallocate(p) | |
| return ttnn.reshape(out, [b, one, phys, self.ple_dim]) | |
| def attention_mask(self, inputs): | |
| if inputs.keypad is None: | |
| return inputs.band | |
| return ttnn.add(inputs.band, inputs.keypad, memory_config=ttnn.DRAM_MEMORY_CONFIG) | |
| def merge_media(self, embeds, media): | |
| merged = ttnn.add(embeds, ttnn.reshape(media, embeds.shape), memory_config=ttnn.DRAM_MEMORY_CONFIG) | |
| ttnn.deallocate(embeds) | |
| return merged | |
| def encode_hidden(self, inputs, media=None): | |
| mask = self.attention_mask(inputs) | |
| embeds = self.embed(inputs.input_ids) | |
| if media is not None: | |
| embeds = self.merge_media(embeds, media) | |
| x = embeds | |
| for layer in self.layers: | |
| k = inputs.kinds[layer.kind] | |
| plan = k["plan"] | |
| ple = self.per_layer_input(embeds, layer.layer_idx) | |
| layer_mask = None | |
| if plan.masked: | |
| layer_mask = inputs.keyrows if layer.kind == FULL else mask | |
| y = layer.forward( | |
| x, | |
| ple, | |
| layer_mask, | |
| k["cos"], | |
| k["sin"], | |
| plan, | |
| inputs.cu_window if plan.folded else None, | |
| ) | |
| ttnn.deallocate(ple) | |
| if x is not embeds: | |
| ttnn.deallocate(x) | |
| x = y | |
| ttnn.deallocate(embeds) | |
| if mask is not None and mask is not inputs.band: | |
| ttnn.deallocate(mask) | |
| return x | |
| def pooled(self, hidden, inputs): | |
| b, one, phys, d = hidden.shape | |
| normed = ttnn.rms_norm( | |
| ttnn.reshape(hidden, [1, 1, b * phys, d]), | |
| epsilon=self.eps, | |
| weight=self.final_norm, | |
| compute_kernel_config=self.head_cfg, | |
| memory_config=ttnn.DRAM_MEMORY_CONFIG, | |
| ) | |
| summed = ttnn.matmul( | |
| inputs.pool, | |
| normed, | |
| compute_kernel_config=self.head_cfg, | |
| memory_config=ttnn.DRAM_MEMORY_CONFIG, | |
| dtype=ttnn.float32, | |
| ) | |
| ttnn.deallocate(normed) | |
| out = ttnn.matmul( | |
| summed, | |
| self.proj, | |
| compute_kernel_config=self.head_cfg, | |
| memory_config=ttnn.DRAM_MEMORY_CONFIG, | |
| dtype=ttnn.float32, | |
| ) | |
| ttnn.deallocate(summed) | |
| return out | |
| def normalize(self, projected, dim): | |
| x = projected | |
| if dim != self.embed_dim: | |
| x = ttnn.slice(projected, [0, 0, 0, 0], [1, 1, projected.shape[2], dim]) | |
| sq = ttnn.multiply(x, x, dtype=ttnn.float32, memory_config=ttnn.DRAM_MEMORY_CONFIG) | |
| ss = ttnn.sum(sq, dim=-1, keepdim=True, compute_kernel_config=self.head_cfg) | |
| ttnn.deallocate(sq) | |
| ss_eps = ttnn.add(ss, L2_EPS * L2_EPS, dtype=ttnn.float32) | |
| ttnn.deallocate(ss) | |
| inv = ttnn.rsqrt(ss_eps, fast_and_approximate_mode=False) | |
| ttnn.deallocate(ss_eps) | |
| out = ttnn.multiply(x, inv, dtype=ttnn.float32, memory_config=ttnn.DRAM_MEMORY_CONFIG) | |
| ttnn.deallocate(inv) | |
| if x is not projected: | |
| ttnn.deallocate(x) | |
| return out | |
| def forward(self, inputs, dims=(768,), outputs=None, media=None): | |
| hidden = self.encode_hidden(inputs, media) | |
| projected = self.pooled(hidden, inputs) | |
| ttnn.deallocate(hidden) | |
| out = {} | |
| for d in dims: | |
| y = self.normalize(projected, d) | |
| if outputs is None: | |
| out[d] = y | |
| else: | |
| ttnn.copy(y, outputs[d]) | |
| ttnn.deallocate(y) | |
| out[d] = outputs[d] | |
| ttnn.deallocate(projected) | |
| return out | |
| def allocate_outputs(self, batch, dims=(768,)): | |
| return { | |
| d: ttnn.from_torch( | |
| torch.zeros(1, 1, pool_rows(batch), d), | |
| dtype=ttnn.float32, | |
| layout=ttnn.TILE_LAYOUT, | |
| device=self.mesh_device, | |
| memory_config=ttnn.DRAM_MEMORY_CONFIG, | |
| ) | |
| for d in dims | |
| } | |
| def to_numpy(t, batch): | |
| x = ttnn.to_torch(ttnn.get_device_tensors(t)[0]).float() | |
| return x.reshape(-1, x.shape[-1])[:batch].numpy() | |
| class TraceSlot: | |
| def __init__(self, model, batch, physical_len, exact=False, dims=(768,)): | |
| self.model = model | |
| self.batch = batch | |
| self.physical_len = physical_len | |
| self.exact = exact | |
| self.dims = tuple(dims) | |
| n = physical_len if exact else 1 | |
| self.inputs = model.prepare_inputs([[2] * n] * batch, [n] * batch, physical_len, exact=exact, refreshable=True) | |
| self.outputs = model.allocate_outputs(batch, self.dims) | |
| self.trace_id = None | |
| self.replays = 0 | |
| self.refreshes = 0 | |
| def key(self): | |
| return (self.batch, self.physical_len, self.exact) | |
| def run_eager(self): | |
| return self.model.forward(self.inputs, self.dims, outputs=self.outputs) | |
| def capture(self): | |
| mesh = self.model.mesh_device | |
| if self.trace_id is not None: | |
| raise RuntimeError(f"slot {self.key()} already captured") | |
| self.trace_id = ttnn.begin_trace_capture(mesh, cq_id=0) | |
| try: | |
| self.model.forward(self.inputs, self.dims, outputs=self.outputs) | |
| finally: | |
| ttnn.end_trace_capture(mesh, self.trace_id, cq_id=0) | |
| def refresh(self, host): | |
| self.model.refresh_inputs(self.inputs, host) | |
| self.refreshes += 1 | |
| def replay(self, blocking=False): | |
| ttnn.execute_trace(self.model.mesh_device, self.trace_id, cq_id=0, blocking=blocking) | |
| self.replays += 1 | |
| def read(self, rows, dims=None): | |
| ttnn.synchronize_device(self.model.mesh_device) | |
| return {d: self.model.to_numpy(self.outputs[d], rows) for d in (dims or self.dims)} | |
| def release(self): | |
| if self.trace_id is not None: | |
| ttnn.release_trace(self.model.mesh_device, self.trace_id) | |
| self.trace_id = None | |
| for t in self.outputs.values(): | |
| ttnn.deallocate(t) | |
| self.outputs = {} | |
| self.model.release(self.inputs) | |
| class TraceSet: | |
| def __init__(self, model, shapes, dims=(768,), extra_slots=()): | |
| self.model = model | |
| self.slots = {} | |
| self.extra = list(extra_slots) | |
| for batch, phys, exact in shapes: | |
| key = (batch, phys, exact) | |
| if key not in self.slots: | |
| self.slots[key] = TraceSlot(model, batch, phys, exact, dims) | |
| everything = list(self.slots.values()) + self.extra | |
| for slot in everything: | |
| slot.run_eager() | |
| ttnn.synchronize_device(model.mesh_device) | |
| for slot in everything: | |
| slot.capture() | |
| self.captures = len(everything) | |
| model.freeze_caches() | |
| def slot_for(self, rows, phys, exact=False): | |
| fits = [s for (b, p, e), s in self.slots.items() if p == phys and e == exact and b >= rows] | |
| return min(fits, key=lambda s: s.batch) if fits else None | |
| def run(self, slot, input_ids, valid_lengths, dims=None): | |
| rows = len(valid_lengths) | |
| ids = list(input_ids) + [[2]] * (slot.batch - rows) | |
| lengths = list(valid_lengths) + [1] * (slot.batch - rows) | |
| host = self.model.host_inputs(ids, lengths, slot.physical_len, exact=slot.exact, refreshable=True) | |
| slot.refresh(host) | |
| slot.replay() | |
| return slot.read(rows, dims) | |
| def release(self): | |
| for slot in list(self.slots.values()) + self.extra: | |
| slot.release() | |
| self.slots = {} | |
| self.extra = [] | |
| self.model.freeze_caches(False) | |
| def mm_config_for(table, key, rows): | |
| blocks = next((b for max_rows, b in table.get(key, ()) if rows <= max_rows), None) | |
| if blocks is None: | |
| return None | |
| mb, kb, nb, sh, sw, gx, gy = blocks | |
| return ttnn.MinimalMatmulConfig( | |
| M_block_size=mb, | |
| K_block_size=kb, | |
| N_block_size=nb, | |
| subblock_h=sh, | |
| subblock_w=sw, | |
| compute_with_storage_grid_size=ttnn.CoreCoord(gx, gy), | |
| ) | |
| def headfold_chunks(physical_len, batch): | |
| if batch == 1: | |
| prefs = ((64, 128), (64, 64), (32, 32)) | |
| elif batch >= 32 and physical_len <= 2048: | |
| prefs = ((128, 64), (64, 64), (32, 32)) | |
| else: | |
| prefs = ((128, 128), (64, 64), (32, 32)) | |
| return next( | |
| (q, k) for q, k in prefs if physical_len % q == 0 and physical_len % k == 0 and (physical_len // k) % 2 == 0 | |
| ) | |
| def folded_chunks(physical_len, batch, cores): | |
| q, k = sdpa_chunks(physical_len, batch, False, cores) | |
| if physical_len >= FOLDED_Q128_MIN_LEN and physical_len % 128 == 0 and (physical_len // 128) % 2 == 0: | |
| q = 128 | |
| return q, k | |
| def sdpa_chunks(physical_len, batch, windowed, cores): | |
| def fits(c): | |
| return physical_len % c == 0 and (physical_len // c) % 2 == 0 | |
| if windowed: | |
| q = 128 if physical_len >= SDPA_WINDOW_LONG_Q and fits(128) else 64 if fits(64) else 32 | |
| return q, q | |
| q = next((c for c in (128, 64, 32) if fits(c) and batch * SDPA_HEADS * physical_len // c >= cores), None) | |
| if q is None: | |
| q = next(c for c in (64, 32) if fits(c)) | |
| k = next(c for c in (128, 64, 32) if fits(c)) | |
| return q, k | |
| def pool_rows(batch): | |
| return TILE * math.ceil(batch / TILE) | |
| def uniform_physical_length(seq_len): | |
| align = 256 if seq_len >= 2048 else 128 if seq_len >= 512 else 64 | |
| return align * math.ceil(seq_len / align) | |
| def mrl_dims(embed_dim): | |
| return tuple(d for d in MRL_DIMS if d <= embed_dim) | |
| def bucket_length(n, buckets): | |
| return next((b for b in buckets if n <= b), n) | |