# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. # SPDX-License-Identifier: Apache-2.0 import json from pathlib import Path import numpy as np import torch import ttnn from models.autoports.google_embeddinggemma_2.tt.model import ( EmbeddingGemma2TextModel, TraceSet, bucket_length, mrl_dims, uniform_physical_length, ) from models.autoports.google_embeddinggemma_2.tt.multimodal import ( AUDIO, IMAGE, MEDIA_TEXT_POLICY, VIDEO, VISION_KINDS, MediaShape, MediaSlot, MultimodalPipeline, item_kind, item_patches, item_tower_n, ) HF_MODEL = "google/embeddinggemma-2" HF_REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b" DEFAULT_BUCKETS = (32, 64, 128, 256, 512, 1024, 2048, 4096, 8192) DEFAULT_MAX_BATCH_TOKENS = 32768 TRACE_BATCHES = (1, 2, 4, 8, 16, 32) IMAGE_PATCHES = 2520 VIDEO_PATCHES = 1260 DEFAULT_MEDIA_SHAPES = ( MediaShape(IMAGE, 1, 512, 1, IMAGE_PATCHES), MediaShape(IMAGE, 8, 512, 8, IMAGE_PATCHES), MediaShape(IMAGE, 1, 8192, 1, IMAGE_PATCHES), MediaShape(IMAGE, 1, 8192, 32, IMAGE_PATCHES), MediaShape(VIDEO, 1, 2048, 8, VIDEO_PATCHES), MediaShape(VIDEO, 1, 8192, 32, VIDEO_PATCHES), MediaShape(AUDIO, 1, 1024, 768), MediaShape(AUDIO, 8, 1024, 768), MediaShape(AUDIO, 1, 8192, 768), ) MAX_EAGER_TOWER_IMAGES = 32 CONTRACT = Path(__file__).resolve().parents[1] / "doc" / "context_contract.json" def default_trace_shapes(buckets=DEFAULT_BUCKETS, batch_size=32, max_batch_tokens=DEFAULT_MAX_BATCH_TOKENS): shapes = [] for bucket in buckets: cap = min(batch_size, max(1, max_batch_tokens // bucket)) phys = uniform_physical_length(bucket) shapes += [(b, phys, False) for b in TRACE_BATCHES if b <= cap and (b, phys, False) not in shapes] return shapes def resolve_snapshot(model_dir): if model_dir is not None and Path(model_dir).is_dir(): return Path(model_dir) from huggingface_hub import snapshot_download return Path(snapshot_download(model_dir or HF_MODEL, revision=HF_REVISION, local_files_only=True)) def supported_tokens(): try: value = json.loads(CONTRACT.read_text()).get("current_supported_context") except (OSError, ValueError): value = None return value if isinstance(value, int) else None class TextEmbedder: def __init__( self, model, tokenizer, prompts, buckets=DEFAULT_BUCKETS, batch_size=32, max_tokens=None, max_batch_tokens=DEFAULT_MAX_BATCH_TOKENS, traced=False, trace_shapes=None, extra_slots=(), ): self.model = model self.tokenizer = tokenizer self.prompts = dict(prompts) self.buckets = tuple(buckets) if buckets else () self.batch_size = batch_size self.max_tokens = max_tokens self.max_batch_tokens = max_batch_tokens self.dims = mrl_dims(model.embed_dim) self.traces = None if traced: shapes = trace_shapes or self.default_trace_shapes() self.traces = TraceSet(model, shapes, self.dims, extra_slots=extra_slots) longest = max(phys for _, phys, _ in shapes) self.max_tokens = longest if self.max_tokens is None else min(self.max_tokens, longest) def default_trace_shapes(self): return default_trace_shapes(self.buckets, self.batch_size, self.max_batch_tokens) def prompt_text(self, prompt_name=None, prompt=None): if prompt is not None: return prompt if prompt_name is None: return "" if prompt_name not in self.prompts: raise ValueError(f"unknown prompt_name {prompt_name!r}; known: {sorted(self.prompts)}") return self.prompts[prompt_name] def tokenize(self, texts, prompt_name=None, prompt=None): p = self.prompt_text(prompt_name, prompt) ids = self.tokenizer([p + t for t in texts], add_special_tokens=True)["input_ids"] if self.max_tokens is not None: too_long = [i for i, x in enumerate(ids) if len(x) > self.max_tokens] if too_long: raise ValueError(f"inputs {too_long[:8]} exceed the supported context of {self.max_tokens} tokens") return ids def seq_len_for(self, lengths): n = max(lengths) return bucket_length(n, self.buckets) if self.buckets else n def batches(self, id_lists, batch_size): order = sorted(range(len(id_lists)), key=lambda i: -len(id_lists[i])) start = 0 while start < len(order): rows = self.seq_len_for([len(id_lists[order[start]])]) n = batch_size if self.buckets and rows > self.buckets[-1]: n = 1 elif self.max_batch_tokens: n = max(1, min(batch_size, self.max_batch_tokens // rows)) yield order[start : start + n] start += n def run_batch(self, id_lists, dims, seq_len=None): lengths = [len(x) for x in id_lists] seq_len = seq_len or self.seq_len_for(lengths) if self.traces is not None: slot = self.traces.slot_for(len(id_lists), uniform_physical_length(seq_len)) if slot is None: raise ValueError(f"no trace slot for {len(id_lists)} rows at {seq_len} tokens") return self.traces.run(slot, id_lists, lengths, dims) inputs = self.model.prepare_inputs(id_lists, lengths, seq_len) try: out = self.model.forward(inputs, dims=dims) result = {d: self.model.to_numpy(t, len(id_lists)) for d, t in out.items()} for t in out.values(): ttnn.deallocate(t) finally: self.model.release(inputs) return result def encode_ids(self, id_lists, dims=(768,), batch_size=None): batch_size = batch_size or self.batch_size for d in dims: if d not in self.dims: raise ValueError(f"dim {d} not in {self.dims}") out = {d: np.zeros((len(id_lists), d), dtype=np.float32) for d in dims} for idx in self.batches(id_lists, batch_size): res = self.run_batch([id_lists[i] for i in idx], dims) for d in dims: out[d][idx] = res[d] return out def encode(self, texts, prompt_name=None, prompt=None, dim=768, batch_size=None): single = isinstance(texts, str) texts = [texts] if single else list(texts) if not texts: return np.zeros((0, dim), dtype=np.float32) ids = self.tokenize(texts, prompt_name, prompt) emb = self.encode_ids(ids, dims=(dim,), batch_size=batch_size)[dim] return emb[0] if single else emb def release(self): if self.traces is not None: self.traces.release() self.traces = None def encode_query(self, texts, **kwargs): return self.encode(texts, prompt_name="query", **kwargs) def encode_document(self, texts, **kwargs): return self.encode(texts, prompt_name="document", **kwargs) def build_embedder(model_dir, mesh_device, precision_policy=None, **kwargs): from transformers import AutoTokenizer snapshot = resolve_snapshot(model_dir) st_cfg = json.loads((snapshot / "config_sentence_transformers.json").read_text()) tokenizer = AutoTokenizer.from_pretrained(str(snapshot)) model_kwargs = { k: kwargs.pop(k) for k in ("policy", "head_policy", "layer_indices", "layer_policies") if k in kwargs } model = EmbeddingGemma2TextModel.from_pretrained(snapshot, mesh_device, precision_policy, **model_kwargs) kwargs.setdefault("max_tokens", supported_tokens()) return TextEmbedder(model, tokenizer, st_cfg["prompts"], **kwargs) def media_item(processed): kinds = [k for k in (IMAGE, VIDEO, AUDIO) if k in processed] if len(kinds) > 1: raise ValueError(f"one input mixes {kinds}; one media kind per input is supported") out = {"input_ids": processed["input_ids"]} if kinds: out.update({k: v for k, v in processed[kinds[0]].items() if torch.is_tensor(v)}) return out class MultimodalEmbedder(TextEmbedder): def __init__(self, model, tokenizer, prompts, pipeline, processor, media_shapes=None, traced=False, **kwargs): self.pipe = pipeline self.processor = processor self.media_slots = {} if traced: for shape in DEFAULT_MEDIA_SHAPES if media_shapes is None else media_shapes: self.media_slots[shape] = MediaSlot(pipeline, shape, mrl_dims(model.embed_dim)) super().__init__( model, tokenizer, prompts, traced=traced, extra_slots=list(self.media_slots.values()), **kwargs ) def media_slot_for(self, kind, items): need = self._need(kind, items) fits = [s for shape, s in self.media_slots.items() if shape.fits(kind, *need)] return min(fits, key=lambda s: (s.shape.physical_len, s.shape.batch, s.shape.tower_n)) if fits else None def _need(self, kind, items): n = max(len(it["input_ids"]) for it in items) phys = uniform_physical_length(self.seq_len_for([n])) if kind in VISION_KINDS: tower = sum(item_tower_n(it, kind) for it in items) else: tower = max(item_tower_n(it, kind) for it in items) return len(items), phys, tower, item_patches(items[0]) def _eager_ok(self, kind, items): rows, phys, tower, _ = self._need(kind, items) if rows > self.batch_size or (rows > 1 and rows * phys > self.max_batch_tokens): return False return kind == AUDIO or tower <= MAX_EAGER_TOWER_IMAGES or rows == 1 def _fits(self, kind, items): if self.traces is not None: return self.media_slot_for(kind, items) is not None return self._eager_ok(kind, items) def batch_cap(self, kind, patches, remaining): if self.traces is None: return self.batch_size sizes = sorted({s.batch for s in self.media_slots if s.kind == kind and s.patches == patches}) if not sizes: raise ValueError(f"no traced slot for {kind} with {patches} patches") return max([b for b in sizes if b <= remaining], default=sizes[0]) def media_batches(self, kind, idx, processed): order = sorted(idx, key=lambda i: -len(processed[i]["input_ids"])) patches = item_patches(processed[order[0]]) batches, pos = [], 0 while pos < len(order): cap = self.batch_cap(kind, patches, len(order) - pos) cur = [] while pos < len(order) and len(cur) < cap and self._fits(kind, [processed[j] for j in cur + [order[pos]]]): cur.append(order[pos]) pos += 1 if not cur: raise ValueError( f"no traced slot fits {kind} input {order[pos]}; build the embedder with media_shapes covering it" ) batches.append(cur) return batches def run_media(self, kind, items, dims): if self.traces is None: return self.pipe.run_eager(kind, items, dims, self.buckets) return self.media_slot_for(kind, items).run(items, dims) def process(self, inputs, prompt_name=None, prompt=None, processing_kwargs=None): p = self.prompt_text(prompt_name, prompt) out = [] for x in inputs: if isinstance(x, str): out.append({"input_ids": self.tokenize([x], prompt=p)[0]}) else: out.append(media_item(self.processor(x, prompt=p, processing_kwargs=processing_kwargs))) if self.traces is None and self.max_tokens is not None: for i, it in enumerate(out): if len(it["input_ids"]) > self.max_tokens: raise ValueError(f"input {i} has {len(it['input_ids'])} tokens, above {self.max_tokens}") return out def encode_processed(self, processed, dims=(768,), batch_size=None): out = {d: np.zeros((len(processed), d), dtype=np.float32) for d in dims} groups = {} for i, it in enumerate(processed): kind = item_kind(it, self.pipe.tokens) groups.setdefault((kind, item_patches(it) if kind else 0), []).append(i) for (kind, _), idx in groups.items(): if kind is None: res = self.encode_ids([processed[i]["input_ids"] for i in idx], dims=dims, batch_size=batch_size) for d in dims: out[d][idx] = res[d] continue batches = self.media_batches(kind, idx, processed) if self.traces is None: for b in batches: res = self.run_media(kind, [processed[i] for i in b], dims) for d in dims: out[d][b] = res[d] continue plan = [(b, self.media_slot_for(kind, [processed[i] for i in b])) for b in batches] staged = plan[0][1].stage([processed[i] for i in plan[0][0]]) for k, (b, slot) in enumerate(plan): slot.load(staged) slot.replay() if k + 1 < len(plan): nb, nslot = plan[k + 1] staged = nslot.stage([processed[i] for i in nb]) res = slot.read(len(b), dims) for d in dims: out[d][b] = res[d] return out def encode(self, inputs, prompt_name=None, prompt=None, dim=768, batch_size=None, processing_kwargs=None): single = not isinstance(inputs, (list, tuple)) inputs = [inputs] if single else list(inputs) if not inputs: return np.zeros((0, dim), dtype=np.float32) processed = self.process(inputs, prompt_name, prompt, processing_kwargs) emb = self.encode_processed(processed, dims=(dim,), batch_size=batch_size)[dim] return emb[0] if single else emb def build_multimodal_embedder( model_dir, mesh_device, modalities=(IMAGE, VIDEO, AUDIO), precision_policy=None, **kwargs ): from transformers import AutoTokenizer from models.autoports.google_embeddinggemma_2.tt.mm_inputs import MultimodalProcessor snapshot = resolve_snapshot(model_dir) st_cfg = json.loads((snapshot / "config_sentence_transformers.json").read_text()) tokenizer = AutoTokenizer.from_pretrained(str(snapshot)) model_kwargs = { k: kwargs.pop(k) for k in ("policy", "head_policy", "layer_indices", "layer_policies") if k in kwargs } tower_kwargs = {k: kwargs.pop(k) for k in ("vision_policy", "audio_policy") if k in kwargs} model = EmbeddingGemma2TextModel.from_pretrained( snapshot, mesh_device, precision_policy, text_base=MEDIA_TEXT_POLICY, **model_kwargs ) pipe = MultimodalPipeline.from_pretrained(snapshot, model, modalities, **tower_kwargs) kwargs.setdefault("max_tokens", supported_tokens()) return MultimodalEmbedder(model, tokenizer, st_cfg["prompts"], pipe, MultimodalProcessor(snapshot), **kwargs)