Download code/models/autoports/google_embeddinggemma_2/tt/embedder.py from tt-hous/embeddinggemma-2: direct link, hf CLI and curl.
- Browser
- Download file 15.1 kB
-
https://huggingface.co/tt-hous/embeddinggemma-2/resolve/main/code/models/autoports/google_embeddinggemma_2/tt/embedder.py
- Command line
-
hf download hf://tt-hous/embeddinggemma-2/code/models/autoports/google_embeddinggemma_2/tt/embedder.py
-
curl -L -o embedder.py https://huggingface.co/tt-hous/embeddinggemma-2/resolve/main/code/models/autoports/google_embeddinggemma_2/tt/embedder.py
15.1 kB
| # SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| import ttnn | |
| from models.autoports.google_embeddinggemma_2.tt.model import ( | |
| EmbeddingGemma2TextModel, | |
| TraceSet, | |
| bucket_length, | |
| mrl_dims, | |
| uniform_physical_length, | |
| ) | |
| from models.autoports.google_embeddinggemma_2.tt.multimodal import ( | |
| AUDIO, | |
| IMAGE, | |
| MEDIA_TEXT_POLICY, | |
| VIDEO, | |
| VISION_KINDS, | |
| MediaShape, | |
| MediaSlot, | |
| MultimodalPipeline, | |
| item_kind, | |
| item_patches, | |
| item_tower_n, | |
| ) | |
| HF_MODEL = "google/embeddinggemma-2" | |
| HF_REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b" | |
| DEFAULT_BUCKETS = (32, 64, 128, 256, 512, 1024, 2048, 4096, 8192) | |
| DEFAULT_MAX_BATCH_TOKENS = 32768 | |
| TRACE_BATCHES = (1, 2, 4, 8, 16, 32) | |
| IMAGE_PATCHES = 2520 | |
| VIDEO_PATCHES = 1260 | |
| DEFAULT_MEDIA_SHAPES = ( | |
| MediaShape(IMAGE, 1, 512, 1, IMAGE_PATCHES), | |
| MediaShape(IMAGE, 8, 512, 8, IMAGE_PATCHES), | |
| MediaShape(IMAGE, 1, 8192, 1, IMAGE_PATCHES), | |
| MediaShape(IMAGE, 1, 8192, 32, IMAGE_PATCHES), | |
| MediaShape(VIDEO, 1, 2048, 8, VIDEO_PATCHES), | |
| MediaShape(VIDEO, 1, 8192, 32, VIDEO_PATCHES), | |
| MediaShape(AUDIO, 1, 1024, 768), | |
| MediaShape(AUDIO, 8, 1024, 768), | |
| MediaShape(AUDIO, 1, 8192, 768), | |
| ) | |
| MAX_EAGER_TOWER_IMAGES = 32 | |
| CONTRACT = Path(__file__).resolve().parents[1] / "doc" / "context_contract.json" | |
| def default_trace_shapes(buckets=DEFAULT_BUCKETS, batch_size=32, max_batch_tokens=DEFAULT_MAX_BATCH_TOKENS): | |
| shapes = [] | |
| for bucket in buckets: | |
| cap = min(batch_size, max(1, max_batch_tokens // bucket)) | |
| phys = uniform_physical_length(bucket) | |
| shapes += [(b, phys, False) for b in TRACE_BATCHES if b <= cap and (b, phys, False) not in shapes] | |
| return shapes | |
| def resolve_snapshot(model_dir): | |
| if model_dir is not None and Path(model_dir).is_dir(): | |
| return Path(model_dir) | |
| from huggingface_hub import snapshot_download | |
| return Path(snapshot_download(model_dir or HF_MODEL, revision=HF_REVISION, local_files_only=True)) | |
| def supported_tokens(): | |
| try: | |
| value = json.loads(CONTRACT.read_text()).get("current_supported_context") | |
| except (OSError, ValueError): | |
| value = None | |
| return value if isinstance(value, int) else None | |
| class TextEmbedder: | |
| def __init__( | |
| self, | |
| model, | |
| tokenizer, | |
| prompts, | |
| buckets=DEFAULT_BUCKETS, | |
| batch_size=32, | |
| max_tokens=None, | |
| max_batch_tokens=DEFAULT_MAX_BATCH_TOKENS, | |
| traced=False, | |
| trace_shapes=None, | |
| extra_slots=(), | |
| ): | |
| self.model = model | |
| self.tokenizer = tokenizer | |
| self.prompts = dict(prompts) | |
| self.buckets = tuple(buckets) if buckets else () | |
| self.batch_size = batch_size | |
| self.max_tokens = max_tokens | |
| self.max_batch_tokens = max_batch_tokens | |
| self.dims = mrl_dims(model.embed_dim) | |
| self.traces = None | |
| if traced: | |
| shapes = trace_shapes or self.default_trace_shapes() | |
| self.traces = TraceSet(model, shapes, self.dims, extra_slots=extra_slots) | |
| longest = max(phys for _, phys, _ in shapes) | |
| self.max_tokens = longest if self.max_tokens is None else min(self.max_tokens, longest) | |
| def default_trace_shapes(self): | |
| return default_trace_shapes(self.buckets, self.batch_size, self.max_batch_tokens) | |
| def prompt_text(self, prompt_name=None, prompt=None): | |
| if prompt is not None: | |
| return prompt | |
| if prompt_name is None: | |
| return "" | |
| if prompt_name not in self.prompts: | |
| raise ValueError(f"unknown prompt_name {prompt_name!r}; known: {sorted(self.prompts)}") | |
| return self.prompts[prompt_name] | |
| def tokenize(self, texts, prompt_name=None, prompt=None): | |
| p = self.prompt_text(prompt_name, prompt) | |
| ids = self.tokenizer([p + t for t in texts], add_special_tokens=True)["input_ids"] | |
| if self.max_tokens is not None: | |
| too_long = [i for i, x in enumerate(ids) if len(x) > self.max_tokens] | |
| if too_long: | |
| raise ValueError(f"inputs {too_long[:8]} exceed the supported context of {self.max_tokens} tokens") | |
| return ids | |
| def seq_len_for(self, lengths): | |
| n = max(lengths) | |
| return bucket_length(n, self.buckets) if self.buckets else n | |
| def batches(self, id_lists, batch_size): | |
| order = sorted(range(len(id_lists)), key=lambda i: -len(id_lists[i])) | |
| start = 0 | |
| while start < len(order): | |
| rows = self.seq_len_for([len(id_lists[order[start]])]) | |
| n = batch_size | |
| if self.buckets and rows > self.buckets[-1]: | |
| n = 1 | |
| elif self.max_batch_tokens: | |
| n = max(1, min(batch_size, self.max_batch_tokens // rows)) | |
| yield order[start : start + n] | |
| start += n | |
| def run_batch(self, id_lists, dims, seq_len=None): | |
| lengths = [len(x) for x in id_lists] | |
| seq_len = seq_len or self.seq_len_for(lengths) | |
| if self.traces is not None: | |
| slot = self.traces.slot_for(len(id_lists), uniform_physical_length(seq_len)) | |
| if slot is None: | |
| raise ValueError(f"no trace slot for {len(id_lists)} rows at {seq_len} tokens") | |
| return self.traces.run(slot, id_lists, lengths, dims) | |
| inputs = self.model.prepare_inputs(id_lists, lengths, seq_len) | |
| try: | |
| out = self.model.forward(inputs, dims=dims) | |
| result = {d: self.model.to_numpy(t, len(id_lists)) for d, t in out.items()} | |
| for t in out.values(): | |
| ttnn.deallocate(t) | |
| finally: | |
| self.model.release(inputs) | |
| return result | |
| def encode_ids(self, id_lists, dims=(768,), batch_size=None): | |
| batch_size = batch_size or self.batch_size | |
| for d in dims: | |
| if d not in self.dims: | |
| raise ValueError(f"dim {d} not in {self.dims}") | |
| out = {d: np.zeros((len(id_lists), d), dtype=np.float32) for d in dims} | |
| for idx in self.batches(id_lists, batch_size): | |
| res = self.run_batch([id_lists[i] for i in idx], dims) | |
| for d in dims: | |
| out[d][idx] = res[d] | |
| return out | |
| def encode(self, texts, prompt_name=None, prompt=None, dim=768, batch_size=None): | |
| single = isinstance(texts, str) | |
| texts = [texts] if single else list(texts) | |
| if not texts: | |
| return np.zeros((0, dim), dtype=np.float32) | |
| ids = self.tokenize(texts, prompt_name, prompt) | |
| emb = self.encode_ids(ids, dims=(dim,), batch_size=batch_size)[dim] | |
| return emb[0] if single else emb | |
| def release(self): | |
| if self.traces is not None: | |
| self.traces.release() | |
| self.traces = None | |
| def encode_query(self, texts, **kwargs): | |
| return self.encode(texts, prompt_name="query", **kwargs) | |
| def encode_document(self, texts, **kwargs): | |
| return self.encode(texts, prompt_name="document", **kwargs) | |
| def build_embedder(model_dir, mesh_device, precision_policy=None, **kwargs): | |
| from transformers import AutoTokenizer | |
| snapshot = resolve_snapshot(model_dir) | |
| st_cfg = json.loads((snapshot / "config_sentence_transformers.json").read_text()) | |
| tokenizer = AutoTokenizer.from_pretrained(str(snapshot)) | |
| model_kwargs = { | |
| k: kwargs.pop(k) for k in ("policy", "head_policy", "layer_indices", "layer_policies") if k in kwargs | |
| } | |
| model = EmbeddingGemma2TextModel.from_pretrained(snapshot, mesh_device, precision_policy, **model_kwargs) | |
| kwargs.setdefault("max_tokens", supported_tokens()) | |
| return TextEmbedder(model, tokenizer, st_cfg["prompts"], **kwargs) | |
| def media_item(processed): | |
| kinds = [k for k in (IMAGE, VIDEO, AUDIO) if k in processed] | |
| if len(kinds) > 1: | |
| raise ValueError(f"one input mixes {kinds}; one media kind per input is supported") | |
| out = {"input_ids": processed["input_ids"]} | |
| if kinds: | |
| out.update({k: v for k, v in processed[kinds[0]].items() if torch.is_tensor(v)}) | |
| return out | |
| class MultimodalEmbedder(TextEmbedder): | |
| def __init__(self, model, tokenizer, prompts, pipeline, processor, media_shapes=None, traced=False, **kwargs): | |
| self.pipe = pipeline | |
| self.processor = processor | |
| self.media_slots = {} | |
| if traced: | |
| for shape in DEFAULT_MEDIA_SHAPES if media_shapes is None else media_shapes: | |
| self.media_slots[shape] = MediaSlot(pipeline, shape, mrl_dims(model.embed_dim)) | |
| super().__init__( | |
| model, tokenizer, prompts, traced=traced, extra_slots=list(self.media_slots.values()), **kwargs | |
| ) | |
| def media_slot_for(self, kind, items): | |
| need = self._need(kind, items) | |
| fits = [s for shape, s in self.media_slots.items() if shape.fits(kind, *need)] | |
| return min(fits, key=lambda s: (s.shape.physical_len, s.shape.batch, s.shape.tower_n)) if fits else None | |
| def _need(self, kind, items): | |
| n = max(len(it["input_ids"]) for it in items) | |
| phys = uniform_physical_length(self.seq_len_for([n])) | |
| if kind in VISION_KINDS: | |
| tower = sum(item_tower_n(it, kind) for it in items) | |
| else: | |
| tower = max(item_tower_n(it, kind) for it in items) | |
| return len(items), phys, tower, item_patches(items[0]) | |
| def _eager_ok(self, kind, items): | |
| rows, phys, tower, _ = self._need(kind, items) | |
| if rows > self.batch_size or (rows > 1 and rows * phys > self.max_batch_tokens): | |
| return False | |
| return kind == AUDIO or tower <= MAX_EAGER_TOWER_IMAGES or rows == 1 | |
| def _fits(self, kind, items): | |
| if self.traces is not None: | |
| return self.media_slot_for(kind, items) is not None | |
| return self._eager_ok(kind, items) | |
| def batch_cap(self, kind, patches, remaining): | |
| if self.traces is None: | |
| return self.batch_size | |
| sizes = sorted({s.batch for s in self.media_slots if s.kind == kind and s.patches == patches}) | |
| if not sizes: | |
| raise ValueError(f"no traced slot for {kind} with {patches} patches") | |
| return max([b for b in sizes if b <= remaining], default=sizes[0]) | |
| def media_batches(self, kind, idx, processed): | |
| order = sorted(idx, key=lambda i: -len(processed[i]["input_ids"])) | |
| patches = item_patches(processed[order[0]]) | |
| batches, pos = [], 0 | |
| while pos < len(order): | |
| cap = self.batch_cap(kind, patches, len(order) - pos) | |
| cur = [] | |
| while pos < len(order) and len(cur) < cap and self._fits(kind, [processed[j] for j in cur + [order[pos]]]): | |
| cur.append(order[pos]) | |
| pos += 1 | |
| if not cur: | |
| raise ValueError( | |
| f"no traced slot fits {kind} input {order[pos]}; build the embedder with media_shapes covering it" | |
| ) | |
| batches.append(cur) | |
| return batches | |
| def run_media(self, kind, items, dims): | |
| if self.traces is None: | |
| return self.pipe.run_eager(kind, items, dims, self.buckets) | |
| return self.media_slot_for(kind, items).run(items, dims) | |
| def process(self, inputs, prompt_name=None, prompt=None, processing_kwargs=None): | |
| p = self.prompt_text(prompt_name, prompt) | |
| out = [] | |
| for x in inputs: | |
| if isinstance(x, str): | |
| out.append({"input_ids": self.tokenize([x], prompt=p)[0]}) | |
| else: | |
| out.append(media_item(self.processor(x, prompt=p, processing_kwargs=processing_kwargs))) | |
| if self.traces is None and self.max_tokens is not None: | |
| for i, it in enumerate(out): | |
| if len(it["input_ids"]) > self.max_tokens: | |
| raise ValueError(f"input {i} has {len(it['input_ids'])} tokens, above {self.max_tokens}") | |
| return out | |
| def encode_processed(self, processed, dims=(768,), batch_size=None): | |
| out = {d: np.zeros((len(processed), d), dtype=np.float32) for d in dims} | |
| groups = {} | |
| for i, it in enumerate(processed): | |
| kind = item_kind(it, self.pipe.tokens) | |
| groups.setdefault((kind, item_patches(it) if kind else 0), []).append(i) | |
| for (kind, _), idx in groups.items(): | |
| if kind is None: | |
| res = self.encode_ids([processed[i]["input_ids"] for i in idx], dims=dims, batch_size=batch_size) | |
| for d in dims: | |
| out[d][idx] = res[d] | |
| continue | |
| batches = self.media_batches(kind, idx, processed) | |
| if self.traces is None: | |
| for b in batches: | |
| res = self.run_media(kind, [processed[i] for i in b], dims) | |
| for d in dims: | |
| out[d][b] = res[d] | |
| continue | |
| plan = [(b, self.media_slot_for(kind, [processed[i] for i in b])) for b in batches] | |
| staged = plan[0][1].stage([processed[i] for i in plan[0][0]]) | |
| for k, (b, slot) in enumerate(plan): | |
| slot.load(staged) | |
| slot.replay() | |
| if k + 1 < len(plan): | |
| nb, nslot = plan[k + 1] | |
| staged = nslot.stage([processed[i] for i in nb]) | |
| res = slot.read(len(b), dims) | |
| for d in dims: | |
| out[d][b] = res[d] | |
| return out | |
| def encode(self, inputs, prompt_name=None, prompt=None, dim=768, batch_size=None, processing_kwargs=None): | |
| single = not isinstance(inputs, (list, tuple)) | |
| inputs = [inputs] if single else list(inputs) | |
| if not inputs: | |
| return np.zeros((0, dim), dtype=np.float32) | |
| processed = self.process(inputs, prompt_name, prompt, processing_kwargs) | |
| emb = self.encode_processed(processed, dims=(dim,), batch_size=batch_size)[dim] | |
| return emb[0] if single else emb | |
| def build_multimodal_embedder( | |
| model_dir, mesh_device, modalities=(IMAGE, VIDEO, AUDIO), precision_policy=None, **kwargs | |
| ): | |
| from transformers import AutoTokenizer | |
| from models.autoports.google_embeddinggemma_2.tt.mm_inputs import MultimodalProcessor | |
| snapshot = resolve_snapshot(model_dir) | |
| st_cfg = json.loads((snapshot / "config_sentence_transformers.json").read_text()) | |
| tokenizer = AutoTokenizer.from_pretrained(str(snapshot)) | |
| model_kwargs = { | |
| k: kwargs.pop(k) for k in ("policy", "head_policy", "layer_indices", "layer_policies") if k in kwargs | |
| } | |
| tower_kwargs = {k: kwargs.pop(k) for k in ("vision_policy", "audio_policy") if k in kwargs} | |
| model = EmbeddingGemma2TextModel.from_pretrained( | |
| snapshot, mesh_device, precision_policy, text_base=MEDIA_TEXT_POLICY, **model_kwargs | |
| ) | |
| pipe = MultimodalPipeline.from_pretrained(snapshot, model, modalities, **tower_kwargs) | |
| kwargs.setdefault("max_tokens", supported_tokens()) | |
| return MultimodalEmbedder(model, tokenizer, st_cfg["prompts"], pipe, MultimodalProcessor(snapshot), **kwargs) | |