tt-hous's picture
tt-model push embeddinggemma-2-p150 (container)
714c70a verified
Raw History Blame Contribute Delete
15.1 kB
# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc.
# SPDX-License-Identifier: Apache-2.0
import json
from pathlib import Path
import numpy as np
import torch
import ttnn
from models.autoports.google_embeddinggemma_2.tt.model import (
EmbeddingGemma2TextModel,
TraceSet,
bucket_length,
mrl_dims,
uniform_physical_length,
)
from models.autoports.google_embeddinggemma_2.tt.multimodal import (
AUDIO,
IMAGE,
MEDIA_TEXT_POLICY,
VIDEO,
VISION_KINDS,
MediaShape,
MediaSlot,
MultimodalPipeline,
item_kind,
item_patches,
item_tower_n,
)
HF_MODEL = "google/embeddinggemma-2"
HF_REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b"
DEFAULT_BUCKETS = (32, 64, 128, 256, 512, 1024, 2048, 4096, 8192)
DEFAULT_MAX_BATCH_TOKENS = 32768
TRACE_BATCHES = (1, 2, 4, 8, 16, 32)
IMAGE_PATCHES = 2520
VIDEO_PATCHES = 1260
DEFAULT_MEDIA_SHAPES = (
MediaShape(IMAGE, 1, 512, 1, IMAGE_PATCHES),
MediaShape(IMAGE, 8, 512, 8, IMAGE_PATCHES),
MediaShape(IMAGE, 1, 8192, 1, IMAGE_PATCHES),
MediaShape(IMAGE, 1, 8192, 32, IMAGE_PATCHES),
MediaShape(VIDEO, 1, 2048, 8, VIDEO_PATCHES),
MediaShape(VIDEO, 1, 8192, 32, VIDEO_PATCHES),
MediaShape(AUDIO, 1, 1024, 768),
MediaShape(AUDIO, 8, 1024, 768),
MediaShape(AUDIO, 1, 8192, 768),
)
MAX_EAGER_TOWER_IMAGES = 32
CONTRACT = Path(__file__).resolve().parents[1] / "doc" / "context_contract.json"
def default_trace_shapes(buckets=DEFAULT_BUCKETS, batch_size=32, max_batch_tokens=DEFAULT_MAX_BATCH_TOKENS):
shapes = []
for bucket in buckets:
cap = min(batch_size, max(1, max_batch_tokens // bucket))
phys = uniform_physical_length(bucket)
shapes += [(b, phys, False) for b in TRACE_BATCHES if b <= cap and (b, phys, False) not in shapes]
return shapes
def resolve_snapshot(model_dir):
if model_dir is not None and Path(model_dir).is_dir():
return Path(model_dir)
from huggingface_hub import snapshot_download
return Path(snapshot_download(model_dir or HF_MODEL, revision=HF_REVISION, local_files_only=True))
def supported_tokens():
try:
value = json.loads(CONTRACT.read_text()).get("current_supported_context")
except (OSError, ValueError):
value = None
return value if isinstance(value, int) else None
class TextEmbedder:
def __init__(
self,
model,
tokenizer,
prompts,
buckets=DEFAULT_BUCKETS,
batch_size=32,
max_tokens=None,
max_batch_tokens=DEFAULT_MAX_BATCH_TOKENS,
traced=False,
trace_shapes=None,
extra_slots=(),
):
self.model = model
self.tokenizer = tokenizer
self.prompts = dict(prompts)
self.buckets = tuple(buckets) if buckets else ()
self.batch_size = batch_size
self.max_tokens = max_tokens
self.max_batch_tokens = max_batch_tokens
self.dims = mrl_dims(model.embed_dim)
self.traces = None
if traced:
shapes = trace_shapes or self.default_trace_shapes()
self.traces = TraceSet(model, shapes, self.dims, extra_slots=extra_slots)
longest = max(phys for _, phys, _ in shapes)
self.max_tokens = longest if self.max_tokens is None else min(self.max_tokens, longest)
def default_trace_shapes(self):
return default_trace_shapes(self.buckets, self.batch_size, self.max_batch_tokens)
def prompt_text(self, prompt_name=None, prompt=None):
if prompt is not None:
return prompt
if prompt_name is None:
return ""
if prompt_name not in self.prompts:
raise ValueError(f"unknown prompt_name {prompt_name!r}; known: {sorted(self.prompts)}")
return self.prompts[prompt_name]
def tokenize(self, texts, prompt_name=None, prompt=None):
p = self.prompt_text(prompt_name, prompt)
ids = self.tokenizer([p + t for t in texts], add_special_tokens=True)["input_ids"]
if self.max_tokens is not None:
too_long = [i for i, x in enumerate(ids) if len(x) > self.max_tokens]
if too_long:
raise ValueError(f"inputs {too_long[:8]} exceed the supported context of {self.max_tokens} tokens")
return ids
def seq_len_for(self, lengths):
n = max(lengths)
return bucket_length(n, self.buckets) if self.buckets else n
def batches(self, id_lists, batch_size):
order = sorted(range(len(id_lists)), key=lambda i: -len(id_lists[i]))
start = 0
while start < len(order):
rows = self.seq_len_for([len(id_lists[order[start]])])
n = batch_size
if self.buckets and rows > self.buckets[-1]:
n = 1
elif self.max_batch_tokens:
n = max(1, min(batch_size, self.max_batch_tokens // rows))
yield order[start : start + n]
start += n
def run_batch(self, id_lists, dims, seq_len=None):
lengths = [len(x) for x in id_lists]
seq_len = seq_len or self.seq_len_for(lengths)
if self.traces is not None:
slot = self.traces.slot_for(len(id_lists), uniform_physical_length(seq_len))
if slot is None:
raise ValueError(f"no trace slot for {len(id_lists)} rows at {seq_len} tokens")
return self.traces.run(slot, id_lists, lengths, dims)
inputs = self.model.prepare_inputs(id_lists, lengths, seq_len)
try:
out = self.model.forward(inputs, dims=dims)
result = {d: self.model.to_numpy(t, len(id_lists)) for d, t in out.items()}
for t in out.values():
ttnn.deallocate(t)
finally:
self.model.release(inputs)
return result
def encode_ids(self, id_lists, dims=(768,), batch_size=None):
batch_size = batch_size or self.batch_size
for d in dims:
if d not in self.dims:
raise ValueError(f"dim {d} not in {self.dims}")
out = {d: np.zeros((len(id_lists), d), dtype=np.float32) for d in dims}
for idx in self.batches(id_lists, batch_size):
res = self.run_batch([id_lists[i] for i in idx], dims)
for d in dims:
out[d][idx] = res[d]
return out
def encode(self, texts, prompt_name=None, prompt=None, dim=768, batch_size=None):
single = isinstance(texts, str)
texts = [texts] if single else list(texts)
if not texts:
return np.zeros((0, dim), dtype=np.float32)
ids = self.tokenize(texts, prompt_name, prompt)
emb = self.encode_ids(ids, dims=(dim,), batch_size=batch_size)[dim]
return emb[0] if single else emb
def release(self):
if self.traces is not None:
self.traces.release()
self.traces = None
def encode_query(self, texts, **kwargs):
return self.encode(texts, prompt_name="query", **kwargs)
def encode_document(self, texts, **kwargs):
return self.encode(texts, prompt_name="document", **kwargs)
def build_embedder(model_dir, mesh_device, precision_policy=None, **kwargs):
from transformers import AutoTokenizer
snapshot = resolve_snapshot(model_dir)
st_cfg = json.loads((snapshot / "config_sentence_transformers.json").read_text())
tokenizer = AutoTokenizer.from_pretrained(str(snapshot))
model_kwargs = {
k: kwargs.pop(k) for k in ("policy", "head_policy", "layer_indices", "layer_policies") if k in kwargs
}
model = EmbeddingGemma2TextModel.from_pretrained(snapshot, mesh_device, precision_policy, **model_kwargs)
kwargs.setdefault("max_tokens", supported_tokens())
return TextEmbedder(model, tokenizer, st_cfg["prompts"], **kwargs)
def media_item(processed):
kinds = [k for k in (IMAGE, VIDEO, AUDIO) if k in processed]
if len(kinds) > 1:
raise ValueError(f"one input mixes {kinds}; one media kind per input is supported")
out = {"input_ids": processed["input_ids"]}
if kinds:
out.update({k: v for k, v in processed[kinds[0]].items() if torch.is_tensor(v)})
return out
class MultimodalEmbedder(TextEmbedder):
def __init__(self, model, tokenizer, prompts, pipeline, processor, media_shapes=None, traced=False, **kwargs):
self.pipe = pipeline
self.processor = processor
self.media_slots = {}
if traced:
for shape in DEFAULT_MEDIA_SHAPES if media_shapes is None else media_shapes:
self.media_slots[shape] = MediaSlot(pipeline, shape, mrl_dims(model.embed_dim))
super().__init__(
model, tokenizer, prompts, traced=traced, extra_slots=list(self.media_slots.values()), **kwargs
)
def media_slot_for(self, kind, items):
need = self._need(kind, items)
fits = [s for shape, s in self.media_slots.items() if shape.fits(kind, *need)]
return min(fits, key=lambda s: (s.shape.physical_len, s.shape.batch, s.shape.tower_n)) if fits else None
def _need(self, kind, items):
n = max(len(it["input_ids"]) for it in items)
phys = uniform_physical_length(self.seq_len_for([n]))
if kind in VISION_KINDS:
tower = sum(item_tower_n(it, kind) for it in items)
else:
tower = max(item_tower_n(it, kind) for it in items)
return len(items), phys, tower, item_patches(items[0])
def _eager_ok(self, kind, items):
rows, phys, tower, _ = self._need(kind, items)
if rows > self.batch_size or (rows > 1 and rows * phys > self.max_batch_tokens):
return False
return kind == AUDIO or tower <= MAX_EAGER_TOWER_IMAGES or rows == 1
def _fits(self, kind, items):
if self.traces is not None:
return self.media_slot_for(kind, items) is not None
return self._eager_ok(kind, items)
def batch_cap(self, kind, patches, remaining):
if self.traces is None:
return self.batch_size
sizes = sorted({s.batch for s in self.media_slots if s.kind == kind and s.patches == patches})
if not sizes:
raise ValueError(f"no traced slot for {kind} with {patches} patches")
return max([b for b in sizes if b <= remaining], default=sizes[0])
def media_batches(self, kind, idx, processed):
order = sorted(idx, key=lambda i: -len(processed[i]["input_ids"]))
patches = item_patches(processed[order[0]])
batches, pos = [], 0
while pos < len(order):
cap = self.batch_cap(kind, patches, len(order) - pos)
cur = []
while pos < len(order) and len(cur) < cap and self._fits(kind, [processed[j] for j in cur + [order[pos]]]):
cur.append(order[pos])
pos += 1
if not cur:
raise ValueError(
f"no traced slot fits {kind} input {order[pos]}; build the embedder with media_shapes covering it"
)
batches.append(cur)
return batches
def run_media(self, kind, items, dims):
if self.traces is None:
return self.pipe.run_eager(kind, items, dims, self.buckets)
return self.media_slot_for(kind, items).run(items, dims)
def process(self, inputs, prompt_name=None, prompt=None, processing_kwargs=None):
p = self.prompt_text(prompt_name, prompt)
out = []
for x in inputs:
if isinstance(x, str):
out.append({"input_ids": self.tokenize([x], prompt=p)[0]})
else:
out.append(media_item(self.processor(x, prompt=p, processing_kwargs=processing_kwargs)))
if self.traces is None and self.max_tokens is not None:
for i, it in enumerate(out):
if len(it["input_ids"]) > self.max_tokens:
raise ValueError(f"input {i} has {len(it['input_ids'])} tokens, above {self.max_tokens}")
return out
def encode_processed(self, processed, dims=(768,), batch_size=None):
out = {d: np.zeros((len(processed), d), dtype=np.float32) for d in dims}
groups = {}
for i, it in enumerate(processed):
kind = item_kind(it, self.pipe.tokens)
groups.setdefault((kind, item_patches(it) if kind else 0), []).append(i)
for (kind, _), idx in groups.items():
if kind is None:
res = self.encode_ids([processed[i]["input_ids"] for i in idx], dims=dims, batch_size=batch_size)
for d in dims:
out[d][idx] = res[d]
continue
batches = self.media_batches(kind, idx, processed)
if self.traces is None:
for b in batches:
res = self.run_media(kind, [processed[i] for i in b], dims)
for d in dims:
out[d][b] = res[d]
continue
plan = [(b, self.media_slot_for(kind, [processed[i] for i in b])) for b in batches]
staged = plan[0][1].stage([processed[i] for i in plan[0][0]])
for k, (b, slot) in enumerate(plan):
slot.load(staged)
slot.replay()
if k + 1 < len(plan):
nb, nslot = plan[k + 1]
staged = nslot.stage([processed[i] for i in nb])
res = slot.read(len(b), dims)
for d in dims:
out[d][b] = res[d]
return out
def encode(self, inputs, prompt_name=None, prompt=None, dim=768, batch_size=None, processing_kwargs=None):
single = not isinstance(inputs, (list, tuple))
inputs = [inputs] if single else list(inputs)
if not inputs:
return np.zeros((0, dim), dtype=np.float32)
processed = self.process(inputs, prompt_name, prompt, processing_kwargs)
emb = self.encode_processed(processed, dims=(dim,), batch_size=batch_size)[dim]
return emb[0] if single else emb
def build_multimodal_embedder(
model_dir, mesh_device, modalities=(IMAGE, VIDEO, AUDIO), precision_policy=None, **kwargs
):
from transformers import AutoTokenizer
from models.autoports.google_embeddinggemma_2.tt.mm_inputs import MultimodalProcessor
snapshot = resolve_snapshot(model_dir)
st_cfg = json.loads((snapshot / "config_sentence_transformers.json").read_text())
tokenizer = AutoTokenizer.from_pretrained(str(snapshot))
model_kwargs = {
k: kwargs.pop(k) for k in ("policy", "head_policy", "layer_indices", "layer_policies") if k in kwargs
}
tower_kwargs = {k: kwargs.pop(k) for k in ("vision_policy", "audio_policy") if k in kwargs}
model = EmbeddingGemma2TextModel.from_pretrained(
snapshot, mesh_device, precision_policy, text_base=MEDIA_TEXT_POLICY, **model_kwargs
)
pipe = MultimodalPipeline.from_pretrained(snapshot, model, modalities, **tower_kwargs)
kwargs.setdefault("max_tokens", supported_tokens())
return MultimodalEmbedder(model, tokenizer, st_cfg["prompts"], pipe, MultimodalProcessor(snapshot), **kwargs)