File size: 12,498 Bytes
714c70a 97e3989 714c70a 97e3989 714c70a e7d4c83 714c70a 97e3989 714c70a 97e3989 714c70a 97e3989 714c70a e7d4c83 97e3989 714c70a e7d4c83 714c70a 97e3989 e7d4c83 714c70a e7d4c83 714c70a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | {
"schema_version": "5.1",
"name": "embeddinggemma-2-p150",
"tt_metal_version": "0.0.0.dev0",
"arch": "blackhole",
"device_count": 1,
"producer": {
"tt_kernel_version": "0.1.0",
"created_at": "2026-10-21T15:24:58.977613+00:00",
"hostname": "qb2-120-p06t08"
},
"weights": {
"repo_id": "google/embeddinggemma-2",
"revision": "914f7f89142e33e77833254d9c9b90c3cef7303b",
"allow_patterns": null,
"ignore_patterns": null,
"repo_type": "model"
},
"auxiliary_weights": [],
"mesh": null,
"entrypoint": null,
"resources": null,
"capabilities": null,
"env": {},
"bundled": null,
"deps": null,
"container": {
"image": {
"registry": "hf",
"repository": "embeddinggemma-2-p150",
"tag": "tt-model/embeddinggemma-2-p150:0bd59b87edea",
"digest": "sha256:0bd59b87edea4b92e96656bdead06a361eac00643fecbca5a1a247c55a460fed"
},
"kind": "tt-dit-server",
"runtime": {
"app": "models.autoports.google_embeddinggemma_2.server.app:app",
"lock": "requirements.lock",
"mesh_shape_env": "EMBEDDINGGEMMA_MESH_SHAPE"
},
"serve": {
"hardware": "p150",
"mesh_device": "P150",
"port": null,
"max_model_len": null,
"max_num_seqs": null,
"block_size": null,
"server_timeout": null,
"capabilities": null,
"additional_config": {},
"args": [],
"env": {
"HF_REVISION": "914f7f89142e33e77833254d9c9b90c3cef7303b",
"TT_MESH_GRAPH_DESC_PATH": "/opt/tt-metal/tt_metal/fabric/mesh_graph_descriptors/p150_mesh_graph_descriptor.textproto",
"EMBEDDINGGEMMA_ALLOW_URLS": "0"
}
},
"serve_profiles": [
{
"hardware": null,
"mesh_device": null,
"port": null,
"max_model_len": null,
"max_num_seqs": null,
"block_size": null,
"server_timeout": null,
"capabilities": null,
"additional_config": {},
"args": [],
"env": {},
"name": "default",
"description": "OpenAI-style /v1/embeddings server only."
},
{
"hardware": null,
"mesh_device": null,
"port": null,
"max_model_len": null,
"max_num_seqs": null,
"block_size": null,
"server_timeout": null,
"capabilities": null,
"additional_config": {},
"args": [],
"env": {
"EMBEDDINGGEMMA_DEMO": "1",
"EMBEDDINGGEMMA_DEMO_CACHE": "/tensor-cache/demo-index",
"GRADIO_ANALYTICS_ENABLED": "False"
},
"name": "demo",
"description": "The /v1/embeddings server plus a Gradio demo at /demo (media, video moments, audio, code, multilingual and Matryoshka search over a bundled openly licensed library)."
}
],
"default_profile": "default",
"code_dir": "code",
"verify": [
"import os, stat; bad = [os.path.join(d, n) for d, ds, fs in os.walk('/opt/tt-metal/models') for n in ds + fs if not os.stat(os.path.join(d, n)).st_mode & stat.S_IROTH or (n in ds and not os.stat(os.path.join(d, n)).st_mode & stat.S_IXOTH)]; assert not bad, bad[:5]",
"import pathlib, models.autoports.google_embeddinggemma_2.tt.precision_policy as p; assert p.DEFAULT_PATH.is_file(), p.DEFAULT_PATH",
"import models.autoports.google_embeddinggemma_2.tt.custom_sdpa as s, models.autoports.google_embeddinggemma_2.tt.custom_geglu as g, pathlib; assert all(pathlib.Path(f).is_file() for f in (s.READER, s.WRITER, s.COMPUTE)); assert (g.KERNEL_DIR / 'compute.cpp').is_file()",
"import models.autoports.google_embeddinggemma_2.tt.embedder as e; assert e.supported_tokens() == 65536, e.supported_tokens()",
"import transformers, torch; assert transformers.__version__ == '5.12.1', transformers.__version__; assert torch.__version__ == '2.11.0+cpu', torch.__version__",
"from transformers.models.gemma4.feature_extraction_gemma4 import Gemma4AudioFeatureExtractor; from transformers.models.gemma4.image_processing_gemma4 import Gemma4ImageProcessor; import soundfile, orjson",
"import gradio, json, models.autoports.google_embeddinggemma_2.demo.ui as u, models.autoports.google_embeddinggemma_2.demo.index as i; assert gradio.__version__ == '6.30.0', gradio.__version__; m = json.loads((i.LIBRARY_DIR / 'manifest.json').read_text()); missing = [x['file'] for x in m['items'] if x.get('file') and not (i.LIBRARY_DIR / x['file']).is_file()]; assert not missing, missing[:5]"
],
"built": {
"image": "tt-model/embeddinggemma-2-p150:0bd59b87edea",
"repo": "tt-hous/embeddinggemma-2",
"tt_model_version": "0.1.0",
"created_at": "2026-10-21T15:23:55+00:00",
"tt_metal": {
"sha": "3be67656b28e3902f0366ea8bab03c41e7e5bd6c",
"describe": null,
"dirty": false,
"scm_version": "0.0.0.dev0",
"mode": "git",
"remote": "https://github.com/tenstorrent/tt-metal.git",
"branch": null,
"pushed": null
},
"code_sha256": "852ada085de3df458409d2183b853e843234bd60acff1efb98eb3c32c62dd77d",
"image_digest": "sha256:0bd59b87edea4b92e96656bdead06a361eac00643fecbca5a1a247c55a460fed"
},
"card": {
"description": "EmbeddingGemma 2, a multimodal embedding model. Text, code, images, video frames and 16 kHz audio each map to one L2-normalized 768-dimensional vector in a shared space, so a text query can find an image, a sound or a video. Matryoshka dimensions 512, 256 and 128 are the first values of the 768-dimensional vector, re-normalized on the chip.\n\nThe text tower is a bidirectional 24-layer Gemma encoder (hidden size 512, sliding and full attention, 8192-token context). Gemma 4 vision (16 layers) and audio (12 layers) towers feed soft tokens into it. Weights stay on the chip and every request shape runs as a captured TTNN trace. Two custom kernels (sliding-window SDPA and a GEGLU matmul) run through stock `ttnn.generic_op`; tt-metal itself is unmodified.\n",
"attribution": "Derived from google/embeddinggemma-2 by Google DeepMind, weights unmodified. The TTNN port is models/autoports/google_embeddinggemma_2, on stock upstream tt-metal.\n",
"prerequisites": "- 1 GB hugepages mounted at `/dev/hugepages-1G`\n- One chip of a p300 also works: the package sets the p150 mesh graph descriptor.\n",
"quickstart": "### Embedding text\n```bash\ncurl -s localhost:20000/v1/embeddings -H 'content-type: application/json' \\\n -d '{\"input\": [\"What is the capital of France?\"], \"input_type\": \"query\", \"dimensions\": 256}'\n```\n\n### Demo\n```bash\ntt serve tt-hous/embeddinggemma-2 --profile demo\n```\nOpen `http://<host>:20000/demo/`. A web page where every query is embedded live on the chip, over a bundled library of 163 openly licensed images, videos, sounds, texts in 10 languages and code snippets: media search, finding a moment in a long video, audio search, code search, multilingual search, and a side-by-side Matryoshka comparison.\n",
"architecture": null,
"status": null,
"intended_use": "Embeddings for retrieval, semantic search, clustering and classification over text, code, images, video and audio, served from one p150. Not for generating text.\n",
"out_of_scope_use": null,
"usage": "`POST /v1/embeddings` follows the OpenAI embeddings request and response shapes, with media extensions.\n\n| field | meaning | default |\n|---|---|---|\n| `input` | a string, or a list of 1 to 2048 strings and media items | required |\n| `dimensions` | 768, 512, 256 or 128 | 768 |\n| `input_type` | a sentence-transformers prompt name, for example `query`, `document`, `CodeRetrieval` | no prompt |\n| `encoding_format` | `float`, or `base64` (little-endian float32) | `float` |\n| `model` | optional; `google/embeddinggemma-2` or `embeddinggemma-2` | |\n\nA media item is an object with optional `text` and at most one media field:\n\n| field | value |\n|---|---|\n| `image` | base64 image bytes (any PIL format) or a `data:image/...;base64,` URL |\n| `audio` | base64 16 kHz WAV; clips over 30 s are cut to 30 s |\n| `video` | `{\"frames\": [<base64 image>, ...], \"fps\": 1}`; at most 32 frames are used |\n\n```bash\nIMG=$(base64 -w0 cats.jpg)\ncurl -s localhost:20000/v1/embeddings -H 'content-type: application/json' \\\n -d \"{\\\"input\\\": [{\\\"image\\\": \\\"$IMG\\\"}, {\\\"text\\\": \\\"two cats on a couch\\\"}]}\"\n```\n\n`GET /health` answers 200 once the traces are captured, with the precision policy and live counters, and 503 before that. `GET /v1/models` lists the model, its dimensions and modalities. Errors use the OpenAI error shape; an input over 8192 tokens gets 400 `context_length_exceeded`. Media URLs are refused: send media as base64.\n",
"performance": "| benchmark (768d) | metric | this package | HF fp32 | card |\n| --- | --- | ---: | ---: | ---: |\n| MTEB multilingual v2 | Mean(Task) | 59.55 | 59.51 | 61.36 |\n| MTEB eng v2 | Mean(Task) | 70.25 | 70.35 | 68.46 |\n| MTEB code v1 | Mean(Task), NDCG@10 | 56.99 | 57.04 | 78.68 |\n| MIEB lite | Mean(TaskType) | 60.93 | 60.50 | 64.64 |\n| MMEB v2 Image | Mean(Task), Hit@1 | 38.85 | 39.10 | 57.28 |\n| MMEB v2 VisDoc | Mean(Task), NDCG@5 | 70.26 | 69.39 | 67.84 |\n| MMEB v2 Video | Mean(Task), Hit@1 | 20.09 | 19.63 | 50.67 |\n| MSEB Retrieval | Mean(Task), MRR@10 | 80.20 | 80.18 | 69.54 |\n| MAEB | Mean(Task) | 58.16 | 58.28 | 49.39 |\n\n| dimensions | storage | MTEB multilingual | MTEB eng | MTEB code | MIEB lite | MMEB overall | MSEB | MAEB |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 768 | 1x | 59.55 | 70.25 | 56.99 | 60.93 | 44.19 | 80.20 | 58.16 |\n| 512 | 1/1.5 | 59.20 | 70.24 | 56.62 | 60.77 | 44.70 | 78.99 | 57.93 |\n| 256 | 1/3 | 58.61 | 69.54 | 55.72 | 59.50 | 42.37 | 76.26 | 57.84 |\n| 128 | 1/6 | 55.76 | 67.90 | 52.30 | 55.57 | 37.12 | 60.25 | 55.25 |\n\n| load, one text of 480 to 512 tokens per request | requests/s | p50 (ms) | p99 (ms) |\n| --- | ---: | ---: | ---: |\n| 1 user | 88.5 | 11.4 | 13.1 |\n| 8 users | 207.7 | 38.5 | 40.5 |\n| 32 users | 277.3 | 114.1 | 122.3 |\n\n| single request, warm | latency (ms) |\n| --- | ---: |\n| text, 128 tokens | 5.8 |\n| text, 512 tokens | 7.7 |\n| text, 8192 tokens | 80.3 |\n| image | 47.8 |\n| video, 8 frames | 138.1 |\n| audio, 30 s | 32.2 |\n\nMeasured on one Blackhole chip of a p300c board opened as a 1x1 mesh (equal to a p150, compute grid 11x10), precision policy x02. Accuracy: a fixed subset of each benchmark, scored with the same inputs, prompts and code on this package and on the Hugging Face fp32 reference on CPU; \"card\" is the full-suite score from the google/embeddinggemma-2 card and is not comparable to subset means. No benchmark mean differs from HF fp32 by more than 1.0 point at any dimension. Load: a real server with closed-loop clients, 60 s per level, 0 errors. Single request: host staging, trace replay and readback, without tokenization or media decoding.\n",
"limitations": "- Bring-up build, not a Tenstorrent release.\n- One chip per server; no multi-chip mode.\n- At most 8192 tokens per input, counting the prompt and media tokens.\n- A vector depends slightly on the device batch it ran in (cosine down to 0.98 on hard inputs). Send one input per request for repeatable vectors.\n- Against HF fp32, 8 of the evaluation texts and 37 audio clips fall below 0.995 cosine (minimum 0.959); benchmark means still match within 1.0 point.\n- Audio must be 16 kHz WAV and is cut to 30 s. Video arrives as decoded frames, at most 32. Images use the default 280-token budget.\n- Never run in fp16: the base model card warns it gives NaN or degraded vectors.\n- The first boot compiles kernels and captures 51 traces (about 6 minutes); later boots take about 30 s.\n- Validated on one chip of a p300c. Performance on a p150 with a larger compute grid was not measured.\n",
"risks": null,
"licensing": "Weights under the upstream Apache-2.0 licence (google/embeddinggemma-2). The TTNN port and the server in `code/` are Apache-2.0 (tt-metal). Demo library items carry their own CC0, public domain, CC BY, CC BY-SA, Apache-2.0 or MIT licences, listed in `code/models/autoports/google_embeddinggemma_2/demo/library/LICENSES.md`.\n",
"related": null,
"license": {
"id": "apache-2.0",
"name": null,
"link": null
},
"pipeline_tag": "feature-extraction",
"base_model": [
"google/embeddinggemma-2"
]
}
}
} |