---
library_name: transformers
base_model:
- google/embeddinggemma-2
---
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [google/embeddinggemma-2](https://huggingface.co/google/embeddinggemma-2).
Structural comparison with `google/embeddinggemma-2`:
| Feature | Source | Tiny |
|---|---:|---:|
| Input modalities | text, image, video, audio | same |
| Vocabulary rows | 262,144 | same |
| Output embedding dimension | 768 | same |
| Text layers | 24 | 12; two 5-sliding + 1-full cycles |
| Text width / local head dimension | 512 / 256 | 16 / 32 |
| Full-attention head dimension | 512 | 64 |
| Vision layers / width | 16 / 768 | 2 / 64 |
| Audio layers / width | 12 / 1,024 | 2 / 64 |
| PLE | projection-only | same |
| Stored precision | BF16 | BF16 |
The output embedding dimension intentionally remains 768; it is a source-model
interface rather than a vocabulary or hidden-width reduction target.
| File path | Size |
|------|------|
| model.safetensors | 12.0MB |
### Example usage:
- Transformers (✅ verified)
```python
import numpy as np
import torch
from PIL import Image
from transformers import AutoProcessor, EmbeddingGemma2Model
from transformers.video_utils import VideoMetadata
model_id = "tiny-random/embeddinggemma-2"
device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" else torch.float32
model = EmbeddingGemma2Model.from_pretrained(model_id, dtype=dtype).to(device).eval()
processor = AutoProcessor.from_pretrained(model_id)
image = Image.new("RGB", (64, 64), "red")
video = np.stack([np.asarray(Image.new("RGB", (64, 64), color)) for color in ("red", "green", "blue", "yellow")])
audio = np.zeros(1_600, dtype=np.float32)
text = f"{processor.image_token} {processor.video_token} {processor.audio_token}"
inputs = processor(
text=[text], images=[[image]], videos=[video], audio=[audio], return_tensors="pt",
do_sample_frames=False,
video_metadata=[VideoMetadata(total_num_frames=4, fps=2, frames_indices=[0, 1, 2, 3])],
).to(device)
with torch.inference_mode():
embeddings = model(**inputs).last_hidden_state.mean(dim=1)
print(embeddings.shape)
```
### Codes to create this repo (✅ verified):
Click to expand
```python
import json
from pathlib import Path
from shutil import copyfile
import torch
from huggingface_hub import hf_hub_download, list_repo_files
from transformers import AutoConfig, EmbeddingGemma2Model, set_seed
source_model_id = "google/embeddinggemma-2"
save_folder = "/tmp/tiny-random/embeddinggemma-2"
source_revision = "914f7f89142e33e77833254d9c9b90c3cef7303b"
set_seed(42)
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
text = raw["text_config"]
text.update(
hidden_size=16, hidden_size_per_layer_input=16, head_dim=32,
num_attention_heads=1, num_key_value_heads=1, intermediate_size=64,
num_hidden_layers=12,
layer_types=["sliding_attention"] * 5 + ["full_attention"] + ["sliding_attention"] * 5 + ["full_attention"],
per_layer_config={"5": {"head_dim": 64, "num_key_value_heads": 1}, "11": {"head_dim": 64, "num_key_value_heads": 1}},
)
raw["vision_config"].update(
hidden_size=64, intermediate_size=128, num_hidden_layers=2,
num_attention_heads=2, num_key_value_heads=2, head_dim=32, global_head_dim=32,
)
raw["audio_config"].update(
hidden_size=64, num_attention_heads=2, num_hidden_layers=2,
# Gemma4Audio flattens frequency bins after two stride-2 convolutions.
# Its projection formula requires the first channel count to remain 128
# for the source processor's 128-bin mel features.
output_proj_dims=64, subsampling_conv_channels=[128, 8],
)
(folder / "config.json").write_text(json.dumps(raw, indent=2), encoding="utf-8")
config = AutoConfig.from_pretrained(folder)
model = EmbeddingGemma2Model(config).to(torch.bfloat16)
with torch.no_grad():
for name, parameter in sorted(model.named_parameters()):
if not parameter.is_floating_point():
continue
if name.endswith("norm.weight"):
parameter.fill_(1)
else:
torch.nn.init.normal_(parameter, mean=0, std=0.02)
model.save_pretrained(folder, safe_serialization=True)
assets = {"1_Pooling/config.json", "2_Normalize/config.json", "chat_template.jinja", "config_sentence_transformers.json", "modules.json", "preprocessor_config.json", "processor_config.json", "sentence_bert_config.json", "tokenizer.json", "tokenizer.model", "tokenizer_config.json"}
for filename in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
destination = folder / filename
destination.parent.mkdir(parents=True, exist_ok=True)
copyfile(hf_hub_download(source_model_id, filename, revision=source_revision), destination)
```
Click to expand
```text
EmbeddingGemma2Model(
(vision_tower): Gemma4VisionModel(
(patch_embedder): Gemma4VisionPatchEmbedder(
(input_proj): Linear(in_features=768, out_features=64, bias=False)
)
(encoder): Gemma4VisionEncoder(
(rotary_emb): Gemma4VisionRotaryEmbedding()
(layers): ModuleList(
(0-1): 2 x Gemma4VisionEncoderLayer(
(self_attn): Gemma4VisionAttention(
(q_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(k_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(v_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(o_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(q_norm): Gemma4RMSNorm()
(k_norm): Gemma4RMSNorm()
(v_norm): Gemma4RMSNorm()
)
(mlp): Gemma4VisionMLP(
(gate_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=128, bias=False)
)
(up_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=128, bias=False)
)
(down_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=128, out_features=64, bias=False)
)
(act_fn): GELUTanh()
)
(input_layernorm): Gemma4RMSNorm()
(post_attention_layernorm): Gemma4RMSNorm()
(pre_feedforward_layernorm): Gemma4RMSNorm()
(post_feedforward_layernorm): Gemma4RMSNorm()
)
)
)
(pooler): Gemma4VisionPooler()
)
(language_model): EmbeddingGemma2TextModel(
(embed_tokens): EmbeddingGemma2TextScaledWordEmbedding(262144, 16, padding_idx=0)
(layers): ModuleList(
(0-4): 5 x EmbeddingGemma2EncoderLayer(
(self_attn): EmbeddingGemma2Attention(
(q_proj): Linear(in_features=16, out_features=32, bias=False)
(k_proj): Linear(in_features=16, out_features=32, bias=False)
(v_proj): Linear(in_features=16, out_features=32, bias=False)
(o_proj): Linear(in_features=32, out_features=16, bias=False)
(q_norm): EmbeddingGemma2RMSNorm()
(k_norm): EmbeddingGemma2RMSNorm()
(v_norm): EmbeddingGemma2RMSNorm()
)
(mlp): EmbeddingGemma2MLP(
(gate_proj): Linear(in_features=16, out_features=64, bias=False)
(up_proj): Linear(in_features=16, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=16, bias=False)
(act_fn): GELUTanh()
)
(input_layernorm): EmbeddingGemma2RMSNorm()
(post_attention_layernorm): EmbeddingGemma2RMSNorm()
(pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(ple_block): EmbeddingGemma2TextPLEBlock(
(act_fn): GELUTanh()
(per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
(per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
(post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
)
)
(5): EmbeddingGemma2EncoderLayer(
(self_attn): EmbeddingGemma2Attention(
(q_proj): Linear(in_features=16, out_features=64, bias=False)
(k_proj): Linear(in_features=16, out_features=64, bias=False)
(v_proj): Linear(in_features=16, out_features=64, bias=False)
(o_proj): Linear(in_features=64, out_features=16, bias=False)
(q_norm): EmbeddingGemma2RMSNorm()
(k_norm): EmbeddingGemma2RMSNorm()
(v_norm): EmbeddingGemma2RMSNorm()
)
(mlp): EmbeddingGemma2MLP(
(gate_proj): Linear(in_features=16, out_features=64, bias=False)
(up_proj): Linear(in_features=16, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=16, bias=False)
(act_fn): GELUTanh()
)
(input_layernorm): EmbeddingGemma2RMSNorm()
(post_attention_layernorm): EmbeddingGemma2RMSNorm()
(pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(ple_block): EmbeddingGemma2TextPLEBlock(
(act_fn): GELUTanh()
(per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
(per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
(post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
)
)
(6-10): 5 x EmbeddingGemma2EncoderLayer(
(self_attn): EmbeddingGemma2Attention(
(q_proj): Linear(in_features=16, out_features=32, bias=False)
(k_proj): Linear(in_features=16, out_features=32, bias=False)
(v_proj): Linear(in_features=16, out_features=32, bias=False)
(o_proj): Linear(in_features=32, out_features=16, bias=False)
(q_norm): EmbeddingGemma2RMSNorm()
(k_norm): EmbeddingGemma2RMSNorm()
(v_norm): EmbeddingGemma2RMSNorm()
)
(mlp): EmbeddingGemma2MLP(
(gate_proj): Linear(in_features=16, out_features=64, bias=False)
(up_proj): Linear(in_features=16, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=16, bias=False)
(act_fn): GELUTanh()
)
(input_layernorm): EmbeddingGemma2RMSNorm()
(post_attention_layernorm): EmbeddingGemma2RMSNorm()
(pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(ple_block): EmbeddingGemma2TextPLEBlock(
(act_fn): GELUTanh()
(per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
(per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
(post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
)
)
(11): EmbeddingGemma2EncoderLayer(
(self_attn): EmbeddingGemma2Attention(
(q_proj): Linear(in_features=16, out_features=64, bias=False)
(k_proj): Linear(in_features=16, out_features=64, bias=False)
(v_proj): Linear(in_features=16, out_features=64, bias=False)
(o_proj): Linear(in_features=64, out_features=16, bias=False)
(q_norm): EmbeddingGemma2RMSNorm()
(k_norm): EmbeddingGemma2RMSNorm()
(v_norm): EmbeddingGemma2RMSNorm()
)
(mlp): EmbeddingGemma2MLP(
(gate_proj): Linear(in_features=16, out_features=64, bias=False)
(up_proj): Linear(in_features=16, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=16, bias=False)
(act_fn): GELUTanh()
)
(input_layernorm): EmbeddingGemma2RMSNorm()
(post_attention_layernorm): EmbeddingGemma2RMSNorm()
(pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
(ple_block): EmbeddingGemma2TextPLEBlock(
(act_fn): GELUTanh()
(per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
(per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
(post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
)
)
)
(norm): EmbeddingGemma2RMSNorm()
(rotary_emb): EmbeddingGemma2RotaryEmbedding()
(ple): EmbeddingGemma2TextPLE(
(per_layer_model_projection): Linear(in_features=16, out_features=192, bias=False)
(per_layer_projection_norm): EmbeddingGemma2RMSNorm()
)
(embedding_projection): Linear(in_features=16, out_features=768, bias=False)
)
(audio_tower): Gemma4AudioModel(
(subsample_conv_projection): Gemma4AudioSubSampleConvProjection(
(layer0): Gemma4AudioSubSampleConvProjectionLayer(
(conv): Conv2d(1, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=False)
(act): ReLU()
)
(layer1): Gemma4AudioSubSampleConvProjectionLayer(
(conv): Conv2d(128, 8, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(norm): LayerNorm((8,), eps=1e-06, elementwise_affine=True, bias=False)
(act): ReLU()
)
(input_proj_linear): Linear(in_features=256, out_features=64, bias=False)
)
(rel_pos_enc): Gemma4AudioRelPositionalEncoding()
(layers): ModuleList(
(0-1): 2 x Gemma4AudioLayer(
(feed_forward1): Gemma4AudioFeedForward(
(ffw_layer_1): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=256, bias=False)
)
(ffw_layer_2): Gemma4ClippableLinear(
(linear): Linear(in_features=256, out_features=64, bias=False)
)
(pre_layer_norm): Gemma4RMSNorm()
(post_layer_norm): Gemma4RMSNorm()
(act_fn): SiLUActivation()
)
(feed_forward2): Gemma4AudioFeedForward(
(ffw_layer_1): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=256, bias=False)
)
(ffw_layer_2): Gemma4ClippableLinear(
(linear): Linear(in_features=256, out_features=64, bias=False)
)
(pre_layer_norm): Gemma4RMSNorm()
(post_layer_norm): Gemma4RMSNorm()
(act_fn): SiLUActivation()
)
(self_attn): Gemma4AudioAttention(
(q_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(k_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(v_proj): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(post): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(relative_k_proj): Linear(in_features=64, out_features=64, bias=False)
)
(lconv1d): Gemma4AudioLightConv1d(
(linear_start): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=128, bias=False)
)
(linear_end): Gemma4ClippableLinear(
(linear): Linear(in_features=64, out_features=64, bias=False)
)
(depthwise_conv1d): Gemma4AudioCausalConv1d(64, 64, kernel_size=(5,), stride=(1,), groups=64, bias=False)
(pre_layer_norm): Gemma4RMSNorm()
(conv_norm): Gemma4RMSNorm()
(act_fn): SiLUActivation()
)
(norm_pre_attn): Gemma4RMSNorm()
(norm_post_attn): Gemma4RMSNorm()
(norm_out): Gemma4RMSNorm()
)
)
(output_proj): Linear(in_features=64, out_features=64, bias=True)
)
(embed_vision): EmbeddingGemma2MultimodalEmbedder(
(embedding_projection): Linear(in_features=64, out_features=16, bias=False)
(embedding_pre_projection_norm): EmbeddingGemma2RMSNorm()
)
(embed_audio): EmbeddingGemma2MultimodalEmbedder(
(embedding_projection): Linear(in_features=64, out_features=16, bias=False)
(embedding_pre_projection_norm): EmbeddingGemma2RMSNorm()
)
)
```