Hugging Face's logo

tiny-random
/
gemma-3n

Image-Text-to-Text
Transformers
Safetensors
gemma3n
conversational
Model card Files
xet
 Community
gemma-3n / README.md
yujiepan's picture
yujiepan
Update README.md
6e0bf15 verified
preview code
|
Raw
History Blame Contribute Delete
25.3 kB
---
library_name: transformers
pipeline_tag: image-text-to-text
inference: true
widget:
 - text: Hello!
 example_title: Hello world
 group: Python
base_model:
- google/gemma-3n-E4B-it
---
This tiny model is for debugging. It is randomly initialized with the config adapted from [google/gemma-3n-E4B-it](https://huggingface.co/google/gemma-3n-E4B-it).
| Model ID | Notes |
| ------------------------------------------------------------------------------------- | ------------------------------- |
| [tiny-random/gemma-3n](https://huggingface.co/tiny-random/gemma-3n) | hidden size is 32 |
| [tiny-random/gemma-3n-dim4](https://huggingface.co/tiny-random/gemma-3n-dim4) | hidden size is 4; potentially not supported in paged attention kernels|
### Example usage:
```python
import torch
from transformers import pipeline
model_id = "tiny-random/gemma-3n"
pipe = pipeline(
 task="image-text-to-text",
 model=model_id,
 device=0,
 torch_dtype=torch.bfloat16
)
# temporary patch for audio tower
from accelerate.hooks import ModelHook, add_hook_to_module
class EnsureDtype(ModelHook):
 def pre_forward(self, module, *args, **kwargs):
 args = list(args)
 args[0] = args[0].to(module.dtype)
 return super().pre_forward(module, *args, **kwargs)
add_hook_to_module(pipe.model.audio_tower, EnsureDtype())
messages = [
 {
 "role": "system",
 "content": [
 {"type": "text", "text": "You are a helpful assistant."}
 ]
 },
 {
 "role": "user",
 "content": [
 {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
 # audio is buggy for now: bf16 x fp32
 {"type": "audio", "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Audio/glass-breaking-151256.mp3"},
 {"type": "text", "text": "Which image is cuter?"},
 ]
 },
]
result = pipe(messages, min_new_tokens=512, max_new_tokens=512, do_sample=True)
print(result)
```
### Codes to create this repo:
```python
import json
from pathlib import Path
import torch
import accelerate
from huggingface_hub import file_exists, hf_hub_download
from timm.models.mobilenetv5 import decode_arch_def
from transformers import (
 AutoConfig,
 AutoModelForCausalLM,
 AutoProcessor,
 AutoTokenizer,
 Gemma3nForConditionalGeneration,
 GenerationConfig,
 set_seed,
)
source_model_id = "google/gemma-3n-E4B-it"
save_folder = "/tmp/tiny-random/gemma-3n"
processor = AutoProcessor.from_pretrained(source_model_id)
processor.save_pretrained(save_folder)
with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
 config_json = json.load(f)
config_json['audio_config'].update({
 "conf_num_attention_heads": 2,
 "conf_num_hidden_layers": 2,
 "hidden_size": 64,
})
config_json['text_config'].update({
 "activation_sparsity_pattern": [0.95, 0.95, 0.0, 0.0],
 "head_dim": 32, # required by vllm
 "hidden_size": 32,
 "hidden_size_per_layer_input": 2,
 "intermediate_size": 64,
 "laurel_rank": 8,
 "layer_types": ['sliding_attention', 'full_attention', 'sliding_attention', 'full_attention'],
 "num_attention_heads": 1,
 "num_hidden_layers": 4,
 "num_key_value_heads": 1,
 "num_kv_shared_layers": 2,
 "sliding_window": 512,
})
block_args = decode_arch_def(
 [
 # Stage 0: 128x128 in
 [
 'er_r1_k3_s2_e4_c32',
 'er_r1_k3_s1_e4_c32',
 ],
 # Stage 1: 256x256 in
 [
 'uir_r1_a3_k5_s2_e6_c32',
 'uir_r1_a5_k0_s1_e4_c32',
 'uir_r1_a3_k0_s1_e4_c32',
 ],
 # Stage 2: 640x640 in
 [
 "uir_r1_a5_k5_s2_e6_c32",
 "uir_r1_a0_k0_s1_e1_c32",
 "mqa_r1_k3_h2_v2_s1_d64_c32",
 "uir_r1_a0_k0_s1_e2_c32",
 ],
 # Stage 3: 1280x1280 in
 [
 "uir_r1_a5_k5_s2_e6_c32",
 "mqa_r1_k3_h2_s1_d64_c32",
 "uir_r1_a0_k0_s1_e2_c32",
 ],
 ]
)
config_json['vision_config'].update({
 "hidden_size": 2048, # hard-coded in timm
 "model_args": {
 "block_args": block_args,
 }
})
config_json['tie_word_embeddings'] = True
with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
 json.dump(config_json, f, indent=2)
config = AutoConfig.from_pretrained(
 save_folder,
 trust_remote_code=True,
)
print(config)
torch.set_default_dtype(torch.bfloat16)
model = Gemma3nForConditionalGeneration(config)
torch.set_default_dtype(torch.float32)
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
 model.generation_config = GenerationConfig.from_pretrained(
 source_model_id, trust_remote_code=True,
 )
set_seed(42)
model = model.cpu()
all_numels = 0
for name, p in sorted(model.named_parameters()):
 all_numels += p.numel()
with torch.no_grad():
 for name, p in sorted(model.named_parameters()):
 torch.nn.init.normal_(p, 0, 0.2)
 print(name, p.shape, f'{p.numel() / all_numels * 100: .4f}%')
model.save_pretrained(save_folder)
```
### Printing the model:
```text
Gemma3nForConditionalGeneration(
 (model): Gemma3nModel(
 (vision_tower): TimmWrapperModel(
 (timm_model): MobileNetV5Encoder(
 (conv_stem): ConvNormAct(
 (conv): Conv2dSame(3, 64, kernel_size=(3, 3), stride=(2, 2), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (blocks): Sequential(
 (0): Sequential(
 (0): EdgeResidual(
 (conv_exp): Conv2dSame(64, 256, kernel_size=(3, 3), stride=(2, 2), bias=False)
 (bn1): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 (aa): Identity()
 (se): Identity()
 (conv_pwl): Conv2d(256, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn2): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 (drop_path): Identity()
 )
 (1): EdgeResidual(
 (conv_exp): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
 (bn1): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 (aa): Identity()
 (se): Identity()
 (conv_pwl): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn2): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 (drop_path): Identity()
 )
 )
 (1): Sequential(
 (0): UniversalInvertedResidual(
 (dw_start): ConvNormAct(
 (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): ConvNormAct(
 (conv): Conv2dSame(192, 192, kernel_size=(5, 5), stride=(2, 2), groups=192, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 (1): UniversalInvertedResidual(
 (dw_start): ConvNormAct(
 (conv): Conv2d(32, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=32, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): Identity()
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 (2): UniversalInvertedResidual(
 (dw_start): ConvNormAct(
 (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): Identity()
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 )
 (2): Sequential(
 (0): UniversalInvertedResidual(
 (dw_start): ConvNormAct(
 (conv): Conv2d(32, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=32, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): ConvNormAct(
 (conv): Conv2dSame(192, 192, kernel_size=(5, 5), stride=(2, 2), groups=192, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 (1): UniversalInvertedResidual(
 (dw_start): Identity()
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): Identity()
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 (2): MobileAttention(
 (norm): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 (attn): MultiQueryAttention2d(
 (query): Sequential(
 (proj): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
 )
 (key): Sequential(
 (down_conv): Conv2dSame(32, 32, kernel_size=(3, 3), stride=(2, 2), groups=32, bias=False)
 (norm): RmsNorm2d()
 (proj): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
 )
 (value): Sequential(
 (down_conv): Conv2dSame(32, 32, kernel_size=(3, 3), stride=(2, 2), groups=32, bias=False)
 (norm): RmsNorm2d()
 (proj): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
 )
 (attn_drop): Dropout(p=0.0, inplace=False)
 (output): Sequential(
 (proj): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (drop): Dropout(p=0.0, inplace=False)
 )
 )
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 (3): UniversalInvertedResidual(
 (dw_start): Identity()
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): Identity()
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(64, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 )
 (3): Sequential(
 (0): UniversalInvertedResidual(
 (dw_start): ConvNormAct(
 (conv): Conv2d(32, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=32, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): ConvNormAct(
 (conv): Conv2dSame(192, 192, kernel_size=(5, 5), stride=(2, 2), groups=192, bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 (1): MobileAttention(
 (norm): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 (attn): MultiQueryAttention2d(
 (query): Sequential(
 (proj): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
 )
 (key): Sequential(
 (proj): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
 )
 (value): Sequential(
 (proj): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
 )
 (attn_drop): Dropout(p=0.0, inplace=False)
 (output): Sequential(
 (proj): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (drop): Dropout(p=0.0, inplace=False)
 )
 )
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 (2): UniversalInvertedResidual(
 (dw_start): Identity()
 (pw_exp): ConvNormAct(
 (conv): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): Identity()
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(64, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): LayerScale2d()
 (drop_path): Identity()
 )
 )
 )
 (msfa): MobileNetV5MultiScaleFusionAdapter(
 (ffn): UniversalInvertedResidual(
 (dw_start): Identity()
 (pw_exp): ConvNormAct(
 (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): GELU(approximate='none')
 )
 )
 (dw_mid): Identity()
 (se): Identity()
 (pw_proj): ConvNormAct(
 (conv): Conv2d(128, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
 (bn): RmsNormAct2d(
 (drop): Identity()
 (act): Identity()
 )
 )
 (dw_end): Identity()
 (layer_scale): Identity()
 (drop_path): Identity()
 )
 (norm): RmsNorm2d()
 )
 )
 )
 (language_model): Gemma3nTextModel(
 (embed_tokens): Gemma3nTextScaledWordEmbedding(262400, 32, padding_idx=0)
 (layers): ModuleList(
 (0-3): 4 x Gemma3nTextDecoderLayer(
 (self_attn): Gemma3nTextAttention(
 (q_proj): Linear(in_features=32, out_features=32, bias=False)
 (k_proj): Linear(in_features=32, out_features=32, bias=False)
 (v_proj): Linear(in_features=32, out_features=32, bias=False)
 (o_proj): Linear(in_features=32, out_features=32, bias=False)
 (q_norm): Gemma3nRMSNorm((32,), eps=1e-06)
 (k_norm): Gemma3nRMSNorm((32,), eps=1e-06)
 (v_norm): Gemma3nRMSNorm((), eps=1e-06)
 )
 (mlp): Gemma3nTextMLP(
 (gate_proj): Linear(in_features=32, out_features=64, bias=False)
 (up_proj): Linear(in_features=32, out_features=64, bias=False)
 (down_proj): Linear(in_features=64, out_features=32, bias=False)
 (act_fn): PytorchGELUTanh()
 )
 (input_layernorm): Gemma3nRMSNorm((32,), eps=1e-06)
 (post_attention_layernorm): Gemma3nRMSNorm((32,), eps=1e-06)
 (pre_feedforward_layernorm): Gemma3nRMSNorm((32,), eps=1e-06)
 (post_feedforward_layernorm): Gemma3nRMSNorm((32,), eps=1e-06)
 (act_fn): PytorchGELUTanh()
 (altup): Gemma3nTextAltUp(
 (correction_coefs): Linear(in_features=4, out_features=4, bias=False)
 (prediction_coefs): Linear(in_features=4, out_features=16, bias=False)
 (modality_router): Linear(in_features=32, out_features=4, bias=False)
 (router_norm): Gemma3nRMSNorm((32,), eps=1e-06)
 )
 (laurel): Gemma3nTextLaurelBlock(
 (linear_left): Linear(in_features=32, out_features=8, bias=False)
 (linear_right): Linear(in_features=8, out_features=32, bias=False)
 (post_laurel_norm): Gemma3nRMSNorm((32,), eps=1e-06)
 )
 (per_layer_input_gate): Linear(in_features=32, out_features=2, bias=False)
 (per_layer_projection): Linear(in_features=2, out_features=32, bias=False)
 (post_per_layer_input_norm): Gemma3nRMSNorm((32,), eps=1e-06)
 )
 )
 (norm): Gemma3nRMSNorm((32,), eps=1e-06)
 (rotary_emb): Gemma3nTextRotaryEmbedding()
 (rotary_emb_local): Gemma3nTextRotaryEmbedding()
 (embed_tokens_per_layer): Gemma3nTextScaledWordEmbedding(262144, 8, padding_idx=0)
 (per_layer_model_projection): Linear(in_features=32, out_features=8, bias=False)
 (per_layer_projection_norm): Gemma3nRMSNorm((2,), eps=1e-06)
 (altup_projections): ModuleList(
 (0-2): 3 x Linear(in_features=32, out_features=32, bias=False)
 )
 (altup_unembed_projections): ModuleList(
 (0-2): 3 x Linear(in_features=32, out_features=32, bias=False)
 )
 )
 (audio_tower): Gemma3nAudioEncoder(
 (subsample_conv_projection): Gemma3nAudioSubSampleConvProjection(
 (conv_0): Gemma3nAudioSSCPConvBlock(
 (conv): Conv2d(1, 128, kernel_size=(3, 3), stride=(2, 2), bias=False)
 (norm): Gemma3nAudioCumulativeGroupNorm()
 (activation): ReLU()
 )
 (conv_1): Gemma3nAudioSSCPConvBlock(
 (conv): Conv2d(128, 32, kernel_size=(3, 3), stride=(2, 2), bias=False)
 (norm): Gemma3nAudioCumulativeGroupNorm()
 (activation): ReLU()
 )
 (input_proj_linear): Linear(in_features=1024, out_features=64, bias=False)
 )
 (conformer): ModuleList(
 (0-1): 2 x Gemma3nAudioConformerBlock(
 (ffw_layer_start): Gemma3nAudioConformerFeedForward(
 (pre_layer_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 (ffw_layer_1): Linear(in_features=64, out_features=256, bias=False)
 (ffw_layer_2): Linear(in_features=256, out_features=64, bias=False)
 (post_layer_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 )
 (attention): Gemma3nAudioConformerAttention(
 (pre_attn_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 (attn): Gemma3nAudioAttention(
 (relative_position_embedding): Gemma3nAudioRelativePositionEmbedding(
 (pos_proj): Linear(in_features=64, out_features=64, bias=False)
 )
 (q_proj): Linear(in_features=64, out_features=64, bias=False)
 (k_proj): Linear(in_features=64, out_features=64, bias=False)
 (v_proj): Linear(in_features=64, out_features=64, bias=False)
 )
 (post): Linear(in_features=64, out_features=64, bias=False)
 (post_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 )
 (lconv1d): Gemma3nAudioConformerLightConv1d(
 (pre_layer_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 (linear_start): Linear(in_features=64, out_features=128, bias=False)
 (depthwise_conv1d): Conv1d(64, 64, kernel_size=(5,), stride=(1,), groups=64, bias=False)
 (conv_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 (linear_end): Linear(in_features=64, out_features=64, bias=False)
 )
 (ffw_layer_end): Gemma3nAudioConformerFeedForward(
 (pre_layer_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 (ffw_layer_1): Linear(in_features=64, out_features=256, bias=False)
 (ffw_layer_2): Linear(in_features=256, out_features=64, bias=False)
 (post_layer_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 )
 (norm): Gemma3nRMSNorm((64,), eps=1e-06)
 )
 )
 )
 (embed_vision): Gemma3nMultimodalEmbedder(
 (embedding): Embedding(128, 2048)
 (hard_embedding_norm): Gemma3nRMSNorm((2048,), eps=1e-06)
 (soft_embedding_norm): Gemma3nRMSNorm((2048,), eps=1e-06)
 (embedding_projection): Linear(in_features=2048, out_features=32, bias=False)
 (embedding_post_projection_norm): Gemma3nRMSNorm((), eps=1e-06)
 )
 (embed_audio): Gemma3nMultimodalEmbedder(
 (embedding): Embedding(128, 64)
 (hard_embedding_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 (soft_embedding_norm): Gemma3nRMSNorm((64,), eps=1e-06)
 (embedding_projection): Linear(in_features=64, out_features=32, bias=False)
 (embedding_post_projection_norm): Gemma3nRMSNorm((), eps=1e-06)
 )
 )
 (lm_head): Linear(in_features=32, out_features=262400, bias=False)
)
```