jandonov commited on
Commit
2d432da
·
verified ·
1 Parent(s): e6c3566

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +4 -177
config.json CHANGED
@@ -1,178 +1,5 @@
1
  {
2
- "architectures": [
3
- "EmbeddingGemma2Model"
4
- ],
5
- "audio_config": {
6
- "_name_or_path": "",
7
- "architectures": null,
8
- "attention_chunk_size": 12,
9
- "attention_context_left": 13,
10
- "attention_context_right": 0,
11
- "attention_invalid_logits_value": -1000000000.0,
12
- "attention_logit_cap": 50.0,
13
- "chunk_size_feed_forward": 0,
14
- "conv_kernel_size": 5,
15
- "dtype": "bfloat16",
16
- "gradient_clipping": 10000000000.0,
17
- "hidden_act": "silu",
18
- "hidden_size": 1024,
19
- "id2label": {
20
- "0": "LABEL_0",
21
- "1": "LABEL_1"
22
- },
23
- "initializer_range": 0.02,
24
- "is_encoder_decoder": false,
25
- "label2id": {
26
- "LABEL_0": 0,
27
- "LABEL_1": 1
28
- },
29
- "model_type": "gemma4_audio",
30
- "num_attention_heads": 8,
31
- "num_hidden_layers": 12,
32
- "output_attentions": false,
33
- "output_hidden_states": false,
34
- "output_proj_dims": 1536,
35
- "problem_type": null,
36
- "residual_weight": 0.5,
37
- "return_dict": true,
38
- "rms_norm_eps": 1e-06,
39
- "subsampling_conv_channels": [
40
- 128,
41
- 32
42
- ],
43
- "use_clipped_linears": true
44
- },
45
- "audio_token_id": 258881,
46
- "boa_token_id": 256000,
47
- "boi_token_id": 255999,
48
- "dtype": "bfloat16",
49
- "eoa_token_index": 258883,
50
- "eoi_token_id": 258882,
51
- "image_token_id": 258880,
52
- "initializer_range": 0.02,
53
- "model_type": "embedding_gemma2",
54
- "text_config": {
55
- "attention_bias": false,
56
- "attention_dropout": 0.0,
57
- "bos_token_id": 2,
58
- "dtype": "bfloat16",
59
- "embedding_dim": 768,
60
- "eos_token_id": 1,
61
- "head_dim": 256,
62
- "hidden_activation": "gelu_pytorch_tanh",
63
- "hidden_size": 512,
64
- "hidden_size_per_layer_input": 512,
65
- "initializer_range": 0.02,
66
- "intermediate_size": 2048,
67
- "layer_types": [
68
- "sliding_attention",
69
- "sliding_attention",
70
- "sliding_attention",
71
- "sliding_attention",
72
- "sliding_attention",
73
- "full_attention",
74
- "sliding_attention",
75
- "sliding_attention",
76
- "sliding_attention",
77
- "sliding_attention",
78
- "sliding_attention",
79
- "full_attention",
80
- "sliding_attention",
81
- "sliding_attention",
82
- "sliding_attention",
83
- "sliding_attention",
84
- "sliding_attention",
85
- "full_attention",
86
- "sliding_attention",
87
- "sliding_attention",
88
- "sliding_attention",
89
- "sliding_attention",
90
- "sliding_attention",
91
- "full_attention"
92
- ],
93
- "max_position_embeddings": 262144,
94
- "model_type": "embedding_gemma2_text",
95
- "num_attention_heads": 4,
96
- "num_hidden_layers": 24,
97
- "num_key_value_heads": 2,
98
- "pad_token_id": 0,
99
- "per_layer_config": {
100
- "05": {
101
- "head_dim": 512,
102
- "num_key_value_heads": 1
103
- },
104
- "11": {
105
- "head_dim": 512,
106
- "num_key_value_heads": 1
107
- },
108
- "17": {
109
- "head_dim": 512,
110
- "num_key_value_heads": 1
111
- },
112
- "23": {
113
- "head_dim": 512,
114
- "num_key_value_heads": 1
115
- }
116
- },
117
- "rms_norm_eps": 1e-06,
118
- "rope_parameters": {
119
- "full_attention": {
120
- "partial_rotary_factor": 0.25,
121
- "rope_theta": 1000000.0,
122
- "rope_type": "proportional"
123
- },
124
- "sliding_attention": {
125
- "rope_theta": 10000.0,
126
- "rope_type": "default"
127
- }
128
- },
129
- "sliding_window": 512,
130
- "vocab_size": 262144
131
- },
132
- "transformers_version": "5.18.0.dev0",
133
- "video_token_id": 258884,
134
- "vision_config": {
135
- "_name_or_path": "",
136
- "architectures": null,
137
- "attention_bias": false,
138
- "attention_dropout": 0.0,
139
- "chunk_size_feed_forward": 0,
140
- "default_output_length": 280,
141
- "dtype": "bfloat16",
142
- "global_head_dim": 64,
143
- "head_dim": 64,
144
- "hidden_activation": "gelu_pytorch_tanh",
145
- "hidden_size": 768,
146
- "id2label": {
147
- "0": "LABEL_0",
148
- "1": "LABEL_1"
149
- },
150
- "initializer_range": 0.02,
151
- "intermediate_size": 3072,
152
- "is_encoder_decoder": false,
153
- "label2id": {
154
- "LABEL_0": 0,
155
- "LABEL_1": 1
156
- },
157
- "max_position_embeddings": 131072,
158
- "model_type": "gemma4_vision",
159
- "num_attention_heads": 12,
160
- "num_hidden_layers": 16,
161
- "num_key_value_heads": 12,
162
- "output_attentions": false,
163
- "output_hidden_states": false,
164
- "patch_size": 16,
165
- "pooling_kernel_size": 3,
166
- "position_embedding_size": 10240,
167
- "problem_type": null,
168
- "return_dict": true,
169
- "rms_norm_eps": 1e-06,
170
- "rope_parameters": {
171
- "rope_theta": 100.0,
172
- "rope_type": "axial"
173
- },
174
- "standardize": false,
175
- "use_clipped_linears": false
176
- },
177
- "vision_soft_tokens_per_image": 280
178
- }
 
1
  {
2
+ "embedding_dimension": 768,
3
+ "pooling_mode": "mean",
4
+ "include_prompt": true
5
+ }