Hugging Face's logo

keras
/
modern_bert_large_en

KerasHub
Model card Files
xet
 Community
laxmareddyp commited on
Commit
38ebaff
·
verified ·
1 Parent(s): 2947e8b

Upload folder using huggingface_hub

Browse files
Files changed (10) hide show
  1. README.md +22 -0
  2. assets/tokenizer/merges.txt +0 -0
  3. assets/tokenizer/vocabulary.json +0 -0
  4. config.json +27 -0
  5. metadata.json +10 -0
  6. model.weights.h5 +3 -0
  7. preprocessor.json +162 -0
  8. task.json +197 -0
  9. task.weights.h5 +3 -0
  10. tokenizer.json +138 -0
README.md ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: keras-hub
3
+ ---
4
+ This is a [`ModernBert` model](https://keras.io/api/keras_hub/models/modern_bert) uploaded using the KerasHub library and can be used with JAX, TensorFlow, and PyTorch backends.
5
+ This model is related to a `MaskedLM` task.
6
+
7
+ Model config:
8
+ * **name:** modern_bert_backbone
9
+ * **trainable:** True
10
+ * **dtype:** {'module': 'keras', 'class_name': 'DTypePolicy', 'config': {'name': 'float32'}, 'registered_name': None}
11
+ * **vocabulary_size:** 50368
12
+ * **hidden_dim:** 1024
13
+ * **intermediate_dim:** 2624
14
+ * **num_layers:** 28
15
+ * **num_heads:** 16
16
+ * **local_attention_window:** 128
17
+ * **global_attn_every_n_layers:** 3
18
+ * **rotary_max_wavelength:** 160000.0
19
+ * **local_rotary_max_wavelength:** 10000.0
20
+ * **layer_norm_epsilon:** 1e-05
21
+
22
+ This model card has been generated automatically and should be completed by the model author. See [Model Cards documentation](https://huggingface.co/docs/hub/model-cards) for more information.
assets/tokenizer/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
assets/tokenizer/vocabulary.json ADDED
The diff for this file is too large to render. See raw diff
 
config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "module": "keras_hub.src.models.modern_bert.modern_bert_backbone",
3
+ "class_name": "ModernBertBackbone",
4
+ "config": {
5
+ "name": "modern_bert_backbone",
6
+ "trainable": true,
7
+ "dtype": {
8
+ "module": "keras",
9
+ "class_name": "DTypePolicy",
10
+ "config": {
11
+ "name": "float32"
12
+ },
13
+ "registered_name": null
14
+ },
15
+ "vocabulary_size": 50368,
16
+ "hidden_dim": 1024,
17
+ "intermediate_dim": 2624,
18
+ "num_layers": 28,
19
+ "num_heads": 16,
20
+ "local_attention_window": 128,
21
+ "global_attn_every_n_layers": 3,
22
+ "rotary_max_wavelength": 160000.0,
23
+ "local_rotary_max_wavelength": 10000.0,
24
+ "layer_norm_epsilon": 1e-05
25
+ },
26
+ "registered_name": "keras_hub>ModernBertBackbone"
27
+ }
metadata.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "keras_version": "3.15.1",
3
+ "keras_hub_version": "0.33.0.dev0",
4
+ "parameter_count": 394781696,
5
+ "date_saved": "2026-10-08@04:35:14",
6
+ "tasks": [
7
+ "MaskedLM",
8
+ "TextClassifier"
9
+ ]
10
+ }
model.weights.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1585ae15567f21609f98372b54efe5702fd9efc426c2a49270d0f36c99caa3d
3
+ size 1579785904
preprocessor.json ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "module": "keras_hub.src.models.modern_bert.modern_bert_masked_lm_preprocessor",
3
+ "class_name": "ModernBertMaskedLMPreprocessor",
4
+ "config": {
5
+ "name": "modern_bert_masked_lm_preprocessor",
6
+ "trainable": true,
7
+ "dtype": {
8
+ "module": "keras",
9
+ "class_name": "DTypePolicy",
10
+ "config": {
11
+ "name": "float32"
12
+ },
13
+ "registered_name": null
14
+ },
15
+ "tokenizer": {
16
+ "module": "keras_hub.src.models.modern_bert.modern_bert_tokenizer",
17
+ "class_name": "ModernBertTokenizer",
18
+ "config": {
19
+ "name": "modern_bert_tokenizer",
20
+ "trainable": true,
21
+ "dtype": {
22
+ "module": "keras",
23
+ "class_name": "DTypePolicy",
24
+ "config": {
25
+ "name": "int32"
26
+ },
27
+ "registered_name": null
28
+ },
29
+ "config_file": "tokenizer.json",
30
+ "sequence_length": null,
31
+ "add_prefix_space": false,
32
+ "unsplittable_tokens": [
33
+ "|||IP_ADDRESS|||",
34
+ "<|padding|>",
35
+ " ",
36
+ " ",
37
+ " ",
38
+ " ",
39
+ " ",
40
+ " ",
41
+ " ",
42
+ " ",
43
+ " ",
44
+ " ",
45
+ " ",
46
+ " ",
47
+ " ",
48
+ " ",
49
+ " ",
50
+ " ",
51
+ " ",
52
+ " ",
53
+ " ",
54
+ " ",
55
+ " ",
56
+ " ",
57
+ " ",
58
+ "|||EMAIL_ADDRESS|||",
59
+ "|||PHONE_NUMBER|||",
60
+ "<|endoftext|>",
61
+ "[UNK]",
62
+ "[CLS]",
63
+ "[SEP]",
64
+ "[PAD]",
65
+ "[MASK]",
66
+ "[unused0]",
67
+ "[unused1]",
68
+ "[unused2]",
69
+ "[unused3]",
70
+ "[unused4]",
71
+ "[unused5]",
72
+ "[unused6]",
73
+ "[unused7]",
74
+ "[unused8]",
75
+ "[unused9]",
76
+ "[unused10]",
77
+ "[unused11]",
78
+ "[unused12]",
79
+ "[unused13]",
80
+ "[unused14]",
81
+ "[unused15]",
82
+ "[unused16]",
83
+ "[unused17]",
84
+ "[unused18]",
85
+ "[unused19]",
86
+ "[unused20]",
87
+ "[unused21]",
88
+ "[unused22]",
89
+ "[unused23]",
90
+ "[unused24]",
91
+ "[unused25]",
92
+ "[unused26]",
93
+ "[unused27]",
94
+ "[unused28]",
95
+ "[unused29]",
96
+ "[unused30]",
97
+ "[unused31]",
98
+ "[unused32]",
99
+ "[unused33]",
100
+ "[unused34]",
101
+ "[unused35]",
102
+ "[unused36]",
103
+ "[unused37]",
104
+ "[unused38]",
105
+ "[unused39]",
106
+ "[unused40]",
107
+ "[unused41]",
108
+ "[unused42]",
109
+ "[unused43]",
110
+ "[unused44]",
111
+ "[unused45]",
112
+ "[unused46]",
113
+ "[unused47]",
114
+ "[unused48]",
115
+ "[unused49]",
116
+ "[unused50]",
117
+ "[unused51]",
118
+ "[unused52]",
119
+ "[unused53]",
120
+ "[unused54]",
121
+ "[unused55]",
122
+ "[unused56]",
123
+ "[unused57]",
124
+ "[unused58]",
125
+ "[unused59]",
126
+ "[unused60]",
127
+ "[unused61]",
128
+ "[unused62]",
129
+ "[unused63]",
130
+ "[unused64]",
131
+ "[unused65]",
132
+ "[unused66]",
133
+ "[unused67]",
134
+ "[unused68]",
135
+ "[unused69]",
136
+ "[unused70]",
137
+ "[unused71]",
138
+ "[unused72]",
139
+ "[unused73]",
140
+ "[unused74]",
141
+ "[unused75]",
142
+ "[unused76]",
143
+ "[unused77]",
144
+ "[unused78]",
145
+ "[unused79]",
146
+ "[unused80]",
147
+ "[unused81]",
148
+ "[unused82]"
149
+ ]
150
+ },
151
+ "registered_name": "keras_hub>ModernBertTokenizer"
152
+ },
153
+ "config_file": "preprocessor.json",
154
+ "sequence_length": 512,
155
+ "truncate": "round_robin",
156
+ "mask_selection_rate": 0.15,
157
+ "mask_selection_length": 96,
158
+ "mask_token_rate": 0.8,
159
+ "random_token_rate": 0.1
160
+ },
161
+ "registered_name": "keras_hub>ModernBertMaskedLMPreprocessor"
162
+ }
task.json ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "module": "keras_hub.src.models.modern_bert.modern_bert_masked_lm",
3
+ "class_name": "ModernBertMaskedLM",
4
+ "config": {
5
+ "backbone": {
6
+ "module": "keras_hub.src.models.modern_bert.modern_bert_backbone",
7
+ "class_name": "ModernBertBackbone",
8
+ "config": {
9
+ "name": "modern_bert_backbone",
10
+ "trainable": true,
11
+ "dtype": {
12
+ "module": "keras",
13
+ "class_name": "DTypePolicy",
14
+ "config": {
15
+ "name": "float32"
16
+ },
17
+ "registered_name": null
18
+ },
19
+ "vocabulary_size": 50368,
20
+ "hidden_dim": 1024,
21
+ "intermediate_dim": 2624,
22
+ "num_layers": 28,
23
+ "num_heads": 16,
24
+ "local_attention_window": 128,
25
+ "global_attn_every_n_layers": 3,
26
+ "rotary_max_wavelength": 160000.0,
27
+ "local_rotary_max_wavelength": 10000.0,
28
+ "layer_norm_epsilon": 1e-05
29
+ },
30
+ "registered_name": "keras_hub>ModernBertBackbone"
31
+ },
32
+ "preprocessor": {
33
+ "module": "keras_hub.src.models.modern_bert.modern_bert_masked_lm_preprocessor",
34
+ "class_name": "ModernBertMaskedLMPreprocessor",
35
+ "config": {
36
+ "name": "modern_bert_masked_lm_preprocessor",
37
+ "trainable": true,
38
+ "dtype": {
39
+ "module": "keras",
40
+ "class_name": "DTypePolicy",
41
+ "config": {
42
+ "name": "float32"
43
+ },
44
+ "registered_name": null
45
+ },
46
+ "tokenizer": {
47
+ "module": "keras_hub.src.models.modern_bert.modern_bert_tokenizer",
48
+ "class_name": "ModernBertTokenizer",
49
+ "config": {
50
+ "name": "modern_bert_tokenizer",
51
+ "trainable": true,
52
+ "dtype": {
53
+ "module": "keras",
54
+ "class_name": "DTypePolicy",
55
+ "config": {
56
+ "name": "int32"
57
+ },
58
+ "registered_name": null
59
+ },
60
+ "config_file": "tokenizer.json",
61
+ "sequence_length": null,
62
+ "add_prefix_space": false,
63
+ "unsplittable_tokens": [
64
+ "|||IP_ADDRESS|||",
65
+ "<|padding|>",
66
+ " ",
67
+ " ",
68
+ " ",
69
+ " ",
70
+ " ",
71
+ " ",
72
+ " ",
73
+ " ",
74
+ " ",
75
+ " ",
76
+ " ",
77
+ " ",
78
+ " ",
79
+ " ",
80
+ " ",
81
+ " ",
82
+ " ",
83
+ " ",
84
+ " ",
85
+ " ",
86
+ " ",
87
+ " ",
88
+ " ",
89
+ "|||EMAIL_ADDRESS|||",
90
+ "|||PHONE_NUMBER|||",
91
+ "<|endoftext|>",
92
+ "[UNK]",
93
+ "[CLS]",
94
+ "[SEP]",
95
+ "[PAD]",
96
+ "[MASK]",
97
+ "[unused0]",
98
+ "[unused1]",
99
+ "[unused2]",
100
+ "[unused3]",
101
+ "[unused4]",
102
+ "[unused5]",
103
+ "[unused6]",
104
+ "[unused7]",
105
+ "[unused8]",
106
+ "[unused9]",
107
+ "[unused10]",
108
+ "[unused11]",
109
+ "[unused12]",
110
+ "[unused13]",
111
+ "[unused14]",
112
+ "[unused15]",
113
+ "[unused16]",
114
+ "[unused17]",
115
+ "[unused18]",
116
+ "[unused19]",
117
+ "[unused20]",
118
+ "[unused21]",
119
+ "[unused22]",
120
+ "[unused23]",
121
+ "[unused24]",
122
+ "[unused25]",
123
+ "[unused26]",
124
+ "[unused27]",
125
+ "[unused28]",
126
+ "[unused29]",
127
+ "[unused30]",
128
+ "[unused31]",
129
+ "[unused32]",
130
+ "[unused33]",
131
+ "[unused34]",
132
+ "[unused35]",
133
+ "[unused36]",
134
+ "[unused37]",
135
+ "[unused38]",
136
+ "[unused39]",
137
+ "[unused40]",
138
+ "[unused41]",
139
+ "[unused42]",
140
+ "[unused43]",
141
+ "[unused44]",
142
+ "[unused45]",
143
+ "[unused46]",
144
+ "[unused47]",
145
+ "[unused48]",
146
+ "[unused49]",
147
+ "[unused50]",
148
+ "[unused51]",
149
+ "[unused52]",
150
+ "[unused53]",
151
+ "[unused54]",
152
+ "[unused55]",
153
+ "[unused56]",
154
+ "[unused57]",
155
+ "[unused58]",
156
+ "[unused59]",
157
+ "[unused60]",
158
+ "[unused61]",
159
+ "[unused62]",
160
+ "[unused63]",
161
+ "[unused64]",
162
+ "[unused65]",
163
+ "[unused66]",
164
+ "[unused67]",
165
+ "[unused68]",
166
+ "[unused69]",
167
+ "[unused70]",
168
+ "[unused71]",
169
+ "[unused72]",
170
+ "[unused73]",
171
+ "[unused74]",
172
+ "[unused75]",
173
+ "[unused76]",
174
+ "[unused77]",
175
+ "[unused78]",
176
+ "[unused79]",
177
+ "[unused80]",
178
+ "[unused81]",
179
+ "[unused82]"
180
+ ]
181
+ },
182
+ "registered_name": "keras_hub>ModernBertTokenizer"
183
+ },
184
+ "config_file": "preprocessor.json",
185
+ "sequence_length": 512,
186
+ "truncate": "round_robin",
187
+ "mask_selection_rate": 0.15,
188
+ "mask_selection_length": 96,
189
+ "mask_token_rate": 0.8,
190
+ "random_token_rate": 0.1
191
+ },
192
+ "registered_name": "keras_hub>ModernBertMaskedLMPreprocessor"
193
+ },
194
+ "name": "modern_bert_masked_lm"
195
+ },
196
+ "registered_name": "keras_hub>ModernBertMaskedLM"
197
+ }
task.weights.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b16acbcfddc8b1fe96e1f15f382e1e47a132d7249e153259c78cbd47cb02df6
3
+ size 4424472
tokenizer.json ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "module": "keras_hub.src.models.modern_bert.modern_bert_tokenizer",
3
+ "class_name": "ModernBertTokenizer",
4
+ "config": {
5
+ "name": "modern_bert_tokenizer",
6
+ "trainable": true,
7
+ "dtype": {
8
+ "module": "keras",
9
+ "class_name": "DTypePolicy",
10
+ "config": {
11
+ "name": "int32"
12
+ },
13
+ "registered_name": null
14
+ },
15
+ "config_file": "tokenizer.json",
16
+ "sequence_length": null,
17
+ "add_prefix_space": false,
18
+ "unsplittable_tokens": [
19
+ "|||IP_ADDRESS|||",
20
+ "<|padding|>",
21
+ " ",
22
+ " ",
23
+ " ",
24
+ " ",
25
+ " ",
26
+ " ",
27
+ " ",
28
+ " ",
29
+ " ",
30
+ " ",
31
+ " ",
32
+ " ",
33
+ " ",
34
+ " ",
35
+ " ",
36
+ " ",
37
+ " ",
38
+ " ",
39
+ " ",
40
+ " ",
41
+ " ",
42
+ " ",
43
+ " ",
44
+ "|||EMAIL_ADDRESS|||",
45
+ "|||PHONE_NUMBER|||",
46
+ "<|endoftext|>",
47
+ "[UNK]",
48
+ "[CLS]",
49
+ "[SEP]",
50
+ "[PAD]",
51
+ "[MASK]",
52
+ "[unused0]",
53
+ "[unused1]",
54
+ "[unused2]",
55
+ "[unused3]",
56
+ "[unused4]",
57
+ "[unused5]",
58
+ "[unused6]",
59
+ "[unused7]",
60
+ "[unused8]",
61
+ "[unused9]",
62
+ "[unused10]",
63
+ "[unused11]",
64
+ "[unused12]",
65
+ "[unused13]",
66
+ "[unused14]",
67
+ "[unused15]",
68
+ "[unused16]",
69
+ "[unused17]",
70
+ "[unused18]",
71
+ "[unused19]",
72
+ "[unused20]",
73
+ "[unused21]",
74
+ "[unused22]",
75
+ "[unused23]",
76
+ "[unused24]",
77
+ "[unused25]",
78
+ "[unused26]",
79
+ "[unused27]",
80
+ "[unused28]",
81
+ "[unused29]",
82
+ "[unused30]",
83
+ "[unused31]",
84
+ "[unused32]",
85
+ "[unused33]",
86
+ "[unused34]",
87
+ "[unused35]",
88
+ "[unused36]",
89
+ "[unused37]",
90
+ "[unused38]",
91
+ "[unused39]",
92
+ "[unused40]",
93
+ "[unused41]",
94
+ "[unused42]",
95
+ "[unused43]",
96
+ "[unused44]",
97
+ "[unused45]",
98
+ "[unused46]",
99
+ "[unused47]",
100
+ "[unused48]",
101
+ "[unused49]",
102
+ "[unused50]",
103
+ "[unused51]",
104
+ "[unused52]",
105
+ "[unused53]",
106
+ "[unused54]",
107
+ "[unused55]",
108
+ "[unused56]",
109
+ "[unused57]",
110
+ "[unused58]",
111
+ "[unused59]",
112
+ "[unused60]",
113
+ "[unused61]",
114
+ "[unused62]",
115
+ "[unused63]",
116
+ "[unused64]",
117
+ "[unused65]",
118
+ "[unused66]",
119
+ "[unused67]",
120
+ "[unused68]",
121
+ "[unused69]",
122
+ "[unused70]",
123
+ "[unused71]",
124
+ "[unused72]",
125
+ "[unused73]",
126
+ "[unused74]",
127
+ "[unused75]",
128
+ "[unused76]",
129
+ "[unused77]",
130
+ "[unused78]",
131
+ "[unused79]",
132
+ "[unused80]",
133
+ "[unused81]",
134
+ "[unused82]"
135
+ ]
136
+ },
137
+ "registered_name": "keras_hub>ModernBertTokenizer"
138
+ }