Update config.json
Browse files- config.json +7 -13
config.json
CHANGED
|
@@ -3,6 +3,10 @@
|
|
| 3 |
"OleanderForCausalLM"
|
| 4 |
],
|
| 5 |
"model_type": "oleander",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
"vocab_size": 32770,
|
| 7 |
"hidden_size": 576,
|
| 8 |
"num_hidden_layers": 28,
|
|
@@ -67,7 +71,7 @@
|
|
| 67 |
"full_attention"
|
| 68 |
],
|
| 69 |
"intermediate_size": 1600,
|
| 70 |
-
"max_position_embeddings":
|
| 71 |
"rms_norm_eps": 1e-06,
|
| 72 |
"num_attention_heads": 9,
|
| 73 |
"num_key_value_heads": 3,
|
|
@@ -87,7 +91,7 @@
|
|
| 87 |
"gqa"
|
| 88 |
],
|
| 89 |
"mlp_hidden": 1600,
|
| 90 |
-
"max_seq_len":
|
| 91 |
"norm_eps": 1e-06,
|
| 92 |
"init_std": 0.02,
|
| 93 |
"tie_embeddings": true,
|
|
@@ -111,15 +115,5 @@
|
|
| 111 |
"value_residual": true
|
| 112 |
}
|
| 113 |
},
|
| 114 |
-
"model_name": "Oleander-150M"
|
| 115 |
-
"training": {
|
| 116 |
-
"step": 3500,
|
| 117 |
-
"kind": "export",
|
| 118 |
-
"max_steps": 28820,
|
| 119 |
-
"percent_complete": 12.144,
|
| 120 |
-
"tokens_seen": 11349786624,
|
| 121 |
-
"compute_tokens_seen": 1891631104
|
| 122 |
-
},
|
| 123 |
-
"transformers_version": "5.14.1",
|
| 124 |
-
"torch_dtype": "float32"
|
| 125 |
}
|
|
|
|
| 3 |
"OleanderForCausalLM"
|
| 4 |
],
|
| 5 |
"model_type": "oleander",
|
| 6 |
+
"auto_map": {
|
| 7 |
+
"AutoConfig": "configuration_oleander.OleanderConfig",
|
| 8 |
+
"AutoModelForCausalLM": "modeling_oleander.OleanderForCausalLM"
|
| 9 |
+
},
|
| 10 |
"vocab_size": 32770,
|
| 11 |
"hidden_size": 576,
|
| 12 |
"num_hidden_layers": 28,
|
|
|
|
| 71 |
"full_attention"
|
| 72 |
],
|
| 73 |
"intermediate_size": 1600,
|
| 74 |
+
"max_position_embeddings": 2048,
|
| 75 |
"rms_norm_eps": 1e-06,
|
| 76 |
"num_attention_heads": 9,
|
| 77 |
"num_key_value_heads": 3,
|
|
|
|
| 91 |
"gqa"
|
| 92 |
],
|
| 93 |
"mlp_hidden": 1600,
|
| 94 |
+
"max_seq_len": 2048,
|
| 95 |
"norm_eps": 1e-06,
|
| 96 |
"init_std": 0.02,
|
| 97 |
"tie_embeddings": true,
|
|
|
|
| 115 |
"value_residual": true
|
| 116 |
}
|
| 117 |
},
|
| 118 |
+
"model_name": "Oleander-150M"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
}
|