feat: add Helios Flagship parameter manifest
Browse files
models/flagship/manifest.json
CHANGED
|
@@ -23,12 +23,14 @@
|
|
| 23 |
"n_heads": 32,
|
| 24 |
"n_kv_heads": 8,
|
| 25 |
"d_ffn": 11008,
|
| 26 |
-
"max_seq_len":
|
| 27 |
-
"rope_theta":
|
| 28 |
"rms_norm_eps": 1e-05,
|
| 29 |
"tie_word_embeddings": true,
|
| 30 |
"initializer_range": 0.02,
|
| 31 |
"dropout": 0.0,
|
|
|
|
|
|
|
| 32 |
"model_family": "Helios",
|
| 33 |
"developer": "Arron Kian Parejas",
|
| 34 |
"developer_email": "parejasarronkian@gmail.com"
|
|
|
|
| 23 |
"n_heads": 32,
|
| 24 |
"n_kv_heads": 8,
|
| 25 |
"d_ffn": 11008,
|
| 26 |
+
"max_seq_len": 262144,
|
| 27 |
+
"rope_theta": 10000000.0,
|
| 28 |
"rms_norm_eps": 1e-05,
|
| 29 |
"tie_word_embeddings": true,
|
| 30 |
"initializer_range": 0.02,
|
| 31 |
"dropout": 0.0,
|
| 32 |
+
"rope_scaling_factor": 1.0,
|
| 33 |
+
"model_version": "0.3.0-flagship",
|
| 34 |
"model_family": "Helios",
|
| 35 |
"developer": "Arron Kian Parejas",
|
| 36 |
"developer_email": "parejasarronkian@gmail.com"
|