Darknecrocities commited on
Commit
0daaeb5
·
verified ·
1 Parent(s): 80a9baf

feat: add Helios Flagship parameter manifest

Browse files
Files changed (1) hide show
  1. models/flagship/manifest.json +4 -2
models/flagship/manifest.json CHANGED
@@ -23,12 +23,14 @@
23
  "n_heads": 32,
24
  "n_kv_heads": 8,
25
  "d_ffn": 11008,
26
- "max_seq_len": 4096,
27
- "rope_theta": 500000.0,
28
  "rms_norm_eps": 1e-05,
29
  "tie_word_embeddings": true,
30
  "initializer_range": 0.02,
31
  "dropout": 0.0,
 
 
32
  "model_family": "Helios",
33
  "developer": "Arron Kian Parejas",
34
  "developer_email": "parejasarronkian@gmail.com"
 
23
  "n_heads": 32,
24
  "n_kv_heads": 8,
25
  "d_ffn": 11008,
26
+ "max_seq_len": 262144,
27
+ "rope_theta": 10000000.0,
28
  "rms_norm_eps": 1e-05,
29
  "tie_word_embeddings": true,
30
  "initializer_range": 0.02,
31
  "dropout": 0.0,
32
+ "rope_scaling_factor": 1.0,
33
+ "model_version": "0.3.0-flagship",
34
  "model_family": "Helios",
35
  "developer": "Arron Kian Parejas",
36
  "developer_email": "parejasarronkian@gmail.com"