Darknecrocities commited on
Commit
3b5dfd6
·
verified ·
1 Parent(s): decdb8e

feat: add Helios Nano architecture configuration

Browse files
Files changed (1) hide show
  1. models/nano/config.json +4 -2
models/nano/config.json CHANGED
@@ -5,12 +5,14 @@
5
  "n_heads": 8,
6
  "n_kv_heads": 2,
7
  "d_ffn": 864,
8
- "max_seq_len": 2048,
9
- "rope_theta": 10000.0,
10
  "rms_norm_eps": 1e-05,
11
  "tie_word_embeddings": true,
12
  "initializer_range": 0.02,
13
  "dropout": 0.0,
 
 
14
  "model_family": "Helios",
15
  "developer": "Arron Kian Parejas",
16
  "developer_email": "parejasarronkian@gmail.com"
 
5
  "n_heads": 8,
6
  "n_kv_heads": 2,
7
  "d_ffn": 864,
8
+ "max_seq_len": 16384,
9
+ "rope_theta": 100000.0,
10
  "rms_norm_eps": 1e-05,
11
  "tie_word_embeddings": true,
12
  "initializer_range": 0.02,
13
  "dropout": 0.0,
14
+ "rope_scaling_factor": 1.0,
15
+ "model_version": "0.3.0-nano",
16
  "model_family": "Helios",
17
  "developer": "Arron Kian Parejas",
18
  "developer_email": "parejasarronkian@gmail.com"