Darknecrocities commited on
Commit
543a8a7
·
verified ·
1 Parent(s): c407770

feat: add Helios Beta parameter manifest

Browse files
Files changed (1) hide show
  1. models/beta/manifest.json +36 -0
models/beta/manifest.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Helios Beta",
3
+ "preset": "beta-300m",
4
+ "tag": "beta",
5
+ "target_size": "300M",
6
+ "hardware_target": "Workstation GPU (16GB RAM / MPS)",
7
+ "developer": "Arron Kian Parejas",
8
+ "developer_email": "parejasarronkian@gmail.com",
9
+ "parameters": {
10
+ "embedding_params": 32768000,
11
+ "layer_params": 11274240,
12
+ "total_layer_params": 270581760,
13
+ "final_norm_params": 1024,
14
+ "tied_total_params": 303350784,
15
+ "untied_total_params": 336118784,
16
+ "tied_million": 303.35,
17
+ "untied_million": 336.12
18
+ },
19
+ "config": {
20
+ "vocab_size": 32000,
21
+ "d_model": 1024,
22
+ "n_layers": 24,
23
+ "n_heads": 16,
24
+ "n_kv_heads": 4,
25
+ "d_ffn": 2816,
26
+ "max_seq_len": 2048,
27
+ "rope_theta": 10000.0,
28
+ "rms_norm_eps": 1e-05,
29
+ "tie_word_embeddings": true,
30
+ "initializer_range": 0.02,
31
+ "dropout": 0.0,
32
+ "model_family": "Helios",
33
+ "developer": "Arron Kian Parejas",
34
+ "developer_email": "parejasarronkian@gmail.com"
35
+ }
36
+ }