Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
50b87e8
·
verified ·
1 Parent(s): b1eb76c

Upload paper checkpoint l8k365m_dense_seed47 checkpoint-44000

Browse files
Files changed (13) hide show
  1. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/config.json +37 -0
  2. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank0.pt +3 -0
  3. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank1.pt +3 -0
  4. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank2.pt +3 -0
  5. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank3.pt +3 -0
  6. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank4.pt +3 -0
  7. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/model.pt +3 -0
  8. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank0.pt +3 -0
  9. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank1.pt +3 -0
  10. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank2.pt +3 -0
  11. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank3.pt +3 -0
  12. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank4.pt +3 -0
  13. checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/training_state.pt +3 -0
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 31,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": null,
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d29ebcba1740400d02a9731a1b2ce2d06da520e9a5f365770ae3366476cfa6bb
3
+ size 6802
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6cbcc77de4742713528531e0ef8cdd89a1973c9e431acb1771b42ca22d7bd1f
3
+ size 6802
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9db5ff6feccc6d1a9e53c22c2c117265883048de542cfdb5f1d72684ce40e97f
3
+ size 6802
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2448f0fdf688f2e5aca811502f056dcbab182fad72e69178143577736b6f0c8e
3
+ size 6802
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/loader_gen_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e5371f5b80aa7358ef9290ec1864b1c0669c69fb1e0f3f24dfe821940c3a22a
3
+ size 6802
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fcbc46b085c405a7e6e03207ec616a98329e7104851e140bee1604e516c39ba5
3
+ size 718031899
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f788c6d59c14d7c16ea7debfe678626698ba4365ccb7298c917ea721473da31d
3
+ size 14757
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a34d85db33c14d0607eafaf6bd76b96ce7f65d4db9736ce37ee257226b67eb9f
3
+ size 14757
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e96a39c808efb86651ff4cf83ff1147dfa341b158d21a6defc7f33543970b46
3
+ size 14885
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:499039a66a3ffcd6e98373202df938050e1153d0c844e9b97c15d2b2e8b021fd
3
+ size 14821
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/rng_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ef8df31e0b5c227a738d3f9369ffa4dda6b0625ef6a72945b60ba9ff28b4047
3
+ size 14821
checkpoints/primary_365m/l8k365m_dense_seed47/checkpoint-44000/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fe64a3649d808af09b395616da85fcf6828bca68a76908519e210cb2c379e2d
3
+ size 1436105299