Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
874cc3d
·
verified ·
1 Parent(s): 4189812

Add final paper checkpoint: checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462

Browse files
Files changed (15) hide show
  1. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/config.json +43 -0
  2. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank0.pt +3 -0
  3. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank1.pt +3 -0
  4. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank2.pt +3 -0
  5. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank3.pt +3 -0
  6. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank4.pt +3 -0
  7. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank5.pt +3 -0
  8. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/model.pt +3 -0
  9. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank0.pt +3 -0
  10. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank1.pt +3 -0
  11. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank2.pt +3 -0
  12. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank3.pt +3 -0
  13. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank4.pt +3 -0
  14. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank5.pt +3 -0
  15. checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/training_state.pt +3 -0
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": true,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
30
+ "sparse_mask_seed": 70,
31
+ "sparse_mixed_nrb_set": [
32
+ 63,
33
+ 47,
34
+ 31,
35
+ 21,
36
+ 11
37
+ ],
38
+ "moba_block_size": 128,
39
+ "moba_top_k": 32,
40
+ "asentmax_alpha": 1.5,
41
+ "asentmax_learn_temp": true,
42
+ "asentmax_init_temp": 1.0
43
+ }
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3d3d87a844d4fd6548dfc4c1235a1ee3e262506bc6260da75e7cbf9d9da819c
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2c029e9127ff4c346ab99e6af8377a6903003909fec0b073b70bfafc01d4517
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:917e752ef93a10f1da3dc49f9c18adc43ae1bb568ee13d528c9824ee4513b511
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:389dee6373df6f406c9ae897b97bc5acc1b426bc8c9810e2754f7d2f892a256f
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:003f8a68f6377e121aa1e72c457466320b22898c8d61a840ded7da21790b0d73
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/loader_gen_state_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3e0c9bf4dad32ac42f0c9f053baa4af26df892924de882f06738359b36322b1
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e33991c18c04fb1a5c29d325bdd39b343c0299e581d93699c62e0d5c6e3b70b
3
+ size 2404819483
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca040b4e3213859fff31ec3fc67cfe849fdb82ba8b23b9a1071944559f0c7844
3
+ size 14757
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9713836b925fec9e3a9c4b4d13fe0241d9ef10ca0a9214076621dfa6e2832860
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1de0a928a6b70dd8f236cabbeb948257ed63806a9233f192c1740d8960f99620
3
+ size 14885
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d730d115312760eacbf364f605b05ff6146c3816bbb0de7f070a066bd4137dc
3
+ size 14885
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af3ae40373129ff4f062ba96852005157b422f017292d6d8f7738632c50d1abe
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/rng_state_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d063d8b4cd439658f32a4b786d0eb1833c14c733aafcd7538381afc1cd9dc38
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_mixed_seed70_repaired/checkpoint-50462/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:964941370917066c5770c6f0d63207dd5182122996e9d891f9f47a2d1e3ebac5
3
+ size 4809681879