Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
c19b704
·
verified ·
1 Parent(s): aa8d5fc

Upload Part T Dense seed 70 final checkpoint step 50462

Browse files
Files changed (11) hide show
  1. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/config.json +40 -0
  2. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank0.pt +3 -0
  3. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank1.pt +3 -0
  4. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank2.pt +3 -0
  5. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank3.pt +3 -0
  6. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/model.pt +3 -0
  7. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank0.pt +3 -0
  8. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank1.pt +3 -0
  9. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank2.pt +3 -0
  10. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank3.pt +3 -0
  11. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/training_state.pt +3 -0
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": true,
26
+ "attention_type": "sparse_stochastic",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 31,
29
+ "sparse_mask_pool_size": 1,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": null,
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": null,
35
+ "moba_block_size": 128,
36
+ "moba_top_k": 32,
37
+ "asentmax_alpha": 1.5,
38
+ "asentmax_learn_temp": true,
39
+ "asentmax_init_temp": 1.0
40
+ }
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3d3d87a844d4fd6548dfc4c1235a1ee3e262506bc6260da75e7cbf9d9da819c
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2c029e9127ff4c346ab99e6af8377a6903003909fec0b073b70bfafc01d4517
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:917e752ef93a10f1da3dc49f9c18adc43ae1bb568ee13d528c9824ee4513b511
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:389dee6373df6f406c9ae897b97bc5acc1b426bc8c9810e2754f7d2f892a256f
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f46af0a046305c535fe16f57aed7b16289af72a9bfe1f486f6c2178dcf304f24
3
+ size 2404833883
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08de481fd08cd3c1878ab188739e636191f6f866d01d31ce9ccd22538f1777e5
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fba7b8b43576198287654e0935c8e1ea215b501c43e9441064362eaa860e2961
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c34c2a2dc8506512d1a3c3edfd2f8bf9a29f85a1e677087c2a3f35e96fa09d8e
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67e2783d9790e5686d5d56b9cf00fe59d116a478f7d6b025d069324c62940473
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-50462-rcp/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ffddfd335d5b5ccfc9f53923a6d38af08f2eaa1047a51deb4cbe55e8fead629
3
+ size 4809712479