Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
94f13a4
·
verified ·
1 Parent(s): eca882c

Add FINAL5 seed-72 Dense resumable checkpoint at step 2000

Browse files
Files changed (13) hide show
  1. FINAL5_20261014/seed72_dense/checkpoint-2000/config.json +38 -0
  2. FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank0.pt +3 -0
  3. FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank1.pt +3 -0
  4. FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank2.pt +3 -0
  5. FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank3.pt +3 -0
  6. FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank4.pt +3 -0
  7. FINAL5_20261014/seed72_dense/checkpoint-2000/model.pt +3 -0
  8. FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank0.pt +3 -0
  9. FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank1.pt +3 -0
  10. FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank2.pt +3 -0
  11. FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank3.pt +3 -0
  12. FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank4.pt +3 -0
  13. FINAL5_20261014/seed72_dense/checkpoint-2000/training_state.pt +3 -0
FINAL5_20261014/seed72_dense/checkpoint-2000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "qk_gain_fp32": true,
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "sparse_stochastic",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 0,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b21b4160791b11c535b1414390199117f5857a94ac58d7b970468583a72bb0bc
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32ec129fe94d28aa0ddb1827422f7723c72b31ebc7a9306e98894b6131bb7fe9
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c5588d56d5c8519bb2c45d4b62d3684da8142cd70467b7c6b989cd944f4e533
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05faccfef6d5dfcc8dd8aca9b50fb4a4f1b5e5ee52a9a29e99e1e68bd3e62b77
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-2000/loader_gen_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e0a234bd3fe283f29b7ed9ee4980692f92fb206de53b64fe8758792b41c82e6
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-2000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2694e7b70058c4a97275d2b870720d0756aaf0ddfe57243c0e981cef660a78ab
3
+ size 2404833883
FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf3f8461a21545d5bfeb0ed4aff0464bf5f8cb8e60b48198288d617ace67ddcc
3
+ size 14821
FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48bed20d6e249d8a32b5e17bcd02c6be69e4d360570b3ddc274278ce1ca5f9af
3
+ size 14885
FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d328ed111f5cf761888e6e562e88a94d1457c15f9ecf5fa4e50a28902161bbc
3
+ size 14821
FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e0050bab53bb96b17649dc0bc1558202e57a72c5dbfeb3d372a763bf74889ec
3
+ size 14885
FINAL5_20261014/seed72_dense/checkpoint-2000/rng_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d0475f089464589ac57151ac13fe9fa04d01e2fcb10adc09c9e5e4cb73ba837
3
+ size 14757
FINAL5_20261014/seed72_dense/checkpoint-2000/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b842c8fd3e2c8d671c17f9d631944dd59ae9d6737b2cb1678585d62ce6371507
3
+ size 4809712479