Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
349dacb
·
verified ·
1 Parent(s): 6f56fe6

EPFL final checkpoints and W&B history 51-100 of 1463

Browse files
Files changed (50) hide show
  1. epfl_final/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700/config.json +30 -0
  2. epfl_final/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700/model.pt +3 -0
  3. epfl_final/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700/config.json +30 -0
  4. epfl_final/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700/model.pt +3 -0
  5. epfl_final/checkpoints/l30k365m_mixed_seed42/run_20260720_041840/checkpoint-11700/config.json +42 -0
  6. epfl_final/checkpoints/l30k365m_mixed_seed42/run_20260720_041840/checkpoint-11700/model.pt +3 -0
  7. epfl_final/checkpoints/l30k365m_mixed_seed43/run_20260725_075512/checkpoint-3000/config.json +42 -0
  8. epfl_final/checkpoints/l30k365m_mixed_seed43/run_20260725_075512/checkpoint-3000/model.pt +3 -0
  9. epfl_final/checkpoints/l30k365m_mixed_seed44/run_20260725_075540/checkpoint-3000/config.json +42 -0
  10. epfl_final/checkpoints/l30k365m_mixed_seed44/run_20260725_075540/checkpoint-3000/model.pt +3 -0
  11. epfl_final/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700/config.json +30 -0
  12. epfl_final/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700/model.pt +3 -0
  13. epfl_final/checkpoints/l8k1_2b_dense_to_mixed_10pct/run_20260727_214510/checkpoint-50462/config.json +42 -0
  14. epfl_final/checkpoints/l8k1_2b_dense_to_mixed_10pct/run_20260727_214510/checkpoint-50462/model.pt +3 -0
  15. epfl_final/checkpoints/l8k1_2b_dense_to_mixed_20pct/run_20260727_214510/checkpoint-50462/config.json +42 -0
  16. epfl_final/checkpoints/l8k1_2b_dense_to_mixed_20pct/run_20260727_214510/checkpoint-50462/model.pt +3 -0
  17. epfl_final/checkpoints/l8k1_2b_mixed_seed43/run_20260720_185617/checkpoint-50462/config.json +42 -0
  18. epfl_final/checkpoints/l8k1_2b_mixed_seed43/run_20260720_185617/checkpoint-50462/model.pt +3 -0
  19. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct/run_20260730_130654/checkpoint-50462/config.json +36 -0
  20. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct/run_20260730_130654/checkpoint-50462/model.pt +3 -0
  21. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct_seed43/run_20260731_161000/checkpoint-50462/config.json +30 -0
  22. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct_seed43/run_20260731_161000/checkpoint-50462/model.pt +3 -0
  23. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct/run_20260730_071548/checkpoint-50462/config.json +36 -0
  24. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct/run_20260730_071548/checkpoint-50462/model.pt +3 -0
  25. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct_seed43/run_20260731_073150/checkpoint-50462/config.json +30 -0
  26. epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct_seed43/run_20260731_073150/checkpoint-50462/model.pt +3 -0
  27. epfl_final/checkpoints/l8k1_2b_moba/run_20260709_223237/checkpoint-44000/config.json +35 -0
  28. epfl_final/checkpoints/l8k1_2b_moba/run_20260709_223237/checkpoint-44000/model.pt +3 -0
  29. epfl_final/checkpoints/l8k1_2b_moba_matched/run_20260711_083403/checkpoint-50462/config.json +35 -0
  30. epfl_final/checkpoints/l8k1_2b_moba_matched/run_20260711_083403/checkpoint-50462/model.pt +3 -0
  31. epfl_final/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-45837/config.json +37 -0
  32. epfl_final/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-45837/model.pt +3 -0
  33. epfl_final/checkpoints/l8k365m_asentmax/run_20260709_035002/checkpoint-3000/config.json +35 -0
  34. epfl_final/checkpoints/l8k365m_asentmax/run_20260709_035002/checkpoint-3000/model.pt +3 -0
  35. epfl_final/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-44000/config.json +37 -0
  36. epfl_final/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-44000/model.pt +3 -0
  37. epfl_final/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750/config.json +55 -0
  38. epfl_final/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750/model.pt +3 -0
  39. epfl_final/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-44000/config.json +30 -0
  40. epfl_final/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-44000/model.pt +3 -0
  41. epfl_final/checkpoints/l8k365m_fixed_s1_seed43/run_20260802_194730/checkpoint-3000/config.json +30 -0
  42. epfl_final/checkpoints/l8k365m_fixed_s1_seed43/run_20260802_194730/checkpoint-3000/model.pt +3 -0
  43. epfl_final/checkpoints/l8k365m_fixed_s1_seed44/run_20260802_194739/checkpoint-3000/config.json +30 -0
  44. epfl_final/checkpoints/l8k365m_fixed_s1_seed44/run_20260802_194739/checkpoint-3000/model.pt +3 -0
  45. epfl_final/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-44000/config.json +30 -0
  46. epfl_final/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-44000/model.pt +3 -0
  47. epfl_final/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-44000/config.json +43 -0
  48. epfl_final/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-44000/model.pt +3 -0
  49. epfl_final/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044926/checkpoint-50/config.json +43 -0
  50. epfl_final/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044926/checkpoint-50/model.pt +3 -0
epfl_final/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s1",
29
+ "sparse_mask_seed": 1
30
+ }
epfl_final/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9011f1ba137c805b0c4ff484446b0078dc6962c4bb81059f200af9809e7d027
3
+ size 718031899
epfl_final/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s2",
29
+ "sparse_mask_seed": 2
30
+ }
epfl_final/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfc58d126ae6df0324b5bb6e5f777cc23d1114475264b0d20db8466f5ff75e0a
3
+ size 718031899
epfl_final/checkpoints/l30k365m_mixed_seed42/run_20260720_041840/checkpoint-11700/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_365m_mixed_seed42",
29
+ "sparse_mask_seed": 42,
30
+ "sparse_mixed_nrb_set": [
31
+ 239,
32
+ 179,
33
+ 120,
34
+ 79,
35
+ 39
36
+ ],
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
epfl_final/checkpoints/l30k365m_mixed_seed42/run_20260720_041840/checkpoint-11700/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87efeb262cefeb1b959400b67b639bd9bd87e05532201c12374125d1e19d6f52
3
+ size 718031899
epfl_final/checkpoints/l30k365m_mixed_seed43/run_20260725_075512/checkpoint-3000/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_365m_mixed_seed42",
29
+ "sparse_mask_seed": 42,
30
+ "sparse_mixed_nrb_set": [
31
+ 239,
32
+ 179,
33
+ 120,
34
+ 79,
35
+ 39
36
+ ],
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
epfl_final/checkpoints/l30k365m_mixed_seed43/run_20260725_075512/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12839b3f2d11942bbf1c85c923cb780109829aa36184cbb5174447b1859fbf5a
3
+ size 718031899
epfl_final/checkpoints/l30k365m_mixed_seed44/run_20260725_075540/checkpoint-3000/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_365m_mixed_seed42",
29
+ "sparse_mask_seed": 42,
30
+ "sparse_mixed_nrb_set": [
31
+ 239,
32
+ 179,
33
+ 120,
34
+ 79,
35
+ 39
36
+ ],
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
epfl_final/checkpoints/l30k365m_mixed_seed44/run_20260725_075540/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:746d07e4d2f767760661f2667f2d79fda10ae6b6ac753a3f462836b3c76eb3dd
3
+ size 718031899
epfl_final/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120",
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d6c1ce3a407a2493866b560b73a9ebef2a054d984b9b11a89e31e9a849015cc
3
+ size 718031899
epfl_final/checkpoints/l8k1_2b_dense_to_mixed_10pct/run_20260727_214510/checkpoint-50462/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": [
31
+ 63,
32
+ 47,
33
+ 31,
34
+ 21,
35
+ 11
36
+ ],
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
epfl_final/checkpoints/l8k1_2b_dense_to_mixed_10pct/run_20260727_214510/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b04bba6c3edf9f3637b8ca08435a44dd6c4b7089f7093eb969f71f9cbb33c98
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_dense_to_mixed_20pct/run_20260727_214510/checkpoint-50462/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": [
31
+ 63,
32
+ 47,
33
+ 31,
34
+ 21,
35
+ 11
36
+ ],
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
epfl_final/checkpoints/l8k1_2b_dense_to_mixed_20pct/run_20260727_214510/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42230b28b22722f47af6764184ffedc39abfc374f8d9751dc33c50a6ecdcb0e1
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_mixed_seed43/run_20260720_185617/checkpoint-50462/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": [
31
+ 63,
32
+ 47,
33
+ 31,
34
+ 21,
35
+ 11
36
+ ],
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
epfl_final/checkpoints/l8k1_2b_mixed_seed43/run_20260720_185617/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de60554820d4b1a84f6eafec436cd81dc7eec7a643856c0a47c19e670907523c
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct/run_20260730_130654/checkpoint-50462/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct/run_20260730_130654/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:602e7c49103689a981030dc5c4600b8ce73eef5566eb958d24d09991e5c61e02
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct_seed43/run_20260731_161000/checkpoint-50462/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_10pct_seed43/run_20260731_161000/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2fff7a22e22e31bb25aa52bf5bdea156ac8371d210e259ca5ef2353af69f8d17
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct/run_20260730_071548/checkpoint-50462/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct/run_20260730_071548/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3132beb84832d0d5daee1d76a04f68a2d8e7089e7e70f1a9d13a161711a0dab
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct_seed43/run_20260731_073150/checkpoint-50462/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k1_2b_mixed_to_dense_20pct_seed43/run_20260731_073150/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aada9e1ee3b9552bd904d7fb2cc64a8c7054751b0006d639b616c2f587197add
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_moba/run_20260709_223237/checkpoint-44000/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "moba",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "moba_block_size": 128,
31
+ "moba_top_k": 32,
32
+ "asentmax_alpha": 1.5,
33
+ "asentmax_learn_temp": true,
34
+ "asentmax_init_temp": 1.0
35
+ }
epfl_final/checkpoints/l8k1_2b_moba/run_20260709_223237/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecff7b042216a0de4f9008522322a50815ec8e407928552b67fb567430d83633
3
+ size 2404819483
epfl_final/checkpoints/l8k1_2b_moba_matched/run_20260711_083403/checkpoint-50462/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "moba",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "moba_block_size": 128,
31
+ "moba_top_k": 32,
32
+ "asentmax_alpha": 1.5,
33
+ "asentmax_learn_temp": true,
34
+ "asentmax_init_temp": 1.0
35
+ }
epfl_final/checkpoints/l8k1_2b_moba_matched/run_20260711_083403/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5ad551d62ef67a7ac8d102d759f3e2f134911674789c309cac2d92a432263cf
3
+ size 2404819483
epfl_final/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-45837/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 31,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": null,
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-45837/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f9053ee044266bb8e803678d0d13c06fc1b740b6429ac5933ba2646d5a7ad5dc
3
+ size 2404819483
epfl_final/checkpoints/l8k365m_asentmax/run_20260709_035002/checkpoint-3000/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": true,
22
+ "attention_type": "asentmax",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "moba_block_size": 128,
31
+ "moba_top_k": 32,
32
+ "asentmax_alpha": 1.5,
33
+ "asentmax_learn_temp": true,
34
+ "asentmax_init_temp": 1.0
35
+ }
epfl_final/checkpoints/l8k365m_asentmax/run_20260709_035002/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6259eeaaf7338308fbcedebb2b5f32c7c14d719f2131114faf74b06338ba1b5
3
+ size 718033387
epfl_final/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 31,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": null,
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:135fac1a8e54cbaa1fbbe54593d125146723a4edd4a7a8eed40eb4fbdba367cd
3
+ size 718031899
epfl_final/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750/config.json ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 31,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": [
29
+ 3,
30
+ 5,
31
+ 7,
32
+ 9,
33
+ 11,
34
+ 13,
35
+ 15,
36
+ 17,
37
+ 19,
38
+ 21,
39
+ 23
40
+ ],
41
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
42
+ "sparse_mask_seed": 0,
43
+ "sparse_mixed_nrb_set": [
44
+ 63,
45
+ 47,
46
+ 31,
47
+ 21,
48
+ 11
49
+ ],
50
+ "moba_block_size": 128,
51
+ "moba_top_k": 32,
52
+ "asentmax_alpha": 1.5,
53
+ "asentmax_learn_temp": true,
54
+ "asentmax_init_temp": 1.0
55
+ }
epfl_final/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2687b427a772f3081c201f096dacccf89a634a61d63e91cf224547f470ea28de
3
+ size 718031899
epfl_final/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s1",
29
+ "sparse_mask_seed": 1
30
+ }
epfl_final/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a09cfc24ea6aa90249500e01e20e99894454e4959078abdcfa1d1eb5ab75188b
3
+ size 718031899
epfl_final/checkpoints/l8k365m_fixed_s1_seed43/run_20260802_194730/checkpoint-3000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s1",
29
+ "sparse_mask_seed": 1
30
+ }
epfl_final/checkpoints/l8k365m_fixed_s1_seed43/run_20260802_194730/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:050d3b6ec081dd3f3a03afcad711c1b7a93fa14f6e8428ba449628908caca0d7
3
+ size 718031899
epfl_final/checkpoints/l8k365m_fixed_s1_seed44/run_20260802_194739/checkpoint-3000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s1",
29
+ "sparse_mask_seed": 1
30
+ }
epfl_final/checkpoints/l8k365m_fixed_s1_seed44/run_20260802_194739/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2f3e0003157d2cb118e86c636cd18e209c40cbcb9133759f0b86b6d773061868
3
+ size 718031899
epfl_final/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s2",
29
+ "sparse_mask_seed": 2
30
+ }
epfl_final/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d72346c2850a838f89692cd6e6fc7efc812eb7d749e2b22a35b37ad02490f12
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-44000/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 31,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": [
32
+ 63,
33
+ 47,
34
+ 31,
35
+ 21,
36
+ 11
37
+ ],
38
+ "moba_block_size": 128,
39
+ "moba_top_k": 32,
40
+ "asentmax_alpha": 1.5,
41
+ "asentmax_learn_temp": true,
42
+ "asentmax_init_temp": 1.0
43
+ }
epfl_final/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b32abe59ae4db187fcabebf0e9d498747a5837fa609d743b710eeb0fdeed292
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044926/checkpoint-50/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 31,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": [
32
+ 63,
33
+ 47,
34
+ 31,
35
+ 21,
36
+ 11
37
+ ],
38
+ "moba_block_size": 128,
39
+ "moba_top_k": 32,
40
+ "asentmax_alpha": 1.5,
41
+ "asentmax_learn_temp": true,
42
+ "asentmax_init_temp": 1.0
43
+ }
epfl_final/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044926/checkpoint-50/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d317b21b61a96c74de54a25ed71b152a6fc9eba9081bedc2affa61b454d4e5e
3
+ size 718031899