Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
6f56fe6
·
verified ·
1 Parent(s): fe19ae1

EPFL final checkpoints and W&B history 1-50 of 1463

Browse files
Files changed (50) hide show
  1. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/config.json +40 -0
  2. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/model.pt +3 -0
  3. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s43/run_20260924_152136/checkpoint-44000/config.json +40 -0
  4. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s43/run_20260924_152136/checkpoint-44000/model.pt +3 -0
  5. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s44/run_20260924_143852/checkpoint-44000/config.json +40 -0
  6. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s44/run_20260924_143852/checkpoint-44000/model.pt +3 -0
  7. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/config.json +46 -0
  8. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/model.pt +3 -0
  9. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s43/run_20260924_050049/checkpoint-44000/config.json +46 -0
  10. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s43/run_20260924_050049/checkpoint-44000/model.pt +3 -0
  11. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s44/run_20260924_144318/checkpoint-44000/config.json +46 -0
  12. epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s44/run_20260924_144318/checkpoint-44000/model.pt +3 -0
  13. epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_noqknorm_s42/run_20260922_224126/checkpoint-44000/config.json +39 -0
  14. epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_noqknorm_s42/run_20260922_224126/checkpoint-44000/model.pt +3 -0
  15. epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_rope1e4_s42/run_20260922_225417/checkpoint-44000/config.json +39 -0
  16. epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_rope1e4_s42/run_20260922_225417/checkpoint-44000/model.pt +3 -0
  17. epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_noqknorm_s42/run_20260922_225813/checkpoint-44000/config.json +45 -0
  18. epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_noqknorm_s42/run_20260922_225813/checkpoint-44000/model.pt +3 -0
  19. epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_rope1e4_s42/run_20260922_225809/checkpoint-44000/config.json +45 -0
  20. epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_rope1e4_s42/run_20260922_225809/checkpoint-44000/model.pt +3 -0
  21. epfl_final/checkpoints/a100_40gb_vram_probe_p28/run_20260819_142832/checkpoint-2/config.json +36 -0
  22. epfl_final/checkpoints/a100_40gb_vram_probe_p28/run_20260819_142832/checkpoint-2/model.pt +3 -0
  23. epfl_final/checkpoints/dropctl_calib_p00/run_20260807_080949/checkpoint-44300/config.json +30 -0
  24. epfl_final/checkpoints/dropctl_calib_p00/run_20260807_080949/checkpoint-44300/model.pt +3 -0
  25. epfl_final/checkpoints/dropctl_calib_p05/run_20260807_080949/checkpoint-44300/config.json +30 -0
  26. epfl_final/checkpoints/dropctl_calib_p05/run_20260807_080949/checkpoint-44300/model.pt +3 -0
  27. epfl_final/checkpoints/dropctl_calib_p10/run_20260807_080949/checkpoint-44300/config.json +30 -0
  28. epfl_final/checkpoints/dropctl_calib_p10/run_20260807_080949/checkpoint-44300/model.pt +3 -0
  29. epfl_final/checkpoints/dropctl_calib_p20/run_20260807_080949/checkpoint-44300/config.json +30 -0
  30. epfl_final/checkpoints/dropctl_calib_p20/run_20260807_080949/checkpoint-44300/model.pt +3 -0
  31. epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_034938/checkpoint-44100/config.json +36 -0
  32. epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_034938/checkpoint-44100/model.pt +3 -0
  33. epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_043051/checkpoint-44300/config.json +36 -0
  34. epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_043051/checkpoint-44300/model.pt +3 -0
  35. epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_034938/checkpoint-44100/config.json +36 -0
  36. epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_034938/checkpoint-44100/model.pt +3 -0
  37. epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_043051/checkpoint-44300/config.json +36 -0
  38. epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_043051/checkpoint-44300/model.pt +3 -0
  39. epfl_final/checkpoints/dropctl_diag_p28_20/run_20260819_041508/checkpoint-44020/config.json +36 -0
  40. epfl_final/checkpoints/dropctl_diag_p28_20/run_20260819_041508/checkpoint-44020/model.pt +3 -0
  41. epfl_final/checkpoints/l30k1_2b_mixed_to_dense_20pct/run_20260730_180750/checkpoint-37465/config.json +30 -0
  42. epfl_final/checkpoints/l30k1_2b_mixed_to_dense_20pct/run_20260730_180750/checkpoint-37465/model.pt +3 -0
  43. epfl_final/checkpoints/l30k1_2b_moba/run_20260718_031723/checkpoint-37465/config.json +36 -0
  44. epfl_final/checkpoints/l30k1_2b_moba/run_20260718_031723/checkpoint-37465/model.pt +3 -0
  45. epfl_final/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700/config.json +30 -0
  46. epfl_final/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700/model.pt +3 -0
  47. epfl_final/checkpoints/l30k365m_dense_seed43/run_20260725_075143/checkpoint-3000/config.json +36 -0
  48. epfl_final/checkpoints/l30k365m_dense_seed43/run_20260725_075143/checkpoint-3000/model.pt +3 -0
  49. epfl_final/checkpoints/l30k365m_dense_seed44/run_20260725_075209/checkpoint-3000/config.json +36 -0
  50. epfl_final/checkpoints/l30k365m_dense_seed44/run_20260725_075209/checkpoint-3000/model.pt +3 -0
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "dense",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 1,
29
+ "sparse_mask_pool_size": 1000,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": null,
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": null,
35
+ "moba_block_size": 128,
36
+ "moba_top_k": 32,
37
+ "asentmax_alpha": 1.5,
38
+ "asentmax_learn_temp": true,
39
+ "asentmax_init_temp": 1.0
40
+ }
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b034be702b8a3d0c65e008c3590ce966511944b1063591381e3000ee1a89587
3
+ size 718046299
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s43/run_20260924_152136/checkpoint-44000/config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "dense",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 1,
29
+ "sparse_mask_pool_size": 1000,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": null,
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": null,
35
+ "moba_block_size": 128,
36
+ "moba_top_k": 32,
37
+ "asentmax_alpha": 1.5,
38
+ "asentmax_learn_temp": true,
39
+ "asentmax_init_temp": 1.0
40
+ }
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s43/run_20260924_152136/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:472dc260dfa2dbf17f6f316c76633792019d84df43706f30da2a78bb7b41300b
3
+ size 718046299
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s44/run_20260924_143852/checkpoint-44000/config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "dense",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 1,
29
+ "sparse_mask_pool_size": 1000,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": null,
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": null,
35
+ "moba_block_size": 128,
36
+ "moba_top_k": 32,
37
+ "asentmax_alpha": 1.5,
38
+ "asentmax_learn_temp": true,
39
+ "asentmax_init_temp": 1.0
40
+ }
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_dense_qkgainfp32_s44/run_20260924_143852/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d001b11b78c34b33029265a428d2cca43f67401d2c22c1e4ff4bf7b4471f804f
3
+ size 718046299
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "sparse_stochastic",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 63,
29
+ "sparse_mask_pool_size": 10000,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": "../mask_cache_8k_mixed",
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": [
35
+ 63,
36
+ 47,
37
+ 31,
38
+ 21,
39
+ 11
40
+ ],
41
+ "moba_block_size": 128,
42
+ "moba_top_k": 32,
43
+ "asentmax_alpha": 1.5,
44
+ "asentmax_learn_temp": true,
45
+ "asentmax_init_temp": 1.0
46
+ }
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s42/run_20260924_045342/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9426c5f86bbfed724b641f5c8d980b93195d89e12a614f3902357a669b41b14
3
+ size 718046299
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s43/run_20260924_050049/checkpoint-44000/config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "sparse_stochastic",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 63,
29
+ "sparse_mask_pool_size": 10000,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": "../mask_cache_8k_mixed",
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": [
35
+ 63,
36
+ 47,
37
+ 31,
38
+ 21,
39
+ 11
40
+ ],
41
+ "moba_block_size": 128,
42
+ "moba_top_k": 32,
43
+ "asentmax_alpha": 1.5,
44
+ "asentmax_learn_temp": true,
45
+ "asentmax_init_temp": 1.0
46
+ }
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s43/run_20260924_050049/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c34a13fb603475199c18cf2aa4494e0022c68208f59c51b508f80c64f13ee1e
3
+ size 718046299
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s44/run_20260924_144318/checkpoint-44000/config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "sparse_stochastic",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 63,
29
+ "sparse_mask_pool_size": 10000,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": "../mask_cache_8k_mixed",
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": [
35
+ 63,
36
+ 47,
37
+ 31,
38
+ 21,
39
+ 11
40
+ ],
41
+ "moba_block_size": 128,
42
+ "moba_top_k": 32,
43
+ "asentmax_alpha": 1.5,
44
+ "asentmax_learn_temp": true,
45
+ "asentmax_init_temp": 1.0
46
+ }
epfl_final/SparseAtt/models/FINAL_20260924/part_a/365m_mixed_qkgainfp32_s44/run_20260924_144318/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1dea864f27a243f39c3a226cdd5726bcb41b8d83e7300a10cd86ceee0abc9f4c
3
+ size 718046299
epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_noqknorm_s42/run_20260922_224126/checkpoint-44000/config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "off",
19
+ "rope_train_position_scale_set": null,
20
+ "attention_bias": false,
21
+ "attention_dropout": 0.0,
22
+ "bos_token_id": 151643,
23
+ "eos_token_id": 151643,
24
+ "checkpoint_activations": false,
25
+ "attention_type": "dense",
26
+ "sparse_block_size": 128,
27
+ "sparse_num_random_blocks": 1,
28
+ "sparse_mask_pool_size": 1000,
29
+ "sparse_max_seq_length": 8192,
30
+ "sparse_layer_indices": null,
31
+ "sparse_mask_cache_dir": null,
32
+ "sparse_mask_seed": 0,
33
+ "sparse_mixed_nrb_set": null,
34
+ "moba_block_size": 128,
35
+ "moba_top_k": 32,
36
+ "asentmax_alpha": 1.5,
37
+ "asentmax_learn_temp": true,
38
+ "asentmax_init_temp": 1.0
39
+ }
epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_noqknorm_s42/run_20260922_224126/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be8b0baaeef77b0f57bfa849ecda17444f89dcde047686518ea86905aff87930
3
+ size 717998843
epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_rope1e4_s42/run_20260922_225417/checkpoint-44000/config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 10000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "rope_train_position_scale_set": null,
20
+ "attention_bias": false,
21
+ "attention_dropout": 0.0,
22
+ "bos_token_id": 151643,
23
+ "eos_token_id": 151643,
24
+ "checkpoint_activations": false,
25
+ "attention_type": "dense",
26
+ "sparse_block_size": 128,
27
+ "sparse_num_random_blocks": 1,
28
+ "sparse_mask_pool_size": 1000,
29
+ "sparse_max_seq_length": 8192,
30
+ "sparse_layer_indices": null,
31
+ "sparse_mask_cache_dir": null,
32
+ "sparse_mask_seed": 0,
33
+ "sparse_mixed_nrb_set": null,
34
+ "moba_block_size": 128,
35
+ "moba_top_k": 32,
36
+ "asentmax_alpha": 1.5,
37
+ "asentmax_learn_temp": true,
38
+ "asentmax_init_temp": 1.0
39
+ }
epfl_final/SparseAtt/models/REGIME_20260922/365m_dense_rope1e4_s42/run_20260922_225417/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5e168863a7f6d46702d54a3672880bab9021828cbd17aafc29cb4ecc7f0459e
3
+ size 718031899
epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_noqknorm_s42/run_20260922_225813/checkpoint-44000/config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "off",
19
+ "rope_train_position_scale_set": null,
20
+ "attention_bias": false,
21
+ "attention_dropout": 0.0,
22
+ "bos_token_id": 151643,
23
+ "eos_token_id": 151643,
24
+ "checkpoint_activations": false,
25
+ "attention_type": "sparse_stochastic",
26
+ "sparse_block_size": 128,
27
+ "sparse_num_random_blocks": 63,
28
+ "sparse_mask_pool_size": 10000,
29
+ "sparse_max_seq_length": 8192,
30
+ "sparse_layer_indices": null,
31
+ "sparse_mask_cache_dir": "../mask_cache_8k_mixed",
32
+ "sparse_mask_seed": 0,
33
+ "sparse_mixed_nrb_set": [
34
+ 63,
35
+ 47,
36
+ 31,
37
+ 21,
38
+ 11
39
+ ],
40
+ "moba_block_size": 128,
41
+ "moba_top_k": 32,
42
+ "asentmax_alpha": 1.5,
43
+ "asentmax_learn_temp": true,
44
+ "asentmax_init_temp": 1.0
45
+ }
epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_noqknorm_s42/run_20260922_225813/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2faf4713ccf4150cf04dd94ebd11dfdeac2b2e6cae3cdd3faecc64ddf53edb25
3
+ size 717998843
epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_rope1e4_s42/run_20260922_225809/checkpoint-44000/config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 10000.0,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "rope_train_position_scale_set": null,
20
+ "attention_bias": false,
21
+ "attention_dropout": 0.0,
22
+ "bos_token_id": 151643,
23
+ "eos_token_id": 151643,
24
+ "checkpoint_activations": false,
25
+ "attention_type": "sparse_stochastic",
26
+ "sparse_block_size": 128,
27
+ "sparse_num_random_blocks": 63,
28
+ "sparse_mask_pool_size": 10000,
29
+ "sparse_max_seq_length": 8192,
30
+ "sparse_layer_indices": null,
31
+ "sparse_mask_cache_dir": "../mask_cache_8k_mixed",
32
+ "sparse_mask_seed": 0,
33
+ "sparse_mixed_nrb_set": [
34
+ 63,
35
+ 47,
36
+ 31,
37
+ 21,
38
+ 11
39
+ ],
40
+ "moba_block_size": 128,
41
+ "moba_top_k": 32,
42
+ "asentmax_alpha": 1.5,
43
+ "asentmax_learn_temp": true,
44
+ "asentmax_init_temp": 1.0
45
+ }
epfl_final/SparseAtt/models/REGIME_20260922/365m_mixed_rope1e4_s42/run_20260922_225809/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b2e96c28040c0d08cf64b2fa0ddaa002412d6c77090f2a8333372c6b32179e3
3
+ size 718031899
epfl_final/checkpoints/a100_40gb_vram_probe_p28/run_20260819_142832/checkpoint-2/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.28,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/a100_40gb_vram_probe_p28/run_20260819_142832/checkpoint-2/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ccb937c4279eb22de3ed231a8653290c91cffe2b1d427b482a22f48f19309e9b
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p00/run_20260807_080949/checkpoint-44300/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/dropctl_calib_p00/run_20260807_080949/checkpoint-44300/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5ea2f029f75340174b35384f28a198a66dbda35f272b2c73c81184d70890781
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p05/run_20260807_080949/checkpoint-44300/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.05,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/dropctl_calib_p05/run_20260807_080949/checkpoint-44300/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f376449165e75b2aacc5104866dacc175204974028657ada0731d09201cd901
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p10/run_20260807_080949/checkpoint-44300/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.1,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/dropctl_calib_p10/run_20260807_080949/checkpoint-44300/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09a9fc02efb94df35f1bbcc369370779984fec5fdf67cb2854bfb076006d0dde
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p20/run_20260807_080949/checkpoint-44300/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.2,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/dropctl_calib_p20/run_20260807_080949/checkpoint-44300/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0fff07fa2dfc5a2a3e55ee402f2cd7af4d5e3fcae5e80db2edfbaa69b55cad6
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_034938/checkpoint-44100/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.28,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_034938/checkpoint-44100/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd27c772be152aa0747d436d06700d9ed9b0d3a1da224a4f53e3e8b026143643
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_043051/checkpoint-44300/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.28,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/dropctl_calib_p28_verify/run_20260819_043051/checkpoint-44300/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f3ed0e1bac976a0f41557786dde6ea527a726bd8857f5d1b57f86c13ee40bcf
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_034938/checkpoint-44100/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.3,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_034938/checkpoint-44100/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00720e6a0c57b43e2945d20f369ee1d5199fb02345fa1bdec61cad5eff68e5f6
3
+ size 718031899
epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_043051/checkpoint-44300/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.3,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/dropctl_calib_p30_verify/run_20260819_043051/checkpoint-44300/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0523c1bbef295b46b60845f6b502f6bbe69ccbaba6a0af6b52ada648facce78
3
+ size 718031899
epfl_final/checkpoints/dropctl_diag_p28_20/run_20260819_041508/checkpoint-44020/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.28,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/dropctl_diag_p28_20/run_20260819_041508/checkpoint-44020/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19604b7e6c13480549d61acca943b2cad1aec6d472154f57392c69c51dc338a7
3
+ size 718031899
epfl_final/checkpoints/l30k1_2b_mixed_to_dense_20pct/run_20260730_180750/checkpoint-37465/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": true,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 42
30
+ }
epfl_final/checkpoints/l30k1_2b_mixed_to_dense_20pct/run_20260730_180750/checkpoint-37465/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49c3c3a3bbe8f4b7fd3340a73952b890fc7a88523a7f758f06009ef0cb6cd712
3
+ size 2404819483
epfl_final/checkpoints/l30k1_2b_moba/run_20260718_031723/checkpoint-37465/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": true,
22
+ "attention_type": "moba",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 120,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/l30k1_2b_moba/run_20260718_031723/checkpoint-37465/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4cd17afe6080858f161c81d0189ea2e8aa674d50090b73490771f6fa64fd3749
3
+ size 2404819483
epfl_final/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ebfddc2cd0dc08ba3fcde95d4aa1ac322cf66d58f63249812187f459c8b39c4
3
+ size 718031899
epfl_final/checkpoints/l30k365m_dense_seed43/run_20260725_075143/checkpoint-3000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/l30k365m_dense_seed43/run_20260725_075143/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1bafa44f58a5aa5eaad538a572940a1474dcee7d12cd0dfb2963b1e57518f09
3
+ size 718031899
epfl_final/checkpoints/l30k365m_dense_seed44/run_20260725_075209/checkpoint-3000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": null,
31
+ "moba_block_size": 128,
32
+ "moba_top_k": 32,
33
+ "asentmax_alpha": 1.5,
34
+ "asentmax_learn_temp": true,
35
+ "asentmax_init_temp": 1.0
36
+ }
epfl_final/checkpoints/l30k365m_dense_seed44/run_20260725_075209/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b7a0f50b9e10a985c0f3d13df96899e823fb7c92b04bcf4658db1715a5c0f3b4
3
+ size 718031899