Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
8445663
·
verified ·
1 Parent(s): 349dacb

EPFL final checkpoints and W&B history 101-150 of 1463

Browse files
Files changed (50) hide show
  1. epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed42/run_20260731_094221/checkpoint-44000/config.json +30 -0
  2. epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed42/run_20260731_094221/checkpoint-44000/model.pt +3 -0
  3. epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed43/run_20260731_094058/checkpoint-44000/config.json +30 -0
  4. epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed43/run_20260731_094058/checkpoint-44000/model.pt +3 -0
  5. epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed44/run_20260731_094255/checkpoint-44000/config.json +30 -0
  6. epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed44/run_20260731_094255/checkpoint-44000/model.pt +3 -0
  7. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed42/run_20260731_001031/checkpoint-44000/config.json +30 -0
  8. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed42/run_20260731_001031/checkpoint-44000/model.pt +3 -0
  9. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed43/run_20260731_001031/checkpoint-44000/config.json +30 -0
  10. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed43/run_20260731_001031/checkpoint-44000/model.pt +3 -0
  11. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed44/run_20260731_001031/checkpoint-44000/config.json +30 -0
  12. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed44/run_20260731_001031/checkpoint-44000/model.pt +3 -0
  13. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed45/run_20260731_001031/checkpoint-44000/config.json +30 -0
  14. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed45/run_20260731_001031/checkpoint-44000/model.pt +3 -0
  15. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed46/run_20260731_093823/checkpoint-44000/config.json +30 -0
  16. epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed46/run_20260731_093823/checkpoint-44000/model.pt +3 -0
  17. epfl_final/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000/config.json +35 -0
  18. epfl_final/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000/model.pt +3 -0
  19. epfl_final/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-8000/config.json +30 -0
  20. epfl_final/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-8000/model.pt +3 -0
  21. epfl_final/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-8000/config.json +30 -0
  22. epfl_final/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-8000/model.pt +3 -0
  23. epfl_final/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-44000/config.json +30 -0
  24. epfl_final/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-44000/model.pt +3 -0
  25. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-44000/config.json +37 -0
  26. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-44000/model.pt +3 -0
  27. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-44000/config.json +37 -0
  28. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-44000/model.pt +3 -0
  29. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-44000/config.json +37 -0
  30. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-44000/model.pt +3 -0
  31. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-44000/config.json +37 -0
  32. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-44000/model.pt +3 -0
  33. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-44000/config.json +37 -0
  34. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-44000/model.pt +3 -0
  35. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed47/run_20260822_133133/checkpoint-44000/config.json +37 -0
  36. epfl_final/checkpoints/l8k365m_resampled_nrb35_seed47/run_20260822_133133/checkpoint-44000/model.pt +3 -0
  37. epfl_final/checkpoints/l8k365m_resampled_seed43/run_20260802_194724/checkpoint-3000/config.json +30 -0
  38. epfl_final/checkpoints/l8k365m_resampled_seed43/run_20260802_194724/checkpoint-3000/model.pt +3 -0
  39. epfl_final/checkpoints/l8k365m_resampled_seed44/run_20260802_194724/checkpoint-3000/config.json +30 -0
  40. epfl_final/checkpoints/l8k365m_resampled_seed44/run_20260802_194724/checkpoint-3000/model.pt +3 -0
  41. epfl_final/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-44000/config.json +37 -0
  42. epfl_final/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-44000/model.pt +3 -0
  43. epfl_final/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-44000/config.json +37 -0
  44. epfl_final/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-44000/model.pt +3 -0
  45. epfl_final/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-44000/config.json +37 -0
  46. epfl_final/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-44000/model.pt +3 -0
  47. epfl_final/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-44000/config.json +37 -0
  48. epfl_final/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-44000/model.pt +3 -0
  49. epfl_final/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-44000/config.json +37 -0
  50. epfl_final/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-44000/model.pt +3 -0
epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed42/run_20260731_094221/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed42/run_20260731_094221/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96b408ce44fd1c84138a9ddb974e10515016a3e96391648f21003ccc040cfdc7
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed43/run_20260731_094058/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed43/run_20260731_094058/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40aa7497f78d5193bcf215c17d45035eefd5786b2b96145ae2c684f9f1d8a8d8
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed44/run_20260731_094255/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_10pct_seed44/run_20260731_094255/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2496fcf625393aaf46df665d295092295d642d9775df77e7ca88462c75c77a84
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed42/run_20260731_001031/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed42/run_20260731_001031/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9255a4fafbd8393ac459cf288dd37fdcb01efe4e31a0d0bd3fd7b02ee15960e2
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed43/run_20260731_001031/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed43/run_20260731_001031/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c92b33304f2fcf1b7dc72b8c04489ba9d2b2c5ca7abfe0453837cb7c474e712
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed44/run_20260731_001031/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed44/run_20260731_001031/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0b63d55254f129c03ca456b32661ae8deee9b8e7b855ae33a0704f9708c13bd
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed45/run_20260731_001031/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed45/run_20260731_001031/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74c00734f7f2cb10ffd19f39697071b1dd79f3b388f4d32c09a5f9df930412bb
3
+ size 718031899
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed46/run_20260731_093823/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_mixed_to_dense_20pct_seed46/run_20260731_093823/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04173da79e74a165fdcca59117d355d549ded5a7c7bb4aef3ed9a04fc4a0c33d
3
+ size 718031899
epfl_final/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "moba",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 1,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": null,
29
+ "sparse_mask_seed": 0,
30
+ "moba_block_size": 128,
31
+ "moba_top_k": 32,
32
+ "asentmax_alpha": 1.5,
33
+ "asentmax_learn_temp": true,
34
+ "asentmax_init_temp": 1.0
35
+ }
epfl_final/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30f0db676a4e29fcb2ef3a620c3f223b55dc5cf42a7e259b272bf18935ac2b1d
3
+ size 718031899
epfl_final/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-8000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32",
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-8000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b9ed3c114684bdf3f828d946fcf25a26905251a68079ee0ba9c46a5e663e00e
3
+ size 718031899
epfl_final/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-8000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 100,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32",
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-8000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:359f0e67562306ea667ea81d84989eac848a49e3aff26644067fd4a25502caa2
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-44000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32",
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62acbaf1983730cf94bc07adb1ec04a4ef474e35b79f75a3c12f1010e1b590c3
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f348f0c5e6648cf51bfe3af418caf7017515737c15bc4759c74fe22aaff98a9
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82cea46906efa6a143441ec44bbc61e798e184002f9a9169ff023bd434d6db3c
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ae84d4f894e73ed3ba1b9acd7e3737c04859a4ba6d69675c19194f3c149fc95
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ddbd35785b6029d05403c4f7b90f5a06c5872ddaa5b066707f5ba638ce1eea3
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce12d02818ea07601e41ea52bfd65d34ddd50ce8510d48e64943cec425abd654
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed47/run_20260822_133133/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 10000,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0",
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_resampled_nrb35_seed47/run_20260822_133133/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4577cf164f76662e90d944e0c6b453f24d6bf29288d2869ef4c1344dc8e84afd
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_seed43/run_20260802_194724/checkpoint-3000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32",
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_resampled_seed43/run_20260802_194724/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f84c81c35ab94583477c8f2b45b798c76c8af09a1e76b2e8c9afab7ffb95586a
3
+ size 718031899
epfl_final/checkpoints/l8k365m_resampled_seed44/run_20260802_194724/checkpoint-3000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32",
29
+ "sparse_mask_seed": 0
30
+ }
epfl_final/checkpoints/l8k365m_resampled_seed44/run_20260802_194724/checkpoint-3000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51acacd8a9bd671d99b36942613b9e3ee9f2723106a0259bf91650f89d458b98
3
+ size 718031899
epfl_final/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1",
30
+ "sparse_mask_seed": 1,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ebb3b159539e5f77c608b469429544ff28cedc84873fb2f5a6eff41ed1b81ca
3
+ size 718031899
epfl_final/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1",
30
+ "sparse_mask_seed": 1,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:245e18bfadc783060f1a9035165cd53e1a04348da5260491726809cb149aed23
3
+ size 718031899
epfl_final/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1",
30
+ "sparse_mask_seed": 1,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:452fa8181bc429b7b253e226d5e4f5c11c23428fcd863dfca0a0fcc83786836c
3
+ size 718031899
epfl_final/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1",
30
+ "sparse_mask_seed": 1,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df9f4c16519938e5a3ea55cb40601a8c9457bb609575fc03e07bf8a611b2e86a
3
+ size 718031899
epfl_final/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-44000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 35,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1",
30
+ "sparse_mask_seed": 1,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
epfl_final/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ced499803cde1f809112826ff90b64c14487c3e67150e103bd821adbe3e2ebff
3
+ size 718031899