Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
a221faa
·
verified ·
1 Parent(s): 160bc7c

Archive FINAL5 seed72_dense checkpoint checkpoint-50462

Browse files
Files changed (16) hide show
  1. FINAL5_20261014/seed72_dense/checkpoint-50462/config.json +40 -0
  2. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank0.pt +3 -0
  3. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank1.pt +3 -0
  4. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank2.pt +3 -0
  5. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank3.pt +3 -0
  6. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank0.pt +3 -0
  7. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank1.pt +3 -0
  8. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank2.pt +3 -0
  9. FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank3.pt +3 -0
  10. FINAL5_20261014/seed72_dense/checkpoint-50462/model.pt +3 -0
  11. FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank0.pt +3 -0
  12. FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank1.pt +3 -0
  13. FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank2.pt +3 -0
  14. FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank3.pt +3 -0
  15. FINAL5_20261014/seed72_dense/checkpoint-50462/training_config.json +68 -0
  16. FINAL5_20261014/seed72_dense/checkpoint-50462/training_state.pt +3 -0
FINAL5_20261014/seed72_dense/checkpoint-50462/config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_gain_fp32": true,
19
+ "fp32_master": "none",
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "sparse_stochastic",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 31,
29
+ "sparse_mask_pool_size": 1,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": null,
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": null,
35
+ "moba_block_size": 128,
36
+ "moba_top_k": 32,
37
+ "asentmax_alpha": 1.5,
38
+ "asentmax_learn_temp": true,
39
+ "asentmax_init_temp": 1.0
40
+ }
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0656ddbff730b3ef739cdb4d980f54ff169c931544ce7a093c21a7dc6fa6f49e
3
+ size 6858
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8f9144eba7ef521ca55f126341e4d11d4584a2c550212432b63364585392f0d
3
+ size 6858
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:587ab30e72f1d5694ef66c539c75743ea5b18c1d61abb5f03cfa4b66d9ef6119
3
+ size 6858
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_epoch_start_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:236c4c8797d7122d7a05793a5caef425fe4459108e266a9aaa7548d90e8340a5
3
+ size 6858
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b21b4160791b11c535b1414390199117f5857a94ac58d7b970468583a72bb0bc
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32ec129fe94d28aa0ddb1827422f7723c72b31ebc7a9306e98894b6131bb7fe9
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c5588d56d5c8519bb2c45d4b62d3684da8142cd70467b7c6b989cd944f4e533
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-50462/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05faccfef6d5dfcc8dd8aca9b50fb4a4f1b5e5ee52a9a29e99e1e68bd3e62b77
3
+ size 6802
FINAL5_20261014/seed72_dense/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4895f6f66039c8a96ec744b89f0307cd0f9a289f401503393208c023f93789e
3
+ size 2404833883
FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1425b013642973443041062ae0a4e4382fa36808b62ff95aee79566184d0de53
3
+ size 14821
FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:890431648449b1827a548c62fb5690748625b604bfbfeb3aa90d77e7aa6352ad
3
+ size 14885
FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1024ecd59cf8c38366eb15e0c061cdca9820f47ba280a3126acfb05240b9f570
3
+ size 14821
FINAL5_20261014/seed72_dense/checkpoint-50462/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff9fc55f2260ff3002b71717bffe0ffbda2705e26e0fd2bf5de40f89e647ae88
3
+ size 14885
FINAL5_20261014/seed72_dense/checkpoint-50462/training_config.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_path": "../../qwen3_06b_base",
3
+ "model_dtype": "bfloat16",
4
+ "init_from_scratch": true,
5
+ "train_path": "../../data/fineweb_large/tokenized",
6
+ "val_path": null,
7
+ "max_seq_length": 8192,
8
+ "num_workers": 8,
9
+ "data_seed": 72,
10
+ "output_dir": "models/FINAL5_20261014/1p2b_dense_qkgainfp32_s72/run_20261023_hf_seed72_20k",
11
+ "resume_from_checkpoint": "models/FINAL5_20261014/1p2b_dense_qkgainfp32_s72/run_20261023_hf_seed72_20k/checkpoint-45000",
12
+ "force_new_run": false,
13
+ "num_epochs": 1,
14
+ "max_steps": 50462,
15
+ "per_device_batch_size": 1,
16
+ "gradient_accumulation_steps": 15,
17
+ "global_batch_size": 60,
18
+ "learning_rate": 0.0005,
19
+ "weight_decay": 0.01,
20
+ "warmup_steps": 750,
21
+ "lr_scheduler": "cosine",
22
+ "max_grad_norm": 1.0,
23
+ "save_steps": 1000,
24
+ "eval_steps": 250,
25
+ "eval_batches": 50,
26
+ "logging_steps": 10,
27
+ "seed": 72,
28
+ "verbose": true,
29
+ "checkpoint_activations": false,
30
+ "resume_micro_step_override": null,
31
+ "strategy": "ddp",
32
+ "mixed_precision": "bf16",
33
+ "wandb_enabled": false,
34
+ "wandb_project": "stochastic_sparsity",
35
+ "wandb_name": "FINAL5_20261014 1p2b_dense_qkgainfp32_s72",
36
+ "wandb_tags": [
37
+ "qwen3",
38
+ "1.2b",
39
+ "8k",
40
+ "p2",
41
+ "dense",
42
+ "seed72"
43
+ ],
44
+ "hidden_size": 1536,
45
+ "intermediate_size": 6144,
46
+ "num_attention_heads": 12,
47
+ "num_key_value_heads": 4,
48
+ "num_hidden_layers": 28,
49
+ "head_dim": null,
50
+ "attention_dropout": null,
51
+ "qk_gain_fp32": true,
52
+ "positional": "rope",
53
+ "fp32_master": "none",
54
+ "keep_last_checkpoints": 3,
55
+ "keep_every_n_steps": 10000,
56
+ "rope_train_position_scale_set": null,
57
+ "attention_type": "sparse_stochastic",
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "sparse_layer_indices": null,
63
+ "sparse_mask_cache_dir": null,
64
+ "sparse_mask_seed": 0,
65
+ "sparse_mixed_nrb_set": null,
66
+ "dense_mode": true,
67
+ "dense_step_intervals": null
68
+ }
FINAL5_20261014/seed72_dense/checkpoint-50462/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0228f0c22bb7493f25a7287d8e42e6fe8c6444b740076d55843dc8ed86fdee75
3
+ size 4809712543