Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
aa8d5fc
·
verified ·
1 Parent(s): f05d2eb

Add 1.2B Dense seed-70 step-25000 resume checkpoint for RCP transfer

Browse files
Files changed (13) hide show
  1. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/config.json +42 -0
  2. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank0.pt +3 -0
  3. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank1.pt +3 -0
  4. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank2.pt +3 -0
  5. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank3.pt +3 -0
  6. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/model.pt +3 -0
  7. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank0.pt +3 -0
  8. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank1.pt +3 -0
  9. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank2.pt +3 -0
  10. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank3.pt +3 -0
  11. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/train_config.json +69 -0
  12. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/trainer_state.json +9 -0
  13. checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/training_state.pt +3 -0
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "rope",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": true,
20
+ "norm_gain_fp32": false,
21
+ "qk_gain_mode": "vector",
22
+ "rope_train_position_scale_set": null,
23
+ "attention_bias": false,
24
+ "attention_dropout": 0.0,
25
+ "bos_token_id": 151643,
26
+ "eos_token_id": 151643,
27
+ "checkpoint_activations": true,
28
+ "attention_type": "sparse_stochastic",
29
+ "sparse_block_size": 128,
30
+ "sparse_num_random_blocks": 31,
31
+ "sparse_mask_pool_size": 1,
32
+ "sparse_max_seq_length": 8192,
33
+ "sparse_layer_indices": null,
34
+ "sparse_mask_cache_dir": null,
35
+ "sparse_mask_seed": 0,
36
+ "sparse_mixed_nrb_set": null,
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3d3d87a844d4fd6548dfc4c1235a1ee3e262506bc6260da75e7cbf9d9da819c
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2c029e9127ff4c346ab99e6af8377a6903003909fec0b073b70bfafc01d4517
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:917e752ef93a10f1da3dc49f9c18adc43ae1bb568ee13d528c9824ee4513b511
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:389dee6373df6f406c9ae897b97bc5acc1b426bc8c9810e2754f7d2f892a256f
3
+ size 6802
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4df93f77562ecb5b28a50c27b3e10c84e0b10609736333f76621d3c4a811cd3d
3
+ size 2404833883
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f92dc12642f0e396c9aea648c347fadfcab8234a07ae545439e4a1e6521ed11
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59241ebd08eebd96b51f7a9bb0ab7333fc4665a17b548a25d73e390cb957c223
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f65c72fae8bdf73a4b4cd50d429dea879b957113952938b6982c2be4d55abd9
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ed370cea6b4dd78fdab7056daa262fcf7c158dc34805993101942cfeec1072d
3
+ size 14821
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/train_config.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_path": "../qwen3_06b_base",
3
+ "model_dtype": "bfloat16",
4
+ "init_from_scratch": true,
5
+ "train_path": "../../data/fineweb_large/tokenized",
6
+ "val_path": null,
7
+ "max_seq_length": 8192,
8
+ "num_workers": 1,
9
+ "data_seed": 70,
10
+ "concatenate_seeded_8k": false,
11
+ "output_dir": "models/FINAL3_20260927/1p2b_dense_qkgainfp32_s70_trillium_resume_4gpu_ckpt6000/run_20260930_ckpt6000",
12
+ "resume_from_checkpoint": "models/FINAL3_20260927/1p2b_dense_qkgainfp32_s70_trillium_resume_4gpu_ckpt6000/run_20260930_ckpt6000/checkpoint-6000",
13
+ "force_new_run": false,
14
+ "num_epochs": 1,
15
+ "max_steps": 50462,
16
+ "per_device_batch_size": 1,
17
+ "gradient_accumulation_steps": 15,
18
+ "learning_rate": 0.0005,
19
+ "weight_decay": 0.01,
20
+ "warmup_steps": 750,
21
+ "lr_scheduler": "cosine",
22
+ "max_grad_norm": 1.0,
23
+ "save_steps": 1000,
24
+ "eval_steps": 250,
25
+ "eval_batches": 50,
26
+ "logging_steps": 10,
27
+ "seed": 70,
28
+ "verbose": true,
29
+ "checkpoint_activations": true,
30
+ "resume_micro_step_override": 90000,
31
+ "strategy": "ddp",
32
+ "mixed_precision": "bf16",
33
+ "wandb_enabled": false,
34
+ "wandb_project": "stochastic_sparsity",
35
+ "wandb_name": "1p2b_dense_qkgainfp32_s70",
36
+ "wandb_tags": [
37
+ "qwen3",
38
+ "1.2b",
39
+ "8k",
40
+ "p2",
41
+ "dense",
42
+ "seed70"
43
+ ],
44
+ "hidden_size": 1536,
45
+ "intermediate_size": 6144,
46
+ "num_attention_heads": 12,
47
+ "num_key_value_heads": 4,
48
+ "num_hidden_layers": 28,
49
+ "head_dim": null,
50
+ "attention_dropout": null,
51
+ "rope_theta": null,
52
+ "rope_train_position_scale_set": null,
53
+ "positional": "rope",
54
+ "qk_norm": "on",
55
+ "qk_gain_fp32": true,
56
+ "norm_gain_fp32": false,
57
+ "qk_gain_mode": "vector",
58
+ "attention_type": "sparse_stochastic",
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0,
66
+ "sparse_mixed_nrb_set": null,
67
+ "dense_mode": true,
68
+ "dense_step_intervals": null
69
+ }
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/trainer_state.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "global_step": 25000,
3
+ "epoch": 0,
4
+ "micro_step": 375000,
5
+ "sampler_seed": 70,
6
+ "seed": 70,
7
+ "world_size": 4,
8
+ "optimizer_state_file": "training_state.pt"
9
+ }
checkpoints/FINAL3_20260927/part_t/1p2b_dense_qkgainfp32_s70/checkpoint-25000-trillium/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c3000852374392d07739338bad444dd3d79b524e2542dbb0d917591826c80c4
3
+ size 4809712479