Sync Seed-72 Mixed checkpoint-44000

FINAL5_20261014/seed72_mixed/checkpoint-44000/config.json

@@ -0,0 +1,46 @@
+{
+  "vocab_size": 151936,
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_hidden_layers": 28,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "max_position_embeddings": 32768,
+  "initializer_range": 0.02,
+  "rms_norm_eps": 1e-06,
+  "use_cache": true,
+  "tie_word_embeddings": true,
+  "rope_theta": 1000000,
+  "rope_scaling": null,
+  "positional": "rope",
+  "qk_gain_fp32": true,
+  "fp32_master": "none",
+  "rope_train_position_scale_set": null,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "checkpoint_activations": false,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_max_seq_length": 8192,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "../../sparse/mask_cache_8k_mixed_seed72",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}

FINAL5_20261014/seed72_mixed/checkpoint-44000/loader_epoch_start_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0656ddbff730b3ef739cdb4d980f54ff169c931544ce7a093c21a7dc6fa6f49e
+size 6858

FINAL5_20261014/seed72_mixed/checkpoint-44000/loader_epoch_start_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8f9144eba7ef521ca55f126341e4d11d4584a2c550212432b63364585392f0d
+size 6858

FINAL5_20261014/seed72_mixed/checkpoint-44000/loader_epoch_start_state_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:587ab30e72f1d5694ef66c539c75743ea5b18c1d61abb5f03cfa4b66d9ef6119
+size 6858

FINAL5_20261014/seed72_mixed/checkpoint-44000/loader_gen_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b21b4160791b11c535b1414390199117f5857a94ac58d7b970468583a72bb0bc
+size 6802

FINAL5_20261014/seed72_mixed/checkpoint-44000/loader_gen_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ec129fe94d28aa0ddb1827422f7723c72b31ebc7a9306e98894b6131bb7fe9
+size 6802

FINAL5_20261014/seed72_mixed/checkpoint-44000/loader_gen_state_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5588d56d5c8519bb2c45d4b62d3684da8142cd70467b7c6b989cd944f4e533
+size 6802

FINAL5_20261014/seed72_mixed/checkpoint-44000/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93e49691cc61baecf002591de2456b35b24c4088126ae0a62bd199bd2e4f4bb9
+size 2404833883

FINAL5_20261014/seed72_mixed/checkpoint-44000/rng_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5511be6d04d8f7e269fb0e5e4891be12cb296d1910425f419edec1f656a98bf7
+size 14821

FINAL5_20261014/seed72_mixed/checkpoint-44000/rng_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61070ecc5635d95a84794e5e85c825aa18c4e007f303d7522ef3d55f429e7c2b
+size 14821

FINAL5_20261014/seed72_mixed/checkpoint-44000/rng_state_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:848353d56afb2726c65fdbd03b7aeb4fd39fe162f825849c0d5d81939677cc0d
+size 14821

FINAL5_20261014/seed72_mixed/checkpoint-44000/training_config.json

@@ -0,0 +1,74 @@
+{
+  "model_path": "../../qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "../../data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 0,
+  "data_seed": 72,
+  "output_dir": "models/FINAL5_20261014/1p2b_mixed_qkgainfp32_s72/run_20261015_193033",
+  "resume_from_checkpoint": "models/FINAL5_20261014/1p2b_mixed_qkgainfp32_s72/run_20261015_193033/checkpoint-41000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 20,
+  "global_batch_size": 60,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "resume_micro_step_override": 820000,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "FINAL5_20261014 1p2b_mixed_qkgainfp32_s72",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "qk_gain_fp32": true,
+  "positional": "rope",
+  "fp32_master": "none",
+  "keep_last_checkpoints": 3,
+  "keep_every_n_steps": 10000,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "../../sparse/mask_cache_8k_mixed_seed72",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}

FINAL5_20261014/seed72_mixed/checkpoint-44000/training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4986e8cae219ef44b69b95dd406fc5fd830e63108ac7759a14712e49f62b9c5
+size 4809712543