Upload seed-72 full resume checkpoint

FINAL5_20261014/seed72_dense/checkpoint-20000/config.json

@@ -0,0 +1,42 @@
+{
+  "vocab_size": 151936,
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_hidden_layers": 28,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "max_position_embeddings": 32768,
+  "initializer_range": 0.02,
+  "rms_norm_eps": 1e-06,
+  "use_cache": true,
+  "tie_word_embeddings": true,
+  "rope_theta": 1000000,
+  "rope_scaling": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "norm_gain_fp32": false,
+  "qk_gain_mode": "vector",
+  "rope_train_position_scale_set": null,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "checkpoint_activations": false,
+  "attention_type": "dense",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_max_seq_length": 8192,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}

FINAL5_20261014/seed72_dense/checkpoint-20000/loader_gen_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b21b4160791b11c535b1414390199117f5857a94ac58d7b970468583a72bb0bc
+size 6802

FINAL5_20261014/seed72_dense/checkpoint-20000/loader_gen_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ec129fe94d28aa0ddb1827422f7723c72b31ebc7a9306e98894b6131bb7fe9
+size 6802

FINAL5_20261014/seed72_dense/checkpoint-20000/loader_gen_state_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5588d56d5c8519bb2c45d4b62d3684da8142cd70467b7c6b989cd944f4e533
+size 6802

FINAL5_20261014/seed72_dense/checkpoint-20000/loader_gen_state_rank3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05faccfef6d5dfcc8dd8aca9b50fb4a4f1b5e5ee52a9a29e99e1e68bd3e62b77
+size 6802

FINAL5_20261014/seed72_dense/checkpoint-20000/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d940e694bf6f47dd65d802a64c48dcd50a91f13bb139fcf1375937fbaffb7eb4
+size 2404833883

FINAL5_20261014/seed72_dense/checkpoint-20000/rng_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa8f3f2527a73f03263b1714be4aa8be5be0e5644561efb31e37452dcb230636
+size 14821

FINAL5_20261014/seed72_dense/checkpoint-20000/rng_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:892527a26e05f13ce99dd25d7deecef48be5561b60d5cb549df9385df1704cb9
+size 14885

FINAL5_20261014/seed72_dense/checkpoint-20000/rng_state_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50617cc1166cd88f7dc063da3765c88e8b238cc8a34ab7aeeaa7be12a7296e61
+size 14821

FINAL5_20261014/seed72_dense/checkpoint-20000/rng_state_rank3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ff4c836741720716968802bd438c08888f256ab52b5e21f2897ee8d810f75de
+size 14885

FINAL5_20261014/seed72_dense/checkpoint-20000/train_config.json

@@ -0,0 +1,68 @@
+{
+  "model_path": "../qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "../../data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "data_seed": 72,
+  "concatenate_seeded_8k": false,
+  "output_dir": "models/FINAL5_20261014/1p2b_dense_qkgainfp32_s72/run_20261014_191331",
+  "resume_from_checkpoint": "models/FINAL5_20261014/1p2b_dense_qkgainfp32_s72/run_20261014_191331/checkpoint-2000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 5000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "resume_micro_step_override": 30000,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "FINAL5 1.2B Dense seed 72",
+  "wandb_tags": [
+    "FINAL5",
+    "1p2b",
+    "dense",
+    "seed72",
+    "qk_gain_fp32"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": 28,
+  "head_dim": 128,
+  "attention_dropout": 0.0,
+  "rope_theta": 1000000,
+  "rope_train_position_scale_set": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "norm_gain_fp32": false,
+  "qk_gain_mode": "vector",
+  "attention_type": "dense",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": false,
+  "dense_step_intervals": null
+}

FINAL5_20261014/seed72_dense/checkpoint-20000/trainer_state.json

@@ -0,0 +1,9 @@
+{
+  "global_step": 20000,
+  "epoch": 0,
+  "micro_step": 300000,
+  "sampler_seed": 72,
+  "seed": 72,
+  "world_size": 4,
+  "optimizer_state_file": "training_state.pt"
+}

FINAL5_20261014/seed72_dense/checkpoint-20000/training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4eb34c63137098695159f5ff02d832f15b07a555af95406f407338020b89428
+size 4809712479