Upload seed-72 full resume checkpoint

FINAL5_20261014/seed72_mixed/checkpoint-500/config.json

@@ -0,0 +1,48 @@
+{
+  "vocab_size": 151936,
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_hidden_layers": 28,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "max_position_embeddings": 32768,
+  "initializer_range": 0.02,
+  "rms_norm_eps": 1e-06,
+  "use_cache": true,
+  "tie_word_embeddings": true,
+  "rope_theta": 1000000,
+  "rope_scaling": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "norm_gain_fp32": false,
+  "qk_gain_mode": "vector",
+  "rope_train_position_scale_set": null,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "checkpoint_activations": false,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 63,
+  "sparse_mask_pool_size": 10000,
+  "sparse_max_seq_length": 8192,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "../mask_cache_8k_mixed_seed72",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}

FINAL5_20261014/seed72_mixed/checkpoint-500/loader_gen_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b21b4160791b11c535b1414390199117f5857a94ac58d7b970468583a72bb0bc
+size 6802

FINAL5_20261014/seed72_mixed/checkpoint-500/loader_gen_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ec129fe94d28aa0ddb1827422f7723c72b31ebc7a9306e98894b6131bb7fe9
+size 6802

FINAL5_20261014/seed72_mixed/checkpoint-500/loader_gen_state_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5588d56d5c8519bb2c45d4b62d3684da8142cd70467b7c6b989cd944f4e533
+size 6802

FINAL5_20261014/seed72_mixed/checkpoint-500/loader_gen_state_rank3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05faccfef6d5dfcc8dd8aca9b50fb4a4f1b5e5ee52a9a29e99e1e68bd3e62b77
+size 6802

FINAL5_20261014/seed72_mixed/checkpoint-500/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b770a8cb9c552b65c578ac7b75dace25bd9605df0393f1b97496a1485417a34d
+size 2404833883

FINAL5_20261014/seed72_mixed/checkpoint-500/rng_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39dbfdd367d228cf5aed65e71acc43238df54b5f694f9d64fc362e2f453e3de7
+size 14821

FINAL5_20261014/seed72_mixed/checkpoint-500/rng_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9b9f5d3673667ad563c5c9f9ff1f6bc57ea372aa87cd657912350974b8d16f6
+size 14821

FINAL5_20261014/seed72_mixed/checkpoint-500/rng_state_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e8f435d9c041c20c5944587efbf32fbd2957b3691ededbeb8789d888037714d
+size 14885

FINAL5_20261014/seed72_mixed/checkpoint-500/rng_state_rank3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73ed84c0d54f5aca61e5bad13c90541be48d538d2b3710a9ec5e3cd901e68a84
+size 14757

FINAL5_20261014/seed72_mixed/checkpoint-500/train_config.json

@@ -0,0 +1,74 @@
+{
+  "model_path": "../qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "../../data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 0,
+  "data_seed": 72,
+  "concatenate_seeded_8k": false,
+  "output_dir": "models/FINAL5_20261014/1p2b_mixed_qkgainfp32_s72/run_20261015_193033",
+  "resume_from_checkpoint": null,
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 500,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "FINAL5 1.2B Mixed seed 72",
+  "wandb_tags": [
+    "FINAL5",
+    "1p2b",
+    "mixed",
+    "seed72",
+    "qk_gain_fp32"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": 28,
+  "head_dim": 128,
+  "attention_dropout": 0.0,
+  "rope_theta": 1000000,
+  "rope_train_position_scale_set": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "norm_gain_fp32": false,
+  "qk_gain_mode": "vector",
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 63,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "../mask_cache_8k_mixed_seed72",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}

FINAL5_20261014/seed72_mixed/checkpoint-500/trainer_state.json

@@ -0,0 +1,9 @@
+{
+  "global_step": 500,
+  "epoch": 0,
+  "micro_step": 7500,
+  "sampler_seed": 72,
+  "seed": 72,
+  "world_size": 4,
+  "optimizer_state_file": "training_state.pt"
+}

FINAL5_20261014/seed72_mixed/checkpoint-500/training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b82330a4e61e7165a71e81f187d7f24830a2842693a1d1bedf46c7c9a49ffaaf
+size 4809712479