Add FINAL5 seed72 Mixed step27000 resume manifest and config

checkpoints/FINAL5_20261014/part_t/1p2b_mixed_qkgainfp32_s72/checkpoint-27000-rcp/RESUME_MANIFEST.md

@@ -0,0 +1,26 @@
+# FINAL5 seed 72 Mixed checkpoint 27000 resume bundle
+
+Captured on 2026-10-28. This is an intermediate resume checkpoint at step 27,000 of 50,462, not a final checkpoint. Do not evaluate it as a completed seed pair.
+
+The archived checkpoint includes model weights, optimizer and scheduler state, the saved training configuration, loader state, and RNG state. It was verified after upload against the source file sizes and SHA-256/LFS hashes (10 of 10 files passed; 7,214,606,594 bytes).
+
+Resume settings: seed 72, qk_gain_fp32=true, maximum step 50,462, global batch 60, per-device batch 1, gradient accumulation 20, and checkpoint cadence 1,000 steps. The saved scheduler LR at step 27,000 was preserved on resume. The three-rank DDP resume used resume_micro_step_override=540000 to restore the epoch-start loader RNG and replay the expected shuffled order.
+
+Training code revision: 9a5045f08a84d089699885e3685fc55d8feb7060; tracked working-tree changes were absent. The exact resume configuration is in ../resume_config_3gpu_global_batch60.json.
+
+The Mixed seed 72 mask cache is already present in this repository at FINAL5_20261014/mask_cache_8k_mixed_seed72/ (manifest plus 28 layer files). The FineWeb-Edu training export and tokenizer are not included in this checkpoint directory and must be available at the paths specified by the resume configuration.
+
+| File | Bytes | SHA-256 |
+|---|---:|---|
+| config.json | 1,153 | 19f448cd58000f7c4c1e368ac043e43e5ac9612188dd2c5df01719255ecabac6 |
+| loader_epoch_start_state_rank0.pt | 6,858 | 0656ddbff730b3ef739cdb4d980f54ff169c931544ce7a093c21a7dc6fa6f49e |
+| loader_epoch_start_state_rank1.pt | 6,858 | c8f9144eba7ef521ca55f126341e4d11d4584a2c550212432b63364585392f0d |
+| loader_gen_state_rank0.pt | 6,802 | b21b4160791b11c535b1414390199117f5857a94ac58d7b970468583a72bb0bc |
+| loader_gen_state_rank1.pt | 6,802 | 32ec129fe94d28aa0ddb1827422f7723c72b31ebc7a9306e98894b6131bb7fe9 |
+| model.pt | 2,404,833,883 | cbddfa7b1fc496462310e94c0140a213b7424d61a29ac7ed1ea37436cd0a8db2 |
+| rng_state_rank0.pt | 14,821 | 60e53db818a76f083feffaac06d1144bc0f16781bd3c027e99c826b473d848e7 |
+| rng_state_rank1.pt | 14,885 | 3c6a920ab76f3c9355035b7fe950d73e6efa67247394e39a1d9f7f46283e0842 |
+| training_config.json | 1,989 | b6a80a40ea56e203679ec84b0e3ab5f3a09d27e390c409ee70ab1eaa6edc75fd |
+| training_state.pt | 4,809,712,543 | 985fc0f558fcb6736ffe50c0e0e5d687dc50c3dd7501bb4f9c54a970374e768e |
+
+The checkpoint was resumed after this archive was made; the 27,000-step checkpoint itself was not evaluated.

checkpoints/FINAL5_20261014/part_t/1p2b_mixed_qkgainfp32_s72/resume_config_3gpu_global_batch60.json

@@ -0,0 +1,80 @@
+{
+  "model": {
+    "path": "../../qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 1536,
+    "intermediate_size": 6144,
+    "num_attention_heads": 12,
+    "num_key_value_heads": 4,
+    "qk_gain_fp32": true
+  },
+  "data": {
+    "train_path": "../../data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 0,
+    "seed": 72
+  },
+  "training": {
+    "output_dir": "models/FINAL5_20261014/1p2b_mixed_qkgainfp32_s72",
+    "resume_from_checkpoint": "models/FINAL5_20261014/1p2b_mixed_qkgainfp32_s72/run_20261015_193033/checkpoint-27000",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 50462,
+    "per_device_batch_size": 1,
+    "gradient_accumulation_steps": 20,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 72,
+    "verbose": true,
+    "checkpoint_activations": false,
+    "global_batch_size": 60,
+    "keep_last_checkpoints": 3,
+    "keep_every_n_steps": 10000,
+    "resume_micro_step_override": 540000
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "FINAL5_20261014 1p2b_mixed_qkgainfp32_s72",
+    "tags": [
+      "qwen3",
+      "1.2b",
+      "8k",
+      "p2",
+      "mixed",
+      "seed72"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": false,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 35,
+    "sparse_mask_pool_size": 10000,
+    "sparse_mixed_nrb_set": [
+      63,
+      47,
+      31,
+      21,
+      11
+    ],
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": "../../sparse/mask_cache_8k_mixed_seed72",
+    "sparse_mask_seed": 72
+  }
+}