Archive FINAL4 Part 4 rope1e4 final checkpoint

FINAL4_20260927/rope1e4/365m_mixed_rope1e4_qkgainfp32_s42/run_20261002_022706/checkpoint-44000/config.json

@@ -0,0 +1,48 @@
+{
+  "vocab_size": 151936,
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_hidden_layers": 28,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "max_position_embeddings": 32768,
+  "initializer_range": 0.02,
+  "rms_norm_eps": 1e-06,
+  "use_cache": true,
+  "tie_word_embeddings": true,
+  "rope_theta": 10000.0,
+  "rope_scaling": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "norm_gain_fp32": false,
+  "qk_gain_mode": "vector",
+  "rope_train_position_scale_set": null,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "checkpoint_activations": false,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 63,
+  "sparse_mask_pool_size": 10000,
+  "sparse_max_seq_length": 8192,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "../mask_cache_8k_mixed",
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}

FINAL4_20260927/rope1e4/365m_mixed_rope1e4_qkgainfp32_s42/run_20261002_022706/checkpoint-44000/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5539dd35647f53c211dfd000c9b026fe287a3c602508a750986a8d1739e6cbf4
+size 718046299

FINAL4_20260927/rope1e4/365m_mixed_rope1e4_qkgainfp32_s42/run_20261002_022706/checkpoint-44000/train_config.json

@@ -0,0 +1,73 @@
+{
+  "model_path": "../qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "../../data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "data_seed": 42,
+  "concatenate_seeded_8k": false,
+  "output_dir": "models/FINAL4_20260927/rope1e4/365m_mixed_rope1e4_qkgainfp32_s42/run_20261002_022706",
+  "resume_from_checkpoint": null,
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 2000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 100,
+  "seed": 42,
+  "verbose": false,
+  "checkpoint_activations": false,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m_mixed_rope1e4_qkgainfp32_s42",
+  "wandb_tags": [
+    "final4_20260927",
+    "rope1e4",
+    "mixed",
+    "seed42"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_theta": 10000.0,
+  "rope_train_position_scale_set": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "norm_gain_fp32": false,
+  "qk_gain_mode": "vector",
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 63,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "../mask_cache_8k_mixed",
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}

FINAL4_20260927/rope1e4/365m_mixed_rope1e4_qkgainfp32_s42/run_20261002_022706/checkpoint-44000/trainer_state.json

@@ -0,0 +1,9 @@
+{
+  "global_step": 44000,
+  "epoch": 0,
+  "micro_step": 440000,
+  "sampler_seed": 42,
+  "seed": 42,
+  "world_size": 2,
+  "optimizer_state_file": "training_state.pt"
+}