checkpoint step=16800 val_loss=2.2509

checkpoints/step_00016800/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8c32d090249c6e7c75a677d78fd68d2b5b03d1b1d389f4ab306c47d835b28b1
+size 20672645408

checkpoints/step_00016800/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0acdab1083d2d8b3d40072d8db1b9e912c34cca1940cd1605b2d60da4a1a0c4
+size 20577839045

checkpoints/step_00016800/training_meta.json

@@ -0,0 +1,48 @@
+{
+  "step": 16800,
+  "epoch": 0,
+  "total_tokens": 1101004800,
+  "val_loss": 2.250926058292389,
+  "config": {
+    "vocab_size": 50304,
+    "d_model": 4096,
+    "n_layers": 56,
+    "n_heads": 32,
+    "n_kv_heads": 8,
+    "max_len": 2048,
+    "rope_theta": 5000000.0,
+    "ffn_hidden": 11008,
+    "use_moe": false,
+    "moe_experts": 8,
+    "moe_top_k": 2,
+    "moe_shared": 1,
+    "moe_hidden": 1408,
+    "moe_every": 4,
+    "moe_bias_speed": 0.001,
+    "use_qk_norm": true,
+    "use_structural_bias": true,
+    "n_struct_rel": 4,
+    "use_fim": true,
+    "fim_rate": 0.5,
+    "fim_pre_id": 50269,
+    "fim_suf_id": 50270,
+    "fim_mid_id": 50271,
+    "grad_checkpointing": true,
+    "lr": 0.0001,
+    "min_lr": 1e-05,
+    "weight_decay": 0.1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "grad_clip": 1.0,
+    "warmup_tokens": 500000,
+    "target_total_tokens": 10000000000,
+    "micro_batch": 4,
+    "grad_accum": 8,
+    "log_every": 10,
+    "ckpt_every_steps": 200,
+    "eval_every_steps": 200,
+    "seed": 42,
+    "max_session_hours": 11.5
+  },
+  "timestamp": 1785233177.519315
+}