periodic checkpoint @ step 7600

README.md

@@ -9,7 +9,7 @@ tags:
 
 # d
 
-In-training periodic checkpoint (step 7500). Architecture:
+In-training periodic checkpoint (step 7600). Architecture:
 Kimi K3 mini port (`KimiLinearForCausalLM`).
 
 **Not a finished model** -- intermediate weights uploaded during training for safekeeping/diffing.

k3_2b-latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b9bbe8fd064c7f19d907188fffcf5a1f7520acd16c62cc9ee37899e57523d3c
+size 10478273275

metrics.jsonl

@@ -833,3 +833,14 @@
 {"wall": 6740.268, "step": 7490, "loss": 2.152279863754908, "lm_loss": 2.152279863754908, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3159566819667816, "tokens": 2208890880, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.498046875, "loss_smoothed": 2.205030154188474, "seq_len": 4096}
 {"wall": 6947.098, "eval_loss": 1.904066413640976, "eval_ppl": 6.713137626647949, "step": 7500}
 {"wall": 6947.1, "step": 7500, "loss": 2.268050566315651, "lm_loss": 2.268050566315651, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.31010136008262634, "tokens": 2211840000, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.498046875, "loss_smoothed": 2.227519405881564, "seq_len": 4096}
+{"wall": 7229.589, "step": 7510, "loss": 2.2065874338150024, "lm_loss": 2.2065874338150024, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.33978474140167236, "tokens": 2214789120, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.202607789138953, "seq_len": 4096}
+{"wall": 7434.265, "step": 7520, "loss": 2.280862679084142, "lm_loss": 2.280862679084142, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.2641179859638214, "tokens": 2217738240, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.2242303557693957, "seq_len": 4096}
+{"wall": 7638.993, "step": 7530, "loss": 2.0638744135697684, "lm_loss": 2.0638744135697684, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.34064438939094543, "tokens": 2220687360, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.1932508826255797, "seq_len": 4096}
+{"wall": 7843.935, "step": 7540, "loss": 2.258356382449468, "lm_loss": 2.258356382449468, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3244863450527191, "tokens": 2223636480, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.1596314653754236, "seq_len": 4096}
+{"wall": 8048.28, "step": 7550, "loss": 2.256014421582222, "lm_loss": 2.256014421582222, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.36427590250968933, "tokens": 2226585600, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.220189055800438, "seq_len": 4096}
+{"wall": 8252.945, "step": 7560, "loss": 2.2816530664761863, "lm_loss": 2.2816530664761863, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3668995499610901, "tokens": 2229534720, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.2056768750151, "seq_len": 4096}
+{"wall": 8458.297, "step": 7570, "loss": 2.1709067448973656, "lm_loss": 2.1709067448973656, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.29076918959617615, "tokens": 2232483840, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.2029625174899894, "seq_len": 4096}
+{"wall": 8662.757, "step": 7580, "loss": 2.24979467689991, "lm_loss": 2.24979467689991, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.37173184752464294, "tokens": 2235432960, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.2057685454686484, "seq_len": 4096}
+{"wall": 8867.914, "step": 7590, "loss": 2.27835022409757, "lm_loss": 2.27835022409757, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3151431977748871, "tokens": 2238382080, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.182744794587294, "seq_len": 4096}
+{"wall": 9075.259, "eval_loss": 1.906115546822548, "eval_ppl": 6.726907730102539, "step": 7600}
+{"wall": 9075.26, "step": 7600, "loss": 2.1286584635575614, "lm_loss": 2.1286584635575614, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.2787507176399231, "tokens": 2241331200, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 85.845703125, "loss_smoothed": 2.2174641465147333, "seq_len": 4096}