periodic checkpoint @ step 7400

README.md

@@ -9,7 +9,7 @@ tags:
 
 # d
 
-In-training periodic checkpoint (step 7200). Architecture:
+In-training periodic checkpoint (step 7400). Architecture:
 Kimi K3 mini port (`KimiLinearForCausalLM`).
 
 **Not a finished model** -- intermediate weights uploaded during training for safekeeping/diffing.

metrics.jsonl

@@ -799,3 +799,26 @@
 {"wall": 40255.975, "step": 7190, "loss": 2.208519327143828, "lm_loss": 2.208519327143828, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3396632671356201, "tokens": 2120417280, "adamw_lr": 0.0004, "muon/grad_norm": 0.2572692930698395, "adam/grad_norm": 0.22219517827033997, "muon/param_norm": 1940.748046875, "muon/update_ratio": 0.010280540212988853, "muon/update_ratio_max": 0.08274061977863312, "adam/param_norm": 703.4539794921875, "adam/update_ratio": 0.001100590219721198, "adam/update_ratio_max": 0.002717256546020508, "ademamix/cos_fast_slow": 0.19853557646274567, "ademamix/slow_fast_norm_ratio": 0.14843077957630157, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.998046875, "loss_smoothed": 2.1960854942599934, "seq_len": 4096}
 {"wall": 40463.299, "eval_loss": 1.899511456489563, "eval_ppl": 6.682629108428955, "step": 7200}
 {"wall": 40463.301, "step": 7200, "loss": 2.3936877051989236, "lm_loss": 2.3936877051989236, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.33779776096343994, "tokens": 2123366400, "adamw_lr": 0.0004, "muon/grad_norm": 0.269137442111969, "adam/grad_norm": 0.20412927865982056, "muon/param_norm": 1940.89306640625, "muon/update_ratio": 0.01032662857323885, "muon/update_ratio_max": 0.07882080227136612, "adam/param_norm": 703.4942016601562, "adam/update_ratio": 0.0011091063497588038, "adam/update_ratio_max": 0.0027374238707125187, "ademamix/cos_fast_slow": 0.26110315322875977, "ademamix/slow_fast_norm_ratio": 0.15337790548801422, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.998046875, "loss_smoothed": 2.2821626265843706, "seq_len": 4096}
+{"wall": 158.4, "eval_loss": 1.8995585590600967, "eval_ppl": 6.682943820953369, "step": 7200}
+{"wall": 892.381, "step": 7210, "loss": 2.2340605705976486, "lm_loss": 2.2340605705976486, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.40077805519104004, "tokens": 2126315520, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.245195053021113, "seq_len": 4096}
+{"wall": 1097.884, "step": 7220, "loss": 2.1312559098005295, "lm_loss": 2.1312559098005295, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.2823336124420166, "tokens": 2129264640, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.24267407009999, "seq_len": 4096}
+{"wall": 1303.333, "step": 7230, "loss": 2.2536853005488715, "lm_loss": 2.2536853005488715, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3660520017147064, "tokens": 2132213760, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.1824468585352106, "seq_len": 4096}
+{"wall": 1508.593, "step": 7240, "loss": 2.3294680068890252, "lm_loss": 2.3294680068890252, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.24897290766239166, "tokens": 2135162880, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.230551139762004, "seq_len": 4096}
+{"wall": 1714.051, "step": 7250, "loss": 2.1645777374505997, "lm_loss": 2.1645777374505997, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.5797768235206604, "tokens": 2138112000, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.252466354270776, "seq_len": 4096}
+{"wall": 1919.382, "step": 7260, "loss": 2.3493137458960214, "lm_loss": 2.3493137458960214, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3170466125011444, "tokens": 2141061120, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.22153766900301, "seq_len": 4096}
+{"wall": 2125.367, "step": 7270, "loss": 2.228617404898008, "lm_loss": 2.228617404898008, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3620855212211609, "tokens": 2144010240, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.229142802457015, "seq_len": 4096}
+{"wall": 2330.442, "step": 7280, "loss": 2.1343268752098083, "lm_loss": 2.1343268752098083, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3332114815711975, "tokens": 2146959360, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.2388343468308447, "seq_len": 4096}
+{"wall": 2535.744, "step": 7290, "loss": 2.2821491956710815, "lm_loss": 2.2821491956710815, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.33235281705856323, "tokens": 2149908480, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.2132319231828053, "seq_len": 4096}
+{"wall": 2743.493, "eval_loss": 1.9102170765399933, "eval_ppl": 6.754554748535156, "step": 7300}
+{"wall": 2743.494, "step": 7300, "loss": 2.2169979264338813, "lm_loss": 2.2169979264338813, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.28496015071868896, "tokens": 2152857600, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.302734375, "loss_smoothed": 2.2011576225360234, "seq_len": 4096}
+{"wall": 2962.36, "step": 7310, "loss": 2.330208937327067, "lm_loss": 2.330208937327067, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.2925664186477661, "tokens": 2155806720, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.2687580565611523, "seq_len": 4096}
+{"wall": 3168.235, "step": 7320, "loss": 2.267301062742869, "lm_loss": 2.267301062742869, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.35944196581840515, "tokens": 2158755840, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.2001725653807322, "seq_len": 4096}
+{"wall": 3373.42, "step": 7330, "loss": 2.3385067731142044, "lm_loss": 2.3385067731142044, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.30972275137901306, "tokens": 2161704960, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.221622159083684, "seq_len": 4096}
+{"wall": 3578.869, "step": 7340, "loss": 2.351225902636846, "lm_loss": 2.351225902636846, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3389928936958313, "tokens": 2164654080, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.2542589376370112, "seq_len": 4096}
+{"wall": 3784.737, "step": 7350, "loss": 2.2483836313088736, "lm_loss": 2.2483836313088736, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3582954704761505, "tokens": 2167603200, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.270545894900958, "seq_len": 4096}
+{"wall": 3989.825, "step": 7360, "loss": 2.2672678331534066, "lm_loss": 2.2672678331534066, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3806975483894348, "tokens": 2170552320, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.219811411698659, "seq_len": 4096}
+{"wall": 4195.34, "step": 7370, "loss": 2.213309958577156, "lm_loss": 2.213309958577156, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.29287073016166687, "tokens": 2173501440, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.244069251169761, "seq_len": 4096}
+{"wall": 4401.168, "step": 7380, "loss": 2.2784912238518396, "lm_loss": 2.2784912238518396, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.318633109331131, "tokens": 2176450560, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.1897954230507213, "seq_len": 4096}
+{"wall": 4606.485, "step": 7390, "loss": 2.2171241293350854, "lm_loss": 2.2171241293350854, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3484538495540619, "tokens": 2179399680, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.228401086727778, "seq_len": 4096}
+{"wall": 4814.343, "eval_loss": 1.906006470322609, "eval_ppl": 6.7261738777160645, "step": 7400}
+{"wall": 4814.345, "step": 7400, "loss": 2.185451557238897, "lm_loss": 2.185451557238897, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3062881827354431, "tokens": 2182348800, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.361328125, "loss_smoothed": 2.2244600216547648, "seq_len": 4096}