Save 30% checkpoint at 9,000,976,384 tokens

checkpoint_metadata.json

@@ -50,10 +50,10 @@
     "model_type": "tiger_alpha",
     "output_attentions": false
   },
-  "training_percent": 25,
-  "step": 7153,
+  "training_percent": 30,
+  "step": 8584,
   "total_steps": 28611,
-  "tokens_seen": 7500464128,
+  "tokens_seen": 9000976384,
   "target_tokens": 30000000000,
   "trained_token_target": 30000807936,
   "tokens_per_step": 1048576,
@@ -69,5 +69,5 @@
   "dataset_id": "HuggingFaceFW/fineweb-edu",
   "dataset_config": "sample-100BT",
   "dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-  "elapsed_seconds": 2283.6879897117615
+  "elapsed_seconds": 2749.0057089328766
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ac67575996534c519fc64ea409662b8879606f9f1e602055c1fc5f69b7d5f3a
-size 13860292
+oid sha256:0fc0c29f1762d82c3d2d03617e7dbff624dbf1e151f255e8ce2dece6f7823d91
+size 12811532

modeling_tiger_alpha.py

@@ -49,13 +49,8 @@ class TigerAlphaRotaryEmbedding(nn.Module):
         positions = torch.arange(config.max_position_embeddings).float()
         frequencies = torch.outer(positions, inverse)
         angles = torch.cat((frequencies, frequencies), dim=-1)
-        # These deterministic caches must be checkpointed. Hugging Face can
-        # instantiate remote-code models under a low-memory/meta-device
-        # context; non-persistent RoPE buffers can then remain invalid after
-        # loading and make every continuation log-probability NaN. This is the
-        # same export/load issue fixed in the original partial-Edu model.
-        self.register_buffer("cosine", angles.cos(), persistent=True)
-        self.register_buffer("sine", angles.sin(), persistent=True)
+        self.register_buffer("cosine", angles.cos(), persistent=False)
+        self.register_buffer("sine", angles.sin(), persistent=False)
 
     def forward(
         self,
@@ -107,32 +102,16 @@ class TigerAlphaAttention(nn.Module):
         repeated_key = key.repeat_interleave(self.kv_repeats, dim=1)
         repeated_value = value.repeat_interleave(self.kv_repeats, dim=1)
 
-        if attention_mask is None or bool(torch.all(attention_mask)):
-            attended = F.scaled_dot_product_attention(
-                query,
-                repeated_key,
-                repeated_value,
-                is_causal=True,
-            )
-        else:
-            # Do not combine a broadcast key-padding mask with is_causal=True.
-            # Some fused CUDA SDPA kernels have produced non-finite outputs for
-            # that combination during batched continuation scoring.  Build the
-            # complete allowed-attention mask explicitly for padded inference.
-            causal_mask = torch.ones(
-                (length, length),
-                dtype=torch.bool,
-                device=hidden_states.device,
-            ).tril_()
+        key_mask = None
+        if attention_mask is not None and not bool(torch.all(attention_mask)):
             key_mask = attention_mask.to(dtype=torch.bool)[:, None, None, :]
-            allowed_mask = causal_mask[None, None, :, :] & key_mask
-            attended = F.scaled_dot_product_attention(
-                query,
-                repeated_key,
-                repeated_value,
-                attn_mask=allowed_mask,
-                is_causal=False,
-            )
+        attended = F.scaled_dot_product_attention(
+            query,
+            repeated_key,
+            repeated_value,
+            attn_mask=key_mask,
+            is_causal=True,
+        )
 
         # XSA removes the component parallel to the current-token value vector.
         grouped = attended.view(

training_metrics.jsonl

@@ -714,3 +714,146 @@
 {"step": 7130, "total_steps": 28611, "tokens": 7476346880, "loss": 2.797290086746216, "ce_loss": 2.795602858066559, "z_loss": 16.872278213500977, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.32375290989875793, "tokens_per_second": 2822323.7098571532, "eta_seconds": 7980.821256375313, "data_wait_seconds": 1.519918441772461e-05}
 {"step": 7140, "total_steps": 28611, "tokens": 7486832640, "loss": 2.8341349959373474, "ce_loss": 2.8325523138046265, "z_loss": 15.82679033279419, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2982735335826874, "tokens_per_second": 2748542.69476825, "eta_seconds": 8191.240885162353, "data_wait_seconds": 0.2758070230484009}
 {"step": 7150, "total_steps": 28611, "tokens": 7497318400, "loss": 2.798942804336548, "ce_loss": 2.797035872936249, "z_loss": 19.069526195526123, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2857823967933655, "tokens_per_second": 4063571.878036848, "eta_seconds": 5537.859354138374, "data_wait_seconds": 1.7404556274414062e-05}
+{"step": 7160, "total_steps": 28611, "tokens": 7507804160, "loss": 2.801280677318573, "ce_loss": 2.799522280693054, "z_loss": 17.583863735198975, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2686731815338135, "tokens_per_second": 8169358.510130258, "eta_seconds": 2753.3378230503627, "data_wait_seconds": 2.3305416107177734e-05}
+{"step": 7170, "total_steps": 28611, "tokens": 7518289920, "loss": 2.7960904240608215, "ce_loss": 2.79440701007843, "z_loss": 16.833951950073242, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3016676902770996, "tokens_per_second": 2954802.488258681, "eta_seconds": 7608.805700325966, "data_wait_seconds": 1.71661376953125e-05}
+{"step": 7180, "total_steps": 28611, "tokens": 7528775680, "loss": 2.7942108511924744, "ce_loss": 2.7925745844841003, "z_loss": 16.362343549728394, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2932041883468628, "tokens_per_second": 2887454.485062275, "eta_seconds": 7782.644669294358, "data_wait_seconds": 1.6570091247558594e-05}
+{"step": 7190, "total_steps": 28611, "tokens": 7539261440, "loss": 2.8055832982063293, "ce_loss": 2.803830564022064, "z_loss": 17.52704668045044, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3062359690666199, "tokens_per_second": 2989955.2103626756, "eta_seconds": 7512.335441732406, "data_wait_seconds": 0.25558823347091675}
+{"step": 7200, "total_steps": 28611, "tokens": 7549747200, "loss": 2.7766306400299072, "ce_loss": 2.7748597860336304, "z_loss": 17.70829486846924, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2873324751853943, "tokens_per_second": 3859583.6459849696, "eta_seconds": 5816.964417743683, "data_wait_seconds": 1.633167266845703e-05}
+{"step": 7210, "total_steps": 28611, "tokens": 7560232960, "loss": 2.7791834473609924, "ce_loss": 2.777418792247772, "z_loss": 17.64683198928833, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2177879363298416, "tokens_per_second": 4162718.8561825184, "eta_seconds": 5390.845683145523, "data_wait_seconds": 2.4020671844482422e-05}
+{"step": 7220, "total_steps": 28611, "tokens": 7570718720, "loss": 2.7831735014915466, "ce_loss": 2.7811816334724426, "z_loss": 19.91889524459839, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30219972133636475, "tokens_per_second": 2677262.5268622832, "eta_seconds": 8377.99393632412, "data_wait_seconds": 1.436471939086914e-05}
+{"step": 7230, "total_steps": 28611, "tokens": 7581204480, "loss": 2.7885351181030273, "ce_loss": 2.7870677709579468, "z_loss": 14.67360806465149, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.262851357460022, "tokens_per_second": 2722633.955890825, "eta_seconds": 8234.52723326683, "data_wait_seconds": 1.823902130126953e-05}
+{"step": 7240, "total_steps": 28611, "tokens": 7591690240, "loss": 2.771252691745758, "ce_loss": 2.7697200179100037, "z_loss": 15.326926708221436, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.31475162506103516, "tokens_per_second": 4760895.80438749, "eta_seconds": 4706.912021756172, "data_wait_seconds": 1.621246337890625e-05}
+{"step": 7250, "total_steps": 28611, "tokens": 7602176000, "loss": 2.793003022670746, "ce_loss": 2.7912317514419556, "z_loss": 17.712397575378418, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.28716546297073364, "tokens_per_second": 2867987.320932995, "eta_seconds": 7809.878297758102, "data_wait_seconds": 1.4603137969970703e-05}
+{"step": 7260, "total_steps": 28611, "tokens": 7612661760, "loss": 2.821839392185211, "ce_loss": 2.82023423910141, "z_loss": 16.051737785339355, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29180315136909485, "tokens_per_second": 2830989.7739165095, "eta_seconds": 7908.239860939979, "data_wait_seconds": 1.5079975128173828e-05}
+{"step": 7270, "total_steps": 28611, "tokens": 7623147520, "loss": 2.783905506134033, "ce_loss": 2.7821696996688843, "z_loss": 17.358184814453125, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.28291812539100647, "tokens_per_second": 4216385.23960952, "eta_seconds": 5307.309257650375, "data_wait_seconds": 1.3649463653564453e-05}
+{"step": 7280, "total_steps": 28611, "tokens": 7633633280, "loss": 2.82589852809906, "ce_loss": 2.824151873588562, "z_loss": 17.46619987487793, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2924087941646576, "tokens_per_second": 3133090.6230852054, "eta_seconds": 7139.012989664077, "data_wait_seconds": 0.25635266304016113}
+{"step": 7290, "total_steps": 28611, "tokens": 7644119040, "loss": 2.860044777393341, "ce_loss": 2.8585484623908997, "z_loss": 14.963523626327515, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.293559193611145, "tokens_per_second": 4228209.483840873, "eta_seconds": 5287.507390880584, "data_wait_seconds": 1.2695789337158203e-05}
+{"step": 7300, "total_steps": 28611, "tokens": 7654604800, "loss": 2.8007644414901733, "ce_loss": 2.7990832328796387, "z_loss": 16.811604261398315, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2996461093425751, "tokens_per_second": 2847466.0126049863, "eta_seconds": 7847.7506095170975, "data_wait_seconds": 1.4126300811767578e-05}
+{"step": 7310, "total_steps": 28611, "tokens": 7665090560, "loss": 2.8152270317077637, "ce_loss": 2.813812553882599, "z_loss": 14.144834995269775, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.25128883123397827, "tokens_per_second": 2858621.232989986, "eta_seconds": 7813.458151865006, "data_wait_seconds": 0.5442037582397461}
+{"step": 7320, "total_steps": 28611, "tokens": 7675576320, "loss": 2.835469603538513, "ce_loss": 2.8338627219200134, "z_loss": 16.06903624534607, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30048325657844543, "tokens_per_second": 4194392.001846353, "eta_seconds": 5322.638324260711, "data_wait_seconds": 1.4662742614746094e-05}
+{"step": 7330, "total_steps": 28611, "tokens": 7686062080, "loss": 2.817129611968994, "ce_loss": 2.8154589533805847, "z_loss": 16.706435203552246, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2904641330242157, "tokens_per_second": 2848305.0795881725, "eta_seconds": 7834.394572377205, "data_wait_seconds": 0.2760661840438843}
+{"step": 7340, "total_steps": 28611, "tokens": 7696547840, "loss": 2.765535533428192, "ce_loss": 2.763844847679138, "z_loss": 16.906526803970337, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2825606167316437, "tokens_per_second": 4507261.97604556, "eta_seconds": 4948.51646399498, "data_wait_seconds": 1.4185905456542969e-05}
+{"step": 7350, "total_steps": 28611, "tokens": 7707033600, "loss": 2.8194339275360107, "ce_loss": 2.81795197725296, "z_loss": 14.818968296051025, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.26579993963241577, "tokens_per_second": 2832789.514631319, "eta_seconds": 7869.901459622383, "data_wait_seconds": 1.728534698486328e-05}
+{"step": 7360, "total_steps": 28611, "tokens": 7717519360, "loss": 2.8325823545455933, "ce_loss": 2.830956220626831, "z_loss": 16.260990858078003, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30176758766174316, "tokens_per_second": 3827512.1861233874, "eta_seconds": 5821.872666215896, "data_wait_seconds": 1.4662742614746094e-05}
+{"step": 7370, "total_steps": 28611, "tokens": 7728005120, "loss": 2.8439570665359497, "ce_loss": 2.842434346675873, "z_loss": 15.226919174194336, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2741280198097229, "tokens_per_second": 2748935.7587782503, "eta_seconds": 8102.336602401733, "data_wait_seconds": 2.09808349609375e-05}
+{"step": 7380, "total_steps": 28611, "tokens": 7738490880, "loss": 2.801985800266266, "ce_loss": 2.8006086349487305, "z_loss": 13.771665811538696, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.31038910150527954, "tokens_per_second": 4155548.2465141905, "eta_seconds": 5357.251494956016, "data_wait_seconds": 1.722574234008789e-05}
+{"step": 7390, "total_steps": 28611, "tokens": 7748976640, "loss": 2.8230409622192383, "ce_loss": 2.821776270866394, "z_loss": 12.64690375328064, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.24939274787902832, "tokens_per_second": 2285965.614359496, "eta_seconds": 9734.10586590767, "data_wait_seconds": 1.895427703857422e-05}
+{"step": 7400, "total_steps": 28611, "tokens": 7759462400, "loss": 2.8117712140083313, "ce_loss": 2.8100315928459167, "z_loss": 17.396262645721436, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.26019710302352905, "tokens_per_second": 3130422.804827747, "eta_seconds": 7104.901453471184, "data_wait_seconds": 1.704692840576172e-05}
+{"step": 7410, "total_steps": 28611, "tokens": 7769948160, "loss": 2.8618265986442566, "ce_loss": 2.8601033687591553, "z_loss": 17.231842517852783, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27591240406036377, "tokens_per_second": 3745124.5015954506, "eta_seconds": 5935.946793365479, "data_wait_seconds": 1.7940998077392578e-05}
+{"step": 7420, "total_steps": 28611, "tokens": 7780433920, "loss": 2.8315808176994324, "ce_loss": 2.830158531665802, "z_loss": 14.2233726978302, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2990984618663788, "tokens_per_second": 2282036.5570835443, "eta_seconds": 9737.080655884743, "data_wait_seconds": 2.0802021026611328e-05}
+{"step": 7430, "total_steps": 28611, "tokens": 7790919680, "loss": 2.8284667134284973, "ce_loss": 2.826790988445282, "z_loss": 16.756941318511963, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2603110373020172, "tokens_per_second": 2849077.27350723, "eta_seconds": 7795.467136859895, "data_wait_seconds": 1.4781951904296875e-05}
+{"step": 7440, "total_steps": 28611, "tokens": 7801405440, "loss": 2.869446635246277, "ce_loss": 2.867709517478943, "z_loss": 17.37147855758667, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.36189571022987366, "tokens_per_second": 2909548.3454278763, "eta_seconds": 7629.844862651825, "data_wait_seconds": 1.7464160919189453e-05}
+{"step": 7450, "total_steps": 28611, "tokens": 7811891200, "loss": 2.8283491134643555, "ce_loss": 2.82672256231308, "z_loss": 16.265615940093994, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3026229441165924, "tokens_per_second": 3066502.4553402597, "eta_seconds": 7235.903789138793, "data_wait_seconds": 1.5854835510253906e-05}
+{"step": 7460, "total_steps": 28611, "tokens": 7822376960, "loss": 2.829996883869171, "ce_loss": 2.8286291360855103, "z_loss": 13.67777705192566, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27820125222206116, "tokens_per_second": 2828731.2124059456, "eta_seconds": 7840.416536831855, "data_wait_seconds": 1.4841556549072266e-05}
+{"step": 7470, "total_steps": 28611, "tokens": 7832862720, "loss": 2.785897135734558, "ce_loss": 2.784358322620392, "z_loss": 15.387907981872559, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27461329102516174, "tokens_per_second": 2796776.6511325743, "eta_seconds": 7926.247956562042, "data_wait_seconds": 0.12810176610946655}
+{"step": 7480, "total_steps": 28611, "tokens": 7843348480, "loss": 2.8074434995651245, "ce_loss": 2.8060725927352905, "z_loss": 13.709191799163818, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.26502472162246704, "tokens_per_second": 4123744.897509427, "eta_seconds": 5373.140193367005, "data_wait_seconds": 1.615285873413086e-05}
+{"step": 7490, "total_steps": 28611, "tokens": 7853834240, "loss": 2.8499537110328674, "ce_loss": 2.8484376668930054, "z_loss": 15.160365104675293, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27157169580459595, "tokens_per_second": 2722718.56866007, "eta_seconds": 8134.139881706237, "data_wait_seconds": 0.2946011424064636}
+{"step": 7500, "total_steps": 28611, "tokens": 7864320000, "loss": 2.826074719429016, "ce_loss": 2.8246424794197083, "z_loss": 14.322731256484985, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3065676689147949, "tokens_per_second": 4497868.608710841, "eta_seconds": 4921.5505969047545, "data_wait_seconds": 1.537799835205078e-05}
+{"step": 7510, "total_steps": 28611, "tokens": 7874805760, "loss": 2.7967828512191772, "ce_loss": 2.795423924922943, "z_loss": 13.589845418930054, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29159921407699585, "tokens_per_second": 2857771.2499192646, "eta_seconds": 7742.397918176651, "data_wait_seconds": 0.28380846977233887}
+{"step": 7520, "total_steps": 28611, "tokens": 7885291520, "loss": 2.839067578315735, "ce_loss": 2.837330996990204, "z_loss": 17.365418434143066, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2833460867404938, "tokens_per_second": 4194227.201406224, "eta_seconds": 5272.846546936035, "data_wait_seconds": 1.430511474609375e-05}
+{"step": 7530, "total_steps": 28611, "tokens": 7895777280, "loss": 2.8192083835601807, "ce_loss": 2.817777097225189, "z_loss": 14.312944412231445, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27506616711616516, "tokens_per_second": 2947219.009364257, "eta_seconds": 7500.301330089569, "data_wait_seconds": 0.4921306371688843}
+{"step": 7540, "total_steps": 28611, "tokens": 7906263040, "loss": 2.8471274971961975, "ce_loss": 2.8458712697029114, "z_loss": 12.562937498092651, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3328515291213989, "tokens_per_second": 3964562.4429595373, "eta_seconds": 5573.009686160088, "data_wait_seconds": 1.3530254364013672e-05}
+{"step": 7550, "total_steps": 28611, "tokens": 7916748800, "loss": 2.854113817214966, "ce_loss": 2.8527172207832336, "z_loss": 13.9660484790802, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29045620560646057, "tokens_per_second": 2609977.992922038, "eta_seconds": 8461.396684527397, "data_wait_seconds": 0.3145901560783386}
+{"step": 7560, "total_steps": 28611, "tokens": 7927234560, "loss": 2.8265665769577026, "ce_loss": 2.824633836746216, "z_loss": 19.327107429504395, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.34986621141433716, "tokens_per_second": 2896366.1204508087, "eta_seconds": 7621.126769900323, "data_wait_seconds": 1.817941665649414e-05}
+{"step": 7570, "total_steps": 28611, "tokens": 7937720320, "loss": 2.821036159992218, "ce_loss": 2.819701135158539, "z_loss": 13.349852085113525, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2901696562767029, "tokens_per_second": 2953877.2930868785, "eta_seconds": 7469.195713591576, "data_wait_seconds": 1.811981201171875e-05}
+{"step": 7580, "total_steps": 28611, "tokens": 7948206080, "loss": 2.8298197388648987, "ce_loss": 2.8280742168426514, "z_loss": 17.455594539642334, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.24848245084285736, "tokens_per_second": 2831024.7622598396, "eta_seconds": 7789.618144631385, "data_wait_seconds": 1.424551010131836e-05}
+{"step": 7590, "total_steps": 28611, "tokens": 7958691840, "loss": 2.832265257835388, "ce_loss": 2.8308276534080505, "z_loss": 14.376134157180786, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3093660771846771, "tokens_per_second": 2256029.164487563, "eta_seconds": 9770.315225958822, "data_wait_seconds": 0.2471410632133484}
+{"step": 7600, "total_steps": 28611, "tokens": 7969177600, "loss": 2.8611544966697693, "ce_loss": 2.8599029779434204, "z_loss": 12.515166282653809, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.33860206604003906, "tokens_per_second": 4066684.516649883, "eta_seconds": 5417.590237402916, "data_wait_seconds": 1.2695789337158203e-05}
+{"step": 7610, "total_steps": 28611, "tokens": 7979663360, "loss": 2.8518078923225403, "ce_loss": 2.8500975370407104, "z_loss": 17.103496551513672, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29505103826522827, "tokens_per_second": 2745398.6257945765, "eta_seconds": 8021.110074543953, "data_wait_seconds": 1.7404556274414062e-05}
+{"step": 7620, "total_steps": 28611, "tokens": 7990149120, "loss": 2.830790638923645, "ce_loss": 2.829412281513214, "z_loss": 13.783782958984375, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29002517461776733, "tokens_per_second": 4355254.394575537, "eta_seconds": 5053.817026948929, "data_wait_seconds": 1.329183578491211e-05}
+{"step": 7630, "total_steps": 28611, "tokens": 8000634880, "loss": 2.8403719663619995, "ce_loss": 2.838901460170746, "z_loss": 14.704530715942383, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.26952359080314636, "tokens_per_second": 2958115.8703551833, "eta_seconds": 7437.22491619587, "data_wait_seconds": 1.8656253814697266e-05}
+{"step": 7640, "total_steps": 28611, "tokens": 8011120640, "loss": 2.8192873001098633, "ce_loss": 2.8174463510513306, "z_loss": 18.410107612609863, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29466772079467773, "tokens_per_second": 3953017.749386898, "eta_seconds": 5562.759564995766, "data_wait_seconds": 1.5854835510253906e-05}
+{"step": 7650, "total_steps": 28611, "tokens": 8021606400, "loss": 2.8412696719169617, "ce_loss": 2.8396135568618774, "z_loss": 16.561192512512207, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.31581583619117737, "tokens_per_second": 2895277.009909603, "eta_seconds": 7591.398495125771, "data_wait_seconds": 1.6391277313232422e-05}
+{"step": 7660, "total_steps": 28611, "tokens": 8032092160, "loss": 2.8389203548431396, "ce_loss": 2.837357819080353, "z_loss": 15.625047445297241, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.37895169854164124, "tokens_per_second": 4045073.033677958, "eta_seconds": 5430.981243872642, "data_wait_seconds": 1.5079975128173828e-05}
+{"step": 7670, "total_steps": 28611, "tokens": 8042577920, "loss": 2.8408098220825195, "ce_loss": 2.839123487472534, "z_loss": 16.863346576690674, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.31690728664398193, "tokens_per_second": 2935763.3411971084, "eta_seconds": 7479.564073801042, "data_wait_seconds": 1.704692840576172e-05}
+{"step": 7680, "total_steps": 28611, "tokens": 8053063680, "loss": 2.8206074237823486, "ce_loss": 2.819244146347046, "z_loss": 13.632554769515991, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27943530678749084, "tokens_per_second": 3060982.6562882033, "eta_seconds": 7170.1628922700875, "data_wait_seconds": 0.2576793432235718}
+{"step": 7690, "total_steps": 28611, "tokens": 8063549440, "loss": 2.8628398776054382, "ce_loss": 2.8615484833717346, "z_loss": 12.9140043258667, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2969810962677002, "tokens_per_second": 3958837.981816902, "eta_seconds": 5541.337785673141, "data_wait_seconds": 1.817941665649414e-05}
+{"step": 7700, "total_steps": 28611, "tokens": 8074035200, "loss": 2.8266555070877075, "ce_loss": 2.8251134753227234, "z_loss": 15.420504808425903, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.31833282113075256, "tokens_per_second": 3062560.1806933177, "eta_seconds": 7159.621833467484, "data_wait_seconds": 1.901388168334961e-05}
+{"step": 7710, "total_steps": 28611, "tokens": 8084520960, "loss": 2.8038904070854187, "ce_loss": 2.8022231459617615, "z_loss": 16.672114610671997, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2679053544998169, "tokens_per_second": 3628612.9283718723, "eta_seconds": 6039.852530050278, "data_wait_seconds": 1.537799835205078e-05}
+{"step": 7720, "total_steps": 28611, "tokens": 8095006720, "loss": 2.8797338604927063, "ce_loss": 2.8782511353492737, "z_loss": 14.827410459518433, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2568381130695343, "tokens_per_second": 2916958.499444338, "eta_seconds": 7509.809008312225, "data_wait_seconds": 1.7404556274414062e-05}
+{"step": 7730, "total_steps": 28611, "tokens": 8105492480, "loss": 2.898803949356079, "ce_loss": 2.8973674178123474, "z_loss": 14.365464925765991, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.34451472759246826, "tokens_per_second": 2665964.223298511, "eta_seconds": 8212.906709194183, "data_wait_seconds": 0.3332664966583252}
+{"step": 7740, "total_steps": 28611, "tokens": 8115978240, "loss": 2.80974543094635, "ce_loss": 2.8080779314041138, "z_loss": 16.67478084564209, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.24343647062778473, "tokens_per_second": 4353401.938880586, "eta_seconds": 5027.063892388343, "data_wait_seconds": 1.3947486877441406e-05}
+{"step": 7750, "total_steps": 28611, "tokens": 8126464000, "loss": 2.837118923664093, "ce_loss": 2.8357242941856384, "z_loss": 13.946267127990723, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2617914378643036, "tokens_per_second": 3043994.186921276, "eta_seconds": 7186.066264510155, "data_wait_seconds": 1.519918441772461e-05}
+{"step": 7760, "total_steps": 28611, "tokens": 8136949760, "loss": 2.852109372615814, "ce_loss": 2.8505718111991882, "z_loss": 15.375538110733032, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.31255337595939636, "tokens_per_second": 3012950.0065930705, "eta_seconds": 7256.628264045716, "data_wait_seconds": 0.23282551765441895}
+{"step": 7770, "total_steps": 28611, "tokens": 8147435520, "loss": 2.877464771270752, "ce_loss": 2.875877320766449, "z_loss": 15.874592781066895, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27597862482070923, "tokens_per_second": 3961626.237672181, "eta_seconds": 5516.26304576397, "data_wait_seconds": 1.5854835510253906e-05}
+{"step": 7780, "total_steps": 28611, "tokens": 8157921280, "loss": 2.8410410284996033, "ce_loss": 2.8397462368011475, "z_loss": 12.948607444763184, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27496564388275146, "tokens_per_second": 3009552.052351145, "eta_seconds": 7257.85308778286, "data_wait_seconds": 0.24335229396820068}
+{"step": 7790, "total_steps": 28611, "tokens": 8168407040, "loss": 2.835705876350403, "ce_loss": 2.8339353799819946, "z_loss": 17.704952716827393, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2823038399219513, "tokens_per_second": 3926373.2551495163, "eta_seconds": 5560.449676394463, "data_wait_seconds": 1.2695789337158203e-05}
+{"step": 7800, "total_steps": 28611, "tokens": 8178892800, "loss": 2.8151214122772217, "ce_loss": 2.8135750889778137, "z_loss": 15.46279788017273, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3277873694896698, "tokens_per_second": 2167354.307876175, "eta_seconds": 10068.457684421539, "data_wait_seconds": 0.3670269250869751}
+{"step": 7810, "total_steps": 28611, "tokens": 8189378560, "loss": 2.8361517190933228, "ce_loss": 2.8344799876213074, "z_loss": 16.717202186584473, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3067009150981903, "tokens_per_second": 3072788.6975000077, "eta_seconds": 7098.252279353142, "data_wait_seconds": 1.7583370208740234e-05}
+{"step": 7820, "total_steps": 28611, "tokens": 8199864320, "loss": 2.8238715529441833, "ce_loss": 2.8221450448036194, "z_loss": 17.264512538909912, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.31892073154449463, "tokens_per_second": 3910799.250343814, "eta_seconds": 5574.549400377274, "data_wait_seconds": 1.5616416931152344e-05}
+{"step": 7830, "total_steps": 28611, "tokens": 8210350080, "loss": 2.845070540904999, "ce_loss": 2.843770444393158, "z_loss": 13.001420259475708, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2809121906757355, "tokens_per_second": 2874691.0776454476, "eta_seconds": 7580.10418074131, "data_wait_seconds": 1.817941665649414e-05}
+{"step": 7840, "total_steps": 28611, "tokens": 8220835840, "loss": 2.8318759202957153, "ce_loss": 2.8305221796035767, "z_loss": 13.53763461112976, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2939044237136841, "tokens_per_second": 2821760.736406321, "eta_seconds": 7718.575077962875, "data_wait_seconds": 0.2901323437690735}
+{"step": 7850, "total_steps": 28611, "tokens": 8231321600, "loss": 2.7933029532432556, "ce_loss": 2.7916588187217712, "z_loss": 16.44132137298584, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.26029524207115173, "tokens_per_second": 4157131.5897863815, "eta_seconds": 5236.660390901566, "data_wait_seconds": 1.4662742614746094e-05}
+{"step": 7860, "total_steps": 28611, "tokens": 8241807360, "loss": 2.8260114789009094, "ce_loss": 2.824548661708832, "z_loss": 14.628187417984009, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.28735655546188354, "tokens_per_second": 2639497.2831734126, "eta_seconds": 8243.615447044373, "data_wait_seconds": 1.9252300262451172e-05}
+{"step": 7870, "total_steps": 28611, "tokens": 8252293120, "loss": 2.8242035508155823, "ce_loss": 2.82255882024765, "z_loss": 16.447771787643433, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2852390706539154, "tokens_per_second": 4046727.8126820973, "eta_seconds": 5374.34584748745, "data_wait_seconds": 1.239776611328125e-05}
+{"step": 7880, "total_steps": 28611, "tokens": 8262778880, "loss": 2.8327550292015076, "ce_loss": 2.8309794664382935, "z_loss": 17.755791664123535, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.274616539478302, "tokens_per_second": 2782182.473880625, "eta_seconds": 7813.300982260704, "data_wait_seconds": 1.4960765838623047e-05}
+{"step": 7890, "total_steps": 28611, "tokens": 8273264640, "loss": 2.817731499671936, "ce_loss": 2.816210150718689, "z_loss": 15.213933229446411, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.337815523147583, "tokens_per_second": 2794353.9342244756, "eta_seconds": 7775.515846395493, "data_wait_seconds": 2.3365020751953125e-05}
+{"step": 7900, "total_steps": 28611, "tokens": 8283750400, "loss": 2.8274554014205933, "ce_loss": 2.826216220855713, "z_loss": 12.3912672996521, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2785104811191559, "tokens_per_second": 2876476.090545382, "eta_seconds": 7549.882860970498, "data_wait_seconds": 1.531839370727539e-05}
+{"step": 7910, "total_steps": 28611, "tokens": 8294236160, "loss": 2.8327569365501404, "ce_loss": 2.831115424633026, "z_loss": 16.41549801826477, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.32349520921707153, "tokens_per_second": 2945837.7561875503, "eta_seconds": 7368.556442189217, "data_wait_seconds": 1.6510486602783203e-05}
+{"step": 7920, "total_steps": 28611, "tokens": 8304721920, "loss": 2.8656616806983948, "ce_loss": 2.8641517758369446, "z_loss": 15.099197387695312, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3100632429122925, "tokens_per_second": 2598207.1208223244, "eta_seconds": 8350.406648540496, "data_wait_seconds": 1.7821788787841797e-05}
+{"step": 7930, "total_steps": 28611, "tokens": 8315207680, "loss": 2.8218806982040405, "ce_loss": 2.820408821105957, "z_loss": 14.719407796859741, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.24940890073776245, "tokens_per_second": 4219625.956953171, "eta_seconds": 5139.223352313042, "data_wait_seconds": 1.615285873413086e-05}
+{"step": 7940, "total_steps": 28611, "tokens": 8325693440, "loss": 2.886754870414734, "ce_loss": 2.8853622674942017, "z_loss": 13.926624774932861, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3139261305332184, "tokens_per_second": 2864051.7836341816, "eta_seconds": 7567.9897339344025, "data_wait_seconds": 1.9788742065429688e-05}
+{"step": 7950, "total_steps": 28611, "tokens": 8336179200, "loss": 2.863364279270172, "ce_loss": 2.8616687655448914, "z_loss": 16.954803466796875, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2930862605571747, "tokens_per_second": 4417090.354458211, "eta_seconds": 4904.7284518718725, "data_wait_seconds": 1.2516975402832031e-05}
+{"step": 7960, "total_steps": 28611, "tokens": 8346664960, "loss": 2.826922655105591, "ce_loss": 2.8253015875816345, "z_loss": 16.210403442382812, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.250663161277771, "tokens_per_second": 2935708.0796293383, "eta_seconds": 7376.122689533234, "data_wait_seconds": 1.3589859008789062e-05}
+{"step": 7970, "total_steps": 28611, "tokens": 8357150720, "loss": 2.852204382419586, "ce_loss": 2.8508216738700867, "z_loss": 13.82711148262024, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.25194722414016724, "tokens_per_second": 3052756.5385799655, "eta_seconds": 7089.873346424102, "data_wait_seconds": 0.536457359790802}
+{"step": 7980, "total_steps": 28611, "tokens": 8367636480, "loss": 2.8142963647842407, "ce_loss": 2.812448263168335, "z_loss": 18.480632305145264, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2724212408065796, "tokens_per_second": 4461272.105510506, "eta_seconds": 4849.103785729408, "data_wait_seconds": 1.33514404296875e-05}
+{"step": 7990, "total_steps": 28611, "tokens": 8378122240, "loss": 2.795308232307434, "ce_loss": 2.793743073940277, "z_loss": 15.651727676391602, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29804185032844543, "tokens_per_second": 2810028.379902874, "eta_seconds": 7694.82822687626, "data_wait_seconds": 1.6808509826660156e-05}
+{"step": 8000, "total_steps": 28611, "tokens": 8388608000, "loss": 2.801842987537384, "ce_loss": 2.8001992106437683, "z_loss": 16.437334299087524, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2732853889465332, "tokens_per_second": 4098820.7125106556, "eta_seconds": 5272.7848939657215, "data_wait_seconds": 1.5676021575927734e-05}
+{"step": 8010, "total_steps": 28611, "tokens": 8399093760, "loss": 2.792961597442627, "ce_loss": 2.7915989756584167, "z_loss": 13.626461267471313, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30972447991371155, "tokens_per_second": 2723146.432464516, "eta_seconds": 7932.630400800705, "data_wait_seconds": 1.341104507446289e-05}
+{"step": 8020, "total_steps": 28611, "tokens": 8409579520, "loss": 2.832617461681366, "ce_loss": 2.830963671207428, "z_loss": 16.537288188934326, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3046140670776367, "tokens_per_second": 2593666.620355444, "eta_seconds": 8324.596633410454, "data_wait_seconds": 0.2781081199645996}
+{"step": 8030, "total_steps": 28611, "tokens": 8420065280, "loss": 2.8244447112083435, "ce_loss": 2.822901964187622, "z_loss": 15.427709817886353, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.23251965641975403, "tokens_per_second": 4219264.461912283, "eta_seconds": 5114.811562728883, "data_wait_seconds": 1.4185905456542969e-05}
+{"step": 8040, "total_steps": 28611, "tokens": 8430551040, "loss": 2.850928843021393, "ce_loss": 2.8493210077285767, "z_loss": 16.07933211326599, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2636032998561859, "tokens_per_second": 3068816.54909437, "eta_seconds": 7028.851855731011, "data_wait_seconds": 1.531839370727539e-05}
+{"step": 8050, "total_steps": 28611, "tokens": 8441036800, "loss": 2.766439735889435, "ce_loss": 2.764879286289215, "z_loss": 15.603840589523315, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2594323456287384, "tokens_per_second": 2850185.2770006675, "eta_seconds": 7564.340223765373, "data_wait_seconds": 1.5437602996826172e-05}
+{"step": 8060, "total_steps": 28611, "tokens": 8451522560, "loss": 2.8342041969299316, "ce_loss": 2.8326293230056763, "z_loss": 15.748641014099121, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2855604887008667, "tokens_per_second": 4420256.532075064, "eta_seconds": 4875.12098441124, "data_wait_seconds": 1.3172626495361328e-05}
+{"step": 8070, "total_steps": 28611, "tokens": 8462008320, "loss": 2.7979369163513184, "ce_loss": 2.7959888577461243, "z_loss": 19.479676723480225, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.28594741225242615, "tokens_per_second": 2901795.076517376, "eta_seconds": 7422.577765846252, "data_wait_seconds": 1.7642974853515625e-05}
+{"step": 8080, "total_steps": 28611, "tokens": 8472494080, "loss": 2.8041998147964478, "ce_loss": 2.8024291396141052, "z_loss": 17.707226753234863, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.26561450958251953, "tokens_per_second": 4541899.833583163, "eta_seconds": 4739.935851693153, "data_wait_seconds": 1.239776611328125e-05}
+{"step": 8090, "total_steps": 28611, "tokens": 8482979840, "loss": 2.839084208011627, "ce_loss": 2.8377649188041687, "z_loss": 13.192646026611328, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2845306992530823, "tokens_per_second": 3108902.040511011, "eta_seconds": 6921.359314513206, "data_wait_seconds": 1.4007091522216797e-05}
+{"step": 8100, "total_steps": 28611, "tokens": 8493465600, "loss": 2.803247630596161, "ce_loss": 2.80161052942276, "z_loss": 16.3710618019104, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2648698091506958, "tokens_per_second": 2824022.138804935, "eta_seconds": 7615.854720282555, "data_wait_seconds": 1.4483928680419922e-05}
+{"step": 8110, "total_steps": 28611, "tokens": 8503951360, "loss": 2.8151272535324097, "ce_loss": 2.8137142658233643, "z_loss": 14.129340410232544, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2691187560558319, "tokens_per_second": 3192995.1386865457, "eta_seconds": 6732.505263018608, "data_wait_seconds": 0.1817617416381836}
+{"step": 8120, "total_steps": 28611, "tokens": 8514437120, "loss": 2.8136000633239746, "ce_loss": 2.8122615814208984, "z_loss": 13.385212898254395, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.22500114142894745, "tokens_per_second": 3353778.4903092994, "eta_seconds": 6406.615964078903, "data_wait_seconds": 1.3649463653564453e-05}
+{"step": 8130, "total_steps": 28611, "tokens": 8524922880, "loss": 2.8254016637802124, "ce_loss": 2.8237916827201843, "z_loss": 16.099401712417603, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2764686644077301, "tokens_per_second": 4080500.3422685266, "eta_seconds": 5263.051894283294, "data_wait_seconds": 1.3709068298339844e-05}
+{"step": 8140, "total_steps": 28611, "tokens": 8535408640, "loss": 2.797206461429596, "ce_loss": 2.7957026958465576, "z_loss": 15.037853479385376, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2357422560453415, "tokens_per_second": 2915477.8957366394, "eta_seconds": 7362.566297411919, "data_wait_seconds": 1.531839370727539e-05}
+{"step": 8150, "total_steps": 28611, "tokens": 8545894400, "loss": 2.8384560346603394, "ce_loss": 2.83685702085495, "z_loss": 15.990272521972656, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3284912407398224, "tokens_per_second": 3984435.779876656, "eta_seconds": 5384.680472040176, "data_wait_seconds": 1.6450881958007812e-05}
+{"step": 8160, "total_steps": 28611, "tokens": 8556380160, "loss": 2.853830397129059, "ce_loss": 2.852532923221588, "z_loss": 12.97488808631897, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.26301172375679016, "tokens_per_second": 2818519.0967094298, "eta_seconds": 7608.402512168885, "data_wait_seconds": 1.609325408935547e-05}
+{"step": 8170, "total_steps": 28611, "tokens": 8566865920, "loss": 2.7942258715629578, "ce_loss": 2.792454242706299, "z_loss": 17.716323852539062, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3102928698062897, "tokens_per_second": 3057571.637425616, "eta_seconds": 7010.119322681427, "data_wait_seconds": 0.2720833420753479}
+{"step": 8180, "total_steps": 28611, "tokens": 8577351680, "loss": 2.8015239238739014, "ce_loss": 2.7999374866485596, "z_loss": 15.864522695541382, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2802903354167938, "tokens_per_second": 3898702.6278768475, "eta_seconds": 5495.021883130074, "data_wait_seconds": 1.3947486877441406e-05}
+{"step": 8190, "total_steps": 28611, "tokens": 8587837440, "loss": 2.796576738357544, "ce_loss": 2.7949941158294678, "z_loss": 15.826934337615967, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.250729501247406, "tokens_per_second": 2839485.364605305, "eta_seconds": 7541.144871854782, "data_wait_seconds": 1.7642974853515625e-05}
+{"step": 8200, "total_steps": 28611, "tokens": 8598323200, "loss": 2.8157283067703247, "ce_loss": 2.81420236825943, "z_loss": 15.259318828582764, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2640817165374756, "tokens_per_second": 2572168.363016961, "eta_seconds": 8320.794643044472, "data_wait_seconds": 2.3543834686279297e-05}
+{"step": 8210, "total_steps": 28611, "tokens": 8608808960, "loss": 2.7986321449279785, "ce_loss": 2.797078013420105, "z_loss": 15.541817426681519, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2606365978717804, "tokens_per_second": 3865870.0648084553, "eta_seconds": 5533.553538370133, "data_wait_seconds": 1.329183578491211e-05}
+{"step": 8220, "total_steps": 28611, "tokens": 8619294720, "loss": 2.821324110031128, "ce_loss": 2.819686532020569, "z_loss": 16.37626552581787, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3322388231754303, "tokens_per_second": 2420606.813569565, "eta_seconds": 8833.121139764786, "data_wait_seconds": 0.24225229024887085}
+{"step": 8230, "total_steps": 28611, "tokens": 8629780480, "loss": 2.792696535587311, "ce_loss": 2.7911415696144104, "z_loss": 15.549434185028076, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3013248145580292, "tokens_per_second": 3012689.337339341, "eta_seconds": 7093.671156573296, "data_wait_seconds": 1.704692840576172e-05}
+{"step": 8240, "total_steps": 28611, "tokens": 8640266240, "loss": 2.733462929725647, "ce_loss": 2.7319385409355164, "z_loss": 15.243753910064697, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29874101281166077, "tokens_per_second": 3673288.6594751263, "eta_seconds": 5815.100221133232, "data_wait_seconds": 1.2934207916259766e-05}
+{"step": 8250, "total_steps": 28611, "tokens": 8650752000, "loss": 2.8118014335632324, "ce_loss": 2.810148239135742, "z_loss": 16.531996965408325, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30402106046676636, "tokens_per_second": 2981876.8560963348, "eta_seconds": 7159.93884601593, "data_wait_seconds": 1.728534698486328e-05}
+{"step": 8260, "total_steps": 28611, "tokens": 8661237760, "loss": 2.8314642310142517, "ce_loss": 2.830165982246399, "z_loss": 12.9829843044281, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29911696910858154, "tokens_per_second": 4353029.654927567, "eta_seconds": 4902.234045624733, "data_wait_seconds": 1.150369644165039e-05}
+{"step": 8270, "total_steps": 28611, "tokens": 8671723520, "loss": 2.794304132461548, "ce_loss": 2.792763411998749, "z_loss": 15.407686948776245, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29759466648101807, "tokens_per_second": 2890609.156314511, "eta_seconds": 7378.750727820397, "data_wait_seconds": 1.7464160919189453e-05}
+{"step": 8280, "total_steps": 28611, "tokens": 8682209280, "loss": 2.7995166182518005, "ce_loss": 2.797978639602661, "z_loss": 15.380031824111938, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2916352450847626, "tokens_per_second": 4241601.814730567, "eta_seconds": 5026.072598791122, "data_wait_seconds": 1.1086463928222656e-05}
+{"step": 8290, "total_steps": 28611, "tokens": 8692695040, "loss": 2.828550159931183, "ce_loss": 2.82722145318985, "z_loss": 13.287074089050293, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30104944109916687, "tokens_per_second": 3009889.6283530523, "eta_seconds": 7079.366862916946, "data_wait_seconds": 1.5735626220703125e-05}
+{"step": 8300, "total_steps": 28611, "tokens": 8703180800, "loss": 2.8140971064567566, "ce_loss": 2.812835156917572, "z_loss": 12.619367122650146, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.29296016693115234, "tokens_per_second": 4203402.092450165, "eta_seconds": 5066.759417152405, "data_wait_seconds": 1.3113021850585938e-05}
+{"step": 8310, "total_steps": 28611, "tokens": 8713666560, "loss": 2.8331817388534546, "ce_loss": 2.8314285278320312, "z_loss": 17.531741619110107, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27594706416130066, "tokens_per_second": 3092917.133696434, "eta_seconds": 6882.545007133484, "data_wait_seconds": 1.4662742614746094e-05}
+{"step": 8320, "total_steps": 28611, "tokens": 8724152320, "loss": 2.8145769834518433, "ce_loss": 2.813095211982727, "z_loss": 14.818042278289795, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3348222076892853, "tokens_per_second": 2767980.55051895, "eta_seconds": 7686.707051467895, "data_wait_seconds": 0.2611338496208191}
+{"step": 8330, "total_steps": 28611, "tokens": 8734638080, "loss": 2.8574329614639282, "ce_loss": 2.85581374168396, "z_loss": 16.192149877548218, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2833656370639801, "tokens_per_second": 4364565.059618654, "eta_seconds": 4872.460271644592, "data_wait_seconds": 1.5020370483398438e-05}
+{"step": 8340, "total_steps": 28611, "tokens": 8745123840, "loss": 2.874287247657776, "ce_loss": 2.8730515837669373, "z_loss": 12.35709547996521, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2778696119785309, "tokens_per_second": 4066663.083119384, "eta_seconds": 5226.812170457841, "data_wait_seconds": 1.6391277313232422e-05}
+{"step": 8350, "total_steps": 28611, "tokens": 8755609600, "loss": 2.79257071018219, "ce_loss": 2.7912225127220154, "z_loss": 13.48187780380249, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27949634194374084, "tokens_per_second": 2882693.1378104137, "eta_seconds": 7369.913244438172, "data_wait_seconds": 1.3887882232666016e-05}
+{"step": 8360, "total_steps": 28611, "tokens": 8766095360, "loss": 2.8245397806167603, "ce_loss": 2.8231603503227234, "z_loss": 13.794836282730103, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2759862244129181, "tokens_per_second": 2811360.480874353, "eta_seconds": 7553.1802913427355, "data_wait_seconds": 1.5437602996826172e-05}
+{"step": 8370, "total_steps": 28611, "tokens": 8776581120, "loss": 2.822333514690399, "ce_loss": 2.821025550365448, "z_loss": 13.079219579696655, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27693191170692444, "tokens_per_second": 4355143.556415397, "eta_seconds": 4873.37019803524, "data_wait_seconds": 1.531839370727539e-05}
+{"step": 8380, "total_steps": 28611, "tokens": 8787066880, "loss": 2.7972800731658936, "ce_loss": 2.796026825904846, "z_loss": 12.53251838684082, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2885633111000061, "tokens_per_second": 2850870.339927365, "eta_seconds": 7441.145519280434, "data_wait_seconds": 1.7583370208740234e-05}
+{"step": 8390, "total_steps": 28611, "tokens": 8797552640, "loss": 2.8382762670516968, "ce_loss": 2.837079644203186, "z_loss": 11.966521263122559, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.32881858944892883, "tokens_per_second": 3976151.040703585, "eta_seconds": 5332.608112454414, "data_wait_seconds": 1.5437602996826172e-05}
+{"step": 8400, "total_steps": 28611, "tokens": 8808038400, "loss": 2.8234041929244995, "ce_loss": 2.8219322562217712, "z_loss": 14.719144105911255, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30414673686027527, "tokens_per_second": 2769811.110770049, "eta_seconds": 7651.341080117226, "data_wait_seconds": 1.4483928680419922e-05}
+{"step": 8410, "total_steps": 28611, "tokens": 8818524160, "loss": 2.803168833255768, "ce_loss": 2.80151504278183, "z_loss": 16.537829160690308, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2772093415260315, "tokens_per_second": 2853781.4612478353, "eta_seconds": 7422.53184542656, "data_wait_seconds": 1.5139579772949219e-05}
+{"step": 8420, "total_steps": 28611, "tokens": 8829009920, "loss": 2.7976872324943542, "ce_loss": 2.796267330646515, "z_loss": 14.199423789978027, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3009188175201416, "tokens_per_second": 4177707.1347169015, "eta_seconds": 5067.803302931786, "data_wait_seconds": 1.7464160919189453e-05}
+{"step": 8430, "total_steps": 28611, "tokens": 8839495680, "loss": 2.824425518512726, "ce_loss": 2.8230987787246704, "z_loss": 13.267140865325928, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2942150831222534, "tokens_per_second": 2946168.886242709, "eta_seconds": 7182.654176688195, "data_wait_seconds": 0.2951688766479492}
+{"step": 8440, "total_steps": 28611, "tokens": 8849981440, "loss": 2.819083869457245, "ce_loss": 2.8178535103797913, "z_loss": 12.303299188613892, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.28354981541633606, "tokens_per_second": 4468843.266789459, "eta_seconds": 4732.953302073479, "data_wait_seconds": 1.4066696166992188e-05}
+{"step": 8450, "total_steps": 28611, "tokens": 8860467200, "loss": 2.818116247653961, "ce_loss": 2.8165464997291565, "z_loss": 15.697863101959229, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27377715706825256, "tokens_per_second": 3176855.8251485135, "eta_seconds": 6654.4854093313215, "data_wait_seconds": 1.5914440155029297e-05}
+{"step": 8460, "total_steps": 28611, "tokens": 8870952960, "loss": 2.7762259244918823, "ce_loss": 2.774462103843689, "z_loss": 17.638339519500732, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3208078444004059, "tokens_per_second": 3903066.626989163, "eta_seconds": 5413.6546965122225, "data_wait_seconds": 1.823902130126953e-05}
+{"step": 8470, "total_steps": 28611, "tokens": 8881438720, "loss": 2.7785022258758545, "ce_loss": 2.7770625352859497, "z_loss": 14.396507740020752, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27266156673431396, "tokens_per_second": 4288312.454865209, "eta_seconds": 4924.867168211937, "data_wait_seconds": 0.273189902305603}
+{"step": 8480, "total_steps": 28611, "tokens": 8891924480, "loss": 2.790109932422638, "ce_loss": 2.7881128787994385, "z_loss": 19.970056533813477, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2785055935382843, "tokens_per_second": 7923774.226177221, "eta_seconds": 2663.9935532569884, "data_wait_seconds": 1.4007091522216797e-05}
+{"step": 8490, "total_steps": 28611, "tokens": 8902410240, "loss": 2.76861572265625, "ce_loss": 2.7671802639961243, "z_loss": 14.353872060775757, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3116050660610199, "tokens_per_second": 4133204.80039125, "eta_seconds": 5104.609791898728, "data_wait_seconds": 1.519918441772461e-05}
+{"step": 8500, "total_steps": 28611, "tokens": 8912896000, "loss": 2.8121728897094727, "ce_loss": 2.810706853866577, "z_loss": 14.660018920898438, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3008674383163452, "tokens_per_second": 3950932.161383182, "eta_seconds": 5337.452295970917, "data_wait_seconds": 1.436471939086914e-05}
+{"step": 8510, "total_steps": 28611, "tokens": 8923381760, "loss": 2.792626202106476, "ce_loss": 2.7909765243530273, "z_loss": 16.497233629226685, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2556679844856262, "tokens_per_second": 4173730.215422178, "eta_seconds": 5050.021225166321, "data_wait_seconds": 2.777576446533203e-05}
+{"step": 8520, "total_steps": 28611, "tokens": 8933867520, "loss": 2.8265843987464905, "ce_loss": 2.8248730897903442, "z_loss": 17.113182544708252, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.33088022470474243, "tokens_per_second": 3354201.291822112, "eta_seconds": 6280.762119841575, "data_wait_seconds": 0.26718616485595703}
+{"step": 8530, "total_steps": 28611, "tokens": 8944353280, "loss": 2.7975372076034546, "ce_loss": 2.796270251274109, "z_loss": 12.669889211654663, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.30843600630760193, "tokens_per_second": 3927824.628953995, "eta_seconds": 5360.843888187408, "data_wait_seconds": 1.3947486877441406e-05}
+{"step": 8540, "total_steps": 28611, "tokens": 8954839040, "loss": 2.8409629464149475, "ce_loss": 2.8397607803344727, "z_loss": 12.021799564361572, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3167344629764557, "tokens_per_second": 2750113.9059771136, "eta_seconds": 7652.762618398667, "data_wait_seconds": 1.5020370483398438e-05}
+{"step": 8550, "total_steps": 28611, "tokens": 8965324800, "loss": 2.7768539786338806, "ce_loss": 2.7751951217651367, "z_loss": 16.58875274658203, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.2570602297782898, "tokens_per_second": 3124133.7856038623, "eta_seconds": 6733.220975661278, "data_wait_seconds": 0.19691991806030273}
+{"step": 8560, "total_steps": 28611, "tokens": 8975810560, "loss": 2.782153606414795, "ce_loss": 2.7804641723632812, "z_loss": 16.894001960754395, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3149415850639343, "tokens_per_second": 3481001.130327209, "eta_seconds": 6039.928339242935, "data_wait_seconds": 1.4960765838623047e-05}
+{"step": 8570, "total_steps": 28611, "tokens": 8986296320, "loss": 2.7826881408691406, "ce_loss": 2.781241238117218, "z_loss": 14.469220876693726, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.3197265565395355, "tokens_per_second": 2460431.147735356, "eta_seconds": 8540.987475037575, "data_wait_seconds": 0.24492710828781128}
+{"step": 8580, "total_steps": 28611, "tokens": 8996782080, "loss": 2.7980841994285583, "ce_loss": 2.79668265581131, "z_loss": 14.01570749282837, "muon_lr": 0.03, "adamw_lr": 0.003, "grad_norm": 0.27325671911239624, "tokens_per_second": 2777287.2285189396, "eta_seconds": 7562.784878826142, "data_wait_seconds": 1.7821788787841797e-05}