train.log

train.log

@@ -0,0 +1,36 @@
+[12:23:09] trial=exp277_lichess_opening_stream arch=squares64 device=cuda vocab=1968
+[12:23:09] train bs=528 accum=1 grad_ckpt=False compile=True elo_every=0 save_every=150
+[12:23:09] model=Squares64RecurrentConfig(encoder_dim=256, hidden_dim=736, num_heads=8, ffn_ratio=4, dropout=0.05, use_swiglu=True, use_qk_norm=True, zero_init_out_proj=True, gradient_checkpointing=False, prefix_layers=4, recurrent_layers=7, recurrent_unrolls=3, suffix_layers=4, policy_head_dim=384, value_hidden=384, n_value_classes=3, use_history=False, use_threat_head=False)
+[12:23:10] params=98.97M
+[12:23:11] WEIGHTS-ONLY WARM START /root/transform/outputs/exp277_lichess_opening_stream/init.pt from eval/SWA snapshot (optimizer/RNG re-initialized; step reset to 0)
+[12:23:11] optimizer=PolarNorMuon (97.7M) + AdamW aux (1.3M) compile_polar=True
+[12:23:11] force_lr muon=0.002 adam=3e-05
+[12:23:12] torch.compile enabled (mode=default); will fall back if inductor fails
+[12:23:12] external eval lichess_opening n=8,192 /root/transform/outputs/exp277_lichess_opening_stream/lichess_eval.pt
+[12:23:12] soft train=71,808 val=0 blocked=0 targets empty=0 unnorm=0 external_eval
+[12:24:37] batch probe ok bs=528 peak_vram=35.57GB compile=True
+[12:24:37] average_recurrent_grads() divides bank .grad by unroll count. That does NOT divide optimizer updates by the same amount: Polar-NorMuon is scale-invariant after orthogonalization, and RMS-Adam (m/sqrt(v)) is also approximately scale-invariant for a stationary grad. What does scale is SGD-style updates and the pre-clip global grad norm (so clipping fires less often after averaging). Use explicit param-group LRs if different update sizes are intended.
+[12:24:37] exposure deep/shallow odds=0.0x (mix=0.0 deep_n=0 shallow_n=71808)
+[12:24:46] step 1/8000 | loss=2.0407 | lr=2.00e-03,3.00e-05 | clip=1/25 | mix s/d/b=528/0/0 q=0 p=0 | 55 pos/s | vram=35.57GB
+[12:24:50] soft inbox +1,760,000 train_n=1,831,808 shards=shard_000001,shard_000002,shard_000003,shard_000004,shard_000005,shard_000006,shard_000007,shard_000008,shard_000009,shard_000010,shard_000011,shard_000012,shard_000013,shard_000014,shard_000015,shard_000016,shard_000017,shard_000018,shard_000019,shard_000020,shard_000021,shard_000022
+[12:25:11] step 25/8000 | loss=1.9109 | lr=2.00e-03,3.00e-05 | clip=19/25 | mix s/d/b=13200/0/0 q=0 p=0 | 391 pos/s | vram=35.72GB
+[12:25:12] soft inbox +480,000 train_n=2,311,808 shards=shard_000023,shard_000024,shard_000025,shard_000026,shard_000027,shard_000028
+[12:25:33] step 50/8000 | loss=1.8446 | lr=2.00e-03,3.00e-05 | clip=7/25 | mix s/d/b=26400/0/0 q=0 p=0 | 467 pos/s | vram=35.72GB
+[12:25:35] soft inbox +400,000 train_n=2,711,808 shards=shard_000029,shard_000030,shard_000031,shard_000032,shard_000033
+[12:25:57] step 75/8000 | loss=1.8202 | lr=2.00e-03,3.00e-05 | clip=0/25 | mix s/d/b=39600/0/0 q=0 p=0 | 495 pos/s | vram=35.72GB
+[12:25:58] soft inbox +480,000 train_n=3,191,808 shards=shard_000034,shard_000035,shard_000036,shard_000037,shard_000038,shard_000039
+[12:26:20] step 100/8000 | loss=1.8066 | lr=2.00e-03,3.00e-05 | clip=2/25 | mix s/d/b=52800/0/0 q=0 p=0 | 510 pos/s | vram=35.72GB
+[12:26:22] soft inbox +312,475 train_n=3,504,283 shards=shard_000040,shard_000041,shard_000042,shard_000043
+[12:26:31] val/lichess_opening hard_ce=1.5399 soft_ce=1.5399 soft_temp_ce=1.5399 teacher_entropy=0.0000 teacher_kl=1.5399 wdl_ce=1.0429 value_rows=8192.0000
+[12:26:52] step 125/8000 | loss=1.8346 | lr=2.00e-03,3.00e-05 | clip=1/25 | mix s/d/b=66000/0/0 q=0 p=0 | 487 pos/s | vram=35.72GB
+[12:26:55] soft inbox +640,000 train_n=4,144,283 shards=shard_000044,shard_000045,shard_000046,shard_000047,shard_000048,shard_000049,shard_000050,shard_000051
+[12:27:17] step 150/8000 | loss=1.8197 | lr=2.00e-03,3.00e-05 | clip=5/25 | mix s/d/b=79200/0/0 q=0 p=0 | 495 pos/s | vram=35.72GB
+[12:27:19] soft inbox +400,000 train_n=4,544,283 shards=shard_000052,shard_000053,shard_000054,shard_000055,shard_000056
+[12:27:20] ckpt latest.pt (live weights) steps=150 status=mid
+[12:27:41] step 175/8000 | loss=1.8112 | lr=2.00e-03,3.00e-05 | clip=2/25 | mix s/d/b=92400/0/0 q=0 p=0 | 501 pos/s | vram=35.72GB
+[12:27:44] soft inbox +480,000 train_n=5,024,283 shards=shard_000057,shard_000058,shard_000059,shard_000060,shard_000061,shard_000062
+[12:28:06] step 200/8000 | loss=1.8148 | lr=2.00e-03,3.00e-05 | clip=0/25 | mix s/d/b=105600/0/0 q=0 p=0 | 506 pos/s | vram=35.72GB
+[12:28:09] soft inbox +480,000 train_n=5,504,283 shards=shard_000063,shard_000064,shard_000065,shard_000066,shard_000067,shard_000068
+[12:28:18] val/lichess_opening hard_ce=1.5239 soft_ce=1.5239 soft_temp_ce=1.5239 teacher_entropy=0.0000 teacher_kl=1.5239 wdl_ce=1.0404 value_rows=8192.0000
+[12:28:40] step 225/8000 | loss=1.8335 | lr=2.00e-03,3.00e-05 | clip=0/25 | mix s/d/b=118800/0/0 q=0 p=0 | 488 pos/s | vram=35.72GB
+[12:28:44] soft inbox +640,000 train_n=6,144,283 shards=shard_000069,shard_000070,shard_000071,shard_000072,shard_000073,shard_000074,shard_000075,shard_000076