Upload logs/v21_train.log with huggingface_hub

logs/v21_train.log

@@ -0,0 +1,107 @@
+PEER21_CONFIG {"ctrl_params": 598145536, "active_params_per_token": 70541312, "experts": 495616, "peer_layers": [4, 8, 12], "h_times_k": 8, "nk": 704, "pool_params": 507510784, "pool_dtype": "torch.float32", "codon_table_bytes": 536870912, "batch": 20, "seq": 512, "variant": "v21-shared-pool"}
+MUON 20971520 ADAM 69663232 SPARSE_POOL 507510784
+LOOP start 0 steps 2000000
+W0725 09:07:20.636000 10525 site-packages/torch/_inductor/utils.py:1250] [0/0] Not enough SMs to use max_autotune_gemm mode
+step=100 tokens=1022000 ce=9.6171 ppl=15019.3 bal=1.006 mut=0.001 tok_s=18,463 avg_tok_s=18,463
+step=200 tokens=2044000 ce=8.5571 ppl=5203.4 bal=1.019 mut=0.015 tok_s=33,723 avg_tok_s=26,093
+step=300 tokens=3066000 ce=7.5502 ppl=1901.0 bal=1.003 mut=0.058 tok_s=33,777 avg_tok_s=28,655
+step=400 tokens=4088000 ce=7.0258 ppl=1125.3 bal=1.001 mut=0.160 tok_s=33,834 avg_tok_s=29,949
+step=500 tokens=5110000 ce=6.9713 ppl=1065.6 bal=1.001 mut=0.267 tok_s=33,836 avg_tok_s=30,727
+step=600 tokens=6132000 ce=6.6434 ppl=767.7 bal=1.002 mut=0.367 tok_s=34,081 avg_tok_s=31,286
+step=700 tokens=7154000 ce=6.5984 ppl=733.9 bal=1.002 mut=0.461 tok_s=34,257 avg_tok_s=31,710
+step=800 tokens=8176000 ce=6.3388 ppl=566.1 bal=1.002 mut=0.546 tok_s=34,082 avg_tok_s=32,007
+step=900 tokens=9198000 ce=6.3211 ppl=556.2 bal=1.003 mut=0.600 tok_s=34,372 avg_tok_s=32,269
+step=1000 tokens=10220000 ce=6.1051 ppl=448.1 bal=1.003 mut=0.643 tok_s=34,421 avg_tok_s=32,485
+step=1100 tokens=11242000 ce=6.1585 ppl=472.7 bal=1.006 mut=0.678 tok_s=34,546 avg_tok_s=32,672
+step=1200 tokens=12264000 ce=6.1807 ppl=483.3 bal=1.006 mut=0.716 tok_s=34,629 avg_tok_s=32,835
+step=1300 tokens=13286000 ce=6.0142 ppl=409.2 bal=1.006 mut=0.741 tok_s=34,729 avg_tok_s=32,981
+step=1400 tokens=14308000 ce=5.9990 ppl=403.0 bal=1.012 mut=0.756 tok_s=34,606 avg_tok_s=33,097
+step=1500 tokens=15330000 ce=5.6967 ppl=297.9 bal=1.015 mut=0.773 tok_s=34,657 avg_tok_s=33,201
+step=1600 tokens=16352000 ce=5.8050 ppl=332.0 bal=1.022 mut=0.794 tok_s=34,535 avg_tok_s=33,284
+step=1700 tokens=17374000 ce=5.8189 ppl=336.6 bal=1.043 mut=0.810 tok_s=34,768 avg_tok_s=33,372
+step=1800 tokens=18396000 ce=5.6231 ppl=276.8 bal=1.061 mut=0.825 tok_s=34,667 avg_tok_s=33,443
+step=1900 tokens=19418000 ce=5.7805 ppl=323.9 bal=1.101 mut=0.837 tok_s=34,426 avg_tok_s=33,495
+step=2000 tokens=20440000 ce=5.5480 ppl=256.7 bal=1.193 mut=0.856 tok_s=34,117 avg_tok_s=33,526
+CKPT step=2000 valid_ce=5.6409 valid_ppl=281.7
+step=2100 tokens=21462000 ce=5.8349 ppl=342.0 bal=1.321 mut=0.858 tok_s=23,507 avg_tok_s=33,049
+step=2200 tokens=22484000 ce=5.5210 ppl=249.9 bal=1.338 mut=0.854 tok_s=33,804 avg_tok_s=33,083
+step=2300 tokens=23506000 ce=5.6261 ppl=277.6 bal=1.446 mut=0.845 tok_s=34,123 avg_tok_s=33,129
+step=2400 tokens=24528000 ce=5.5507 ppl=257.4 bal=1.399 mut=0.836 tok_s=33,899 avg_tok_s=33,161
+step=2500 tokens=25550000 ce=5.4747 ppl=238.6 bal=1.554 mut=0.827 tok_s=33,876 avg_tok_s=33,189
+step=2600 tokens=26572000 ce=5.5843 ppl=266.2 bal=2.039 mut=0.817 tok_s=33,526 avg_tok_s=33,202
+step=2700 tokens=27594000 ce=5.4278 ppl=227.7 bal=1.551 mut=0.815 tok_s=33,683 avg_tok_s=33,220
+step=2800 tokens=28616000 ce=5.5181 ppl=249.2 bal=1.812 mut=0.799 tok_s=33,594 avg_tok_s=33,234
+step=2900 tokens=29638000 ce=5.4603 ppl=235.2 bal=1.731 mut=0.787 tok_s=33,748 avg_tok_s=33,251
+step=3000 tokens=30660000 ce=5.3871 ppl=218.6 bal=2.565 mut=0.770 tok_s=33,759 avg_tok_s=33,268
+step=3100 tokens=31682000 ce=5.4449 ppl=231.6 bal=1.962 mut=0.719 tok_s=33,846 avg_tok_s=33,287
+step=3200 tokens=32704000 ce=5.1785 ppl=177.4 bal=2.069 mut=0.639 tok_s=33,765 avg_tok_s=33,302
+step=3300 tokens=33726000 ce=5.2387 ppl=188.4 bal=2.143 mut=0.602 tok_s=33,947 avg_tok_s=33,321
+step=3400 tokens=34748000 ce=5.4613 ppl=235.4 bal=2.512 mut=0.602 tok_s=33,770 avg_tok_s=33,335
+step=3500 tokens=35770000 ce=5.1858 ppl=178.7 bal=4.191 mut=0.618 tok_s=33,992 avg_tok_s=33,353
+step=3600 tokens=36792000 ce=5.3097 ppl=202.3 bal=2.291 mut=0.656 tok_s=33,892 avg_tok_s=33,368
+step=3700 tokens=37814000 ce=5.3004 ppl=200.4 bal=2.296 mut=0.666 tok_s=33,994 avg_tok_s=33,385
+step=3800 tokens=38836000 ce=5.4552 ppl=234.0 bal=2.178 mut=0.609 tok_s=34,014 avg_tok_s=33,402
+step=3900 tokens=39858000 ce=5.2438 ppl=189.4 bal=2.286 mut=0.568 tok_s=33,718 avg_tok_s=33,410
+step=4000 tokens=40880000 ce=5.2567 ppl=191.9 bal=3.247 mut=0.543 tok_s=33,895 avg_tok_s=33,422
+CKPT step=4000 valid_ce=5.2336 valid_ppl=187.5
+step=4100 tokens=41902000 ce=5.1815 ppl=178.0 bal=2.538 mut=0.572 tok_s=29,881 avg_tok_s=33,336
+step=4200 tokens=42924000 ce=5.3228 ppl=205.0 bal=2.771 mut=0.599 tok_s=33,941 avg_tok_s=33,350
+step=4300 tokens=43946000 ce=5.1848 ppl=178.5 bal=4.126 mut=0.623 tok_s=33,678 avg_tok_s=33,358
+step=4400 tokens=44968000 ce=5.1099 ppl=165.7 bal=2.398 mut=0.622 tok_s=33,840 avg_tok_s=33,369
+step=4500 tokens=45990000 ce=5.3259 ppl=205.6 bal=2.584 mut=0.621 tok_s=33,906 avg_tok_s=33,381
+step=4600 tokens=47012000 ce=5.3504 ppl=210.7 bal=2.438 mut=0.611 tok_s=33,912 avg_tok_s=33,392
+step=4700 tokens=48034000 ce=4.9649 ppl=143.3 bal=2.337 mut=0.586 tok_s=33,687 avg_tok_s=33,398
+step=4800 tokens=49056000 ce=5.1726 ppl=176.4 bal=2.220 mut=0.586 tok_s=33,761 avg_tok_s=33,406
+step=4900 tokens=50078000 ce=5.0785 ppl=160.5 bal=2.316 mut=0.595 tok_s=33,918 avg_tok_s=33,416
+step=5000 tokens=51100000 ce=5.0890 ppl=162.2 bal=2.871 mut=0.601 tok_s=33,875 avg_tok_s=33,426
+step=5100 tokens=52122000 ce=4.9443 ppl=140.4 bal=2.315 mut=0.611 tok_s=33,854 avg_tok_s=33,434
+step=5200 tokens=53144000 ce=4.9766 ppl=145.0 bal=2.761 mut=0.609 tok_s=33,826 avg_tok_s=33,441
+step=5300 tokens=54166000 ce=5.1149 ppl=166.5 bal=2.294 mut=0.593 tok_s=33,820 avg_tok_s=33,449
+step=5400 tokens=55188000 ce=4.9950 ppl=147.7 bal=2.640 mut=0.569 tok_s=33,832 avg_tok_s=33,456
+step=5500 tokens=56210000 ce=4.9615 ppl=142.8 bal=2.426 mut=0.560 tok_s=33,485 avg_tok_s=33,456
+step=5600 tokens=57232000 ce=4.9050 ppl=135.0 bal=2.344 mut=0.560 tok_s=33,633 avg_tok_s=33,459
+step=5700 tokens=58254000 ce=4.9484 ppl=140.9 bal=2.408 mut=0.559 tok_s=33,679 avg_tok_s=33,463
+step=5800 tokens=59276000 ce=5.0004 ppl=148.5 bal=2.389 mut=0.565 tok_s=33,676 avg_tok_s=33,467
+step=5900 tokens=60298000 ce=4.9394 ppl=139.7 bal=2.374 mut=0.544 tok_s=33,624 avg_tok_s=33,470
+step=6000 tokens=61320000 ce=4.9389 ppl=139.6 bal=2.541 mut=0.545 tok_s=33,443 avg_tok_s=33,469
+CKPT step=6000 valid_ce=5.0876 valid_ppl=162.0
+step=6100 tokens=62342000 ce=4.9210 ppl=137.1 bal=2.385 mut=0.556 tok_s=29,236 avg_tok_s=33,400
+step=6200 tokens=63364000 ce=4.7055 ppl=110.6 bal=2.540 mut=0.546 tok_s=33,479 avg_tok_s=33,401
+step=6300 tokens=64386000 ce=5.0065 ppl=149.4 bal=2.731 mut=0.549 tok_s=33,486 avg_tok_s=33,402
+step=6400 tokens=65408000 ce=4.6567 ppl=105.3 bal=2.588 mut=0.578 tok_s=33,367 avg_tok_s=33,402
+step=6500 tokens=66430000 ce=4.8899 ppl=132.9 bal=2.382 mut=0.599 tok_s=33,409 avg_tok_s=33,402
+step=6600 tokens=67452000 ce=4.8022 ppl=121.8 bal=3.084 mut=0.600 tok_s=33,358 avg_tok_s=33,401
+step=6700 tokens=68474000 ce=4.9022 ppl=134.6 bal=2.546 mut=0.578 tok_s=33,334 avg_tok_s=33,400
+step=6800 tokens=69496000 ce=4.8258 ppl=124.7 bal=2.462 mut=0.586 tok_s=33,145 avg_tok_s=33,397
+step=6900 tokens=70518000 ce=4.8288 ppl=125.1 bal=2.482 mut=0.606 tok_s=32,974 avg_tok_s=33,390
+step=7000 tokens=71540000 ce=4.7357 ppl=113.9 bal=2.266 mut=0.618 tok_s=33,183 avg_tok_s=33,387
+step=7100 tokens=72562000 ce=4.9510 ppl=141.3 bal=2.559 mut=0.609 tok_s=33,302 avg_tok_s=33,386
+step=7200 tokens=73584000 ce=4.8792 ppl=131.5 bal=2.579 mut=0.603 tok_s=33,328 avg_tok_s=33,385
+step=7300 tokens=74606000 ce=4.8494 ppl=127.7 bal=2.413 mut=0.602 tok_s=33,596 avg_tok_s=33,388
+step=7400 tokens=75628000 ce=4.8537 ppl=128.2 bal=2.475 mut=0.619 tok_s=33,409 avg_tok_s=33,389
+step=7500 tokens=76650000 ce=4.6232 ppl=101.8 bal=2.694 mut=0.613 tok_s=33,649 avg_tok_s=33,392
+step=7600 tokens=77672000 ce=4.8696 ppl=130.3 bal=2.435 mut=0.617 tok_s=33,491 avg_tok_s=33,393
+step=7700 tokens=78694000 ce=4.6999 ppl=109.9 bal=2.337 mut=0.608 tok_s=33,580 avg_tok_s=33,396
+step=7800 tokens=79716000 ce=4.6800 ppl=107.8 bal=2.298 mut=0.611 tok_s=33,307 avg_tok_s=33,395
+step=7900 tokens=80738000 ce=5.0483 ppl=155.8 bal=2.868 mut=0.617 tok_s=33,506 avg_tok_s=33,396
+step=8000 tokens=81760000 ce=4.6872 ppl=108.5 bal=2.746 mut=0.632 tok_s=33,403 avg_tok_s=33,396
+CKPT step=8000 valid_ce=4.8020 valid_ppl=121.8
+step=8100 tokens=82782000 ce=4.7238 ppl=112.6 bal=2.205 mut=0.630 tok_s=29,320 avg_tok_s=33,346
+step=8200 tokens=83804000 ce=4.6063 ppl=100.1 bal=2.304 mut=0.637 tok_s=33,441 avg_tok_s=33,347
+step=8300 tokens=84826000 ce=4.7280 ppl=113.1 bal=2.461 mut=0.637 tok_s=33,194 avg_tok_s=33,345
+step=8400 tokens=85848000 ce=4.7639 ppl=117.2 bal=2.576 mut=0.645 tok_s=33,310 avg_tok_s=33,345
+step=8500 tokens=86870000 ce=4.7843 ppl=119.6 bal=2.413 mut=0.649 tok_s=33,337 avg_tok_s=33,345
+step=8600 tokens=87892000 ce=4.6088 ppl=100.4 bal=2.291 mut=0.642 tok_s=33,366 avg_tok_s=33,345
+step=8700 tokens=88914000 ce=4.6687 ppl=106.6 bal=2.177 mut=0.640 tok_s=33,315 avg_tok_s=33,345
+step=8800 tokens=89936000 ce=4.8576 ppl=128.7 bal=2.237 mut=0.649 tok_s=33,187 avg_tok_s=33,343
+step=8900 tokens=90958000 ce=4.6739 ppl=107.1 bal=2.313 mut=0.652 tok_s=33,287 avg_tok_s=33,342
+step=9000 tokens=91980000 ce=4.6667 ppl=106.4 bal=2.148 mut=0.665 tok_s=33,264 avg_tok_s=33,341
+step=9100 tokens=93002000 ce=4.6407 ppl=103.6 bal=2.344 mut=0.650 tok_s=33,063 avg_tok_s=33,338
+step=9200 tokens=94024000 ce=4.8073 ppl=122.4 bal=2.258 mut=0.658 tok_s=33,111 avg_tok_s=33,336
+step=9300 tokens=95046000 ce=4.7432 ppl=114.8 bal=2.300 mut=0.679 tok_s=33,139 avg_tok_s=33,334
+step=9400 tokens=96068000 ce=4.7647 ppl=117.3 bal=2.264 mut=0.670 tok_s=33,214 avg_tok_s=33,332
+step=9500 tokens=97090000 ce=4.8375 ppl=126.1 bal=2.328 mut=0.676 tok_s=33,178 avg_tok_s=33,331
+step=9600 tokens=98112000 ce=4.9331 ppl=138.8 bal=4.250 mut=0.669 tok_s=32,721 avg_tok_s=33,324
+step=9700 tokens=99134000 ce=4.5852 ppl=98.0 bal=2.140 mut=0.662 tok_s=33,031 avg_tok_s=33,321
+step=9800 tokens=100156000 ce=4.7557 ppl=116.2 bal=2.377 mut=0.663 tok_s=32,996 avg_tok_s=33,318
+step=9900 tokens=101178000 ce=4.8145 ppl=123.3 bal=2.296 mut=0.672 tok_s=32,907 avg_tok_s=33,314