File size: 13,114 Bytes
8a7fbf8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 |
=== variant: baseline | steps=1526 active=75.5M kv_heads=4 qk_norm=True ===
step 20 | loss 8.5372 | lr 1.57e-04 | gnorm 1.87 | 55027 tok/s | mfu 46.1%
step 40 | loss 7.1584 | lr 3.07e-04 | gnorm 1.19 | 42982 tok/s | mfu 40.6%
step 60 | loss 6.5715 | lr 4.57e-04 | gnorm 0.98 | 42606 tok/s | mfu 37.9%
step 80 | loss 6.4003 | lr 6.00e-04 | gnorm 1.79 | 44604 tok/s | mfu 34.9%
step 100 | loss 6.1538 | lr 6.00e-04 | gnorm 0.53 | 42507 tok/s | mfu 34.4%
step 120 | loss 5.9617 | lr 5.99e-04 | gnorm 1.17 | 42811 tok/s | mfu 34.0%
step 140 | loss 5.7707 | lr 5.98e-04 | gnorm 0.65 | 38468 tok/s | mfu 33.1%
step 160 | loss 5.5960 | lr 5.96e-04 | gnorm 0.39 | 42081 tok/s | mfu 32.6%
step 180 | loss 5.4231 | lr 5.94e-04 | gnorm 0.60 | 41134 tok/s | mfu 32.1%
step 200 | loss 5.3285 | lr 5.91e-04 | gnorm 0.40 | 42419 tok/s | mfu 33.0%
step 220 | loss 5.1528 | lr 5.88e-04 | gnorm 0.38 | 42484 tok/s | mfu 33.5%
step 240 | loss 5.0686 | lr 5.84e-04 | gnorm 0.44 | 40167 tok/s | mfu 33.8%
step 260 | loss 4.9688 | lr 5.80e-04 | gnorm 0.63 | 39197 tok/s | mfu 32.9%
step 280 | loss 4.9279 | lr 5.75e-04 | gnorm 0.50 | 41673 tok/s | mfu 32.2%
step 300 | loss 4.8723 | lr 5.70e-04 | gnorm 0.42 | 42409 tok/s | mfu 32.1%
step 320 | loss 4.7988 | lr 5.64e-04 | gnorm 0.41 | 42722 tok/s | mfu 33.0%
step 340 | loss 4.7484 | lr 5.58e-04 | gnorm 0.45 | 42223 tok/s | mfu 33.8%
step 360 | loss 4.6266 | lr 5.52e-04 | gnorm 0.45 | 43267 tok/s | mfu 34.0%
step 380 | loss 4.5745 | lr 5.45e-04 | gnorm 0.35 | 41488 tok/s | mfu 34.1%
step 400 | loss 4.5889 | lr 5.37e-04 | gnorm 0.40 | 41204 tok/s | mfu 34.0%
step 420 | loss 4.4896 | lr 5.30e-04 | gnorm 0.35 | 42736 tok/s | mfu 33.2%
step 440 | loss 4.5185 | lr 5.22e-04 | gnorm 0.33 | 39767 tok/s | mfu 32.8%
step 460 | loss 4.3627 | lr 5.13e-04 | gnorm 0.34 | 42151 tok/s | mfu 33.2%
step 480 | loss 4.3364 | lr 5.04e-04 | gnorm 0.37 | 41317 tok/s | mfu 33.7%
step 500 | loss 4.3175 | lr 4.95e-04 | gnorm 0.34 | 37705 tok/s | mfu 33.3%
step 520 | loss 4.2735 | lr 4.86e-04 | gnorm 0.36 | 37650 tok/s | mfu 31.9%
step 540 | loss 4.2721 | lr 4.76e-04 | gnorm 0.33 | 38594 tok/s | mfu 31.2%
step 560 | loss 4.2306 | lr 4.66e-04 | gnorm 0.31 | 37482 tok/s | mfu 30.8%
step 580 | loss 4.2080 | lr 4.56e-04 | gnorm 0.37 | 38511 tok/s | mfu 30.5%
step 600 | loss 4.1947 | lr 4.45e-04 | gnorm 0.33 | 37695 tok/s | mfu 30.4%
step 620 | loss 4.1235 | lr 4.35e-04 | gnorm 0.30 | 43363 tok/s | mfu 30.8%
step 640 | loss 4.1144 | lr 4.24e-04 | gnorm 0.31 | 41134 tok/s | mfu 31.8%
step 660 | loss 4.0693 | lr 4.13e-04 | gnorm 0.31 | 40037 tok/s | mfu 32.4%
step 680 | loss 4.0814 | lr 4.01e-04 | gnorm 0.31 | 39738 tok/s | mfu 32.4%
step 700 | loss 4.0742 | lr 3.90e-04 | gnorm 0.34 | 43006 tok/s | mfu 32.4%
step 720 | loss 4.0225 | lr 3.78e-04 | gnorm 0.31 | 41853 tok/s | mfu 33.1%
step 740 | loss 4.0209 | lr 3.67e-04 | gnorm 0.30 | 43795 tok/s | mfu 33.8%
step 760 | loss 4.0052 | lr 3.55e-04 | gnorm 0.33 | 45156 tok/s | mfu 34.5%
step 780 | loss 4.0007 | lr 3.43e-04 | gnorm 0.32 | 40651 tok/s | mfu 34.6%
step 800 | loss 3.9838 | lr 3.32e-04 | gnorm 0.27 | 40845 tok/s | mfu 33.6%
step 820 | loss 3.9496 | lr 3.20e-04 | gnorm 0.31 | 37278 tok/s | mfu 32.1%
step 840 | loss 3.9211 | lr 3.08e-04 | gnorm 0.31 | 39670 tok/s | mfu 31.2%
step 860 | loss 3.8893 | lr 2.97e-04 | gnorm 0.30 | 43320 tok/s | mfu 31.4%
step 880 | loss 3.9187 | lr 2.85e-04 | gnorm 0.29 | 39639 tok/s | mfu 33.0%
step 900 | loss 3.8916 | lr 2.74e-04 | gnorm 0.29 | 44306 tok/s | mfu 33.9%
step 920 | loss 3.8696 | lr 2.62e-04 | gnorm 0.32 | 44465 tok/s | mfu 33.8%
step 940 | loss 3.8972 | lr 2.51e-04 | gnorm 0.30 | 40442 tok/s | mfu 33.3%
step 960 | loss 3.8392 | lr 2.40e-04 | gnorm 0.28 | 42065 tok/s | mfu 32.7%
step 980 | loss 3.9060 | lr 2.29e-04 | gnorm 0.30 | 42386 tok/s | mfu 32.8%
step 1000 | loss 3.8391 | lr 2.18e-04 | gnorm 0.29 | 38505 tok/s | mfu 33.0%
step 1020 | loss 3.7921 | lr 2.07e-04 | gnorm 0.28 | 38108 tok/s | mfu 31.9%
step 1040 | loss 3.7709 | lr 1.97e-04 | gnorm 0.30 | 37706 tok/s | mfu 30.9%
step 1060 | loss 3.8668 | lr 1.87e-04 | gnorm 0.28 | 38858 tok/s | mfu 30.6%
step 1080 | loss 3.8370 | lr 1.77e-04 | gnorm 0.29 | 39063 tok/s | mfu 30.4%
step 1100 | loss 3.8401 | lr 1.68e-04 | gnorm 0.28 | 35781 tok/s | mfu 30.0%
step 1120 | loss 3.8065 | lr 1.58e-04 | gnorm 0.29 | 39247 tok/s | mfu 29.8%
step 1140 | loss 3.7553 | lr 1.50e-04 | gnorm 0.29 | 37239 tok/s | mfu 30.3%
step 1160 | loss 3.8441 | lr 1.41e-04 | gnorm 0.29 | 38951 tok/s | mfu 30.9%
step 1180 | loss 3.7467 | lr 1.33e-04 | gnorm 0.28 | 42882 tok/s | mfu 31.0%
step 1200 | loss 3.7402 | lr 1.25e-04 | gnorm 0.29 | 41228 tok/s | mfu 31.5%
step 1220 | loss 3.7622 | lr 1.18e-04 | gnorm 0.28 | 40652 tok/s | mfu 31.7%
step 1240 | loss 3.7827 | lr 1.10e-04 | gnorm 0.29 | 40908 tok/s | mfu 32.6%
step 1260 | loss 3.6756 | lr 1.04e-04 | gnorm 0.28 | 42273 tok/s | mfu 33.3%
step 1280 | loss 3.7270 | lr 9.77e-05 | gnorm 0.29 | 39554 tok/s | mfu 32.9%
step 1300 | loss 3.7490 | lr 9.19e-05 | gnorm 0.28 | 41916 tok/s | mfu 32.2%
step 1320 | loss 3.8031 | lr 8.66e-05 | gnorm 0.29 | 43407 tok/s | mfu 32.4%
step 1340 | loss 3.7544 | lr 8.17e-05 | gnorm 0.29 | 40600 tok/s | mfu 32.9%
step 1360 | loss 3.7156 | lr 7.74e-05 | gnorm 0.28 | 42365 tok/s | mfu 33.4%
step 1380 | loss 3.7191 | lr 7.35e-05 | gnorm 0.28 | 43099 tok/s | mfu 33.8%
step 1400 | loss 3.7607 | lr 7.01e-05 | gnorm 0.29 | 42788 tok/s | mfu 33.8%
step 1420 | loss 3.7190 | lr 6.71e-05 | gnorm 0.30 | 42979 tok/s | mfu 34.2%
step 1440 | loss 3.7330 | lr 6.47e-05 | gnorm 0.29 | 43298 tok/s | mfu 34.7%
step 1460 | loss 3.7146 | lr 6.28e-05 | gnorm 0.30 | 43686 tok/s | mfu 35.1%
step 1480 | loss 3.7217 | lr 6.13e-05 | gnorm 0.29 | 37792 tok/s | mfu 33.8%
step 1500 | loss 3.7263 | lr 6.04e-05 | gnorm 0.29 | 41984 tok/s | mfu 32.6%
step 1520 | loss 3.7223 | lr 6.00e-05 | gnorm 0.28 | 44049 tok/s | mfu 32.8%
=== variant: muon | steps=1526 active=75.5M kv_heads=4 qk_norm=True ===
step 20 | loss 8.6267 | lr 5.25e-03 | gnorm 6.41 | 42743 tok/s | mfu 34.5%
step 40 | loss 6.9825 | lr 1.02e-02 | gnorm 1.61 | 43638 tok/s | mfu 35.4%
step 60 | loss 6.3518 | lr 1.52e-02 | gnorm 0.81 | 42974 tok/s | mfu 35.0%
step 80 | loss 6.0728 | lr 2.00e-02 | gnorm 0.75 | 43504 tok/s | mfu 34.8%
step 100 | loss 5.8258 | lr 2.00e-02 | gnorm 0.70 | 43436 tok/s | mfu 34.6%
step 120 | loss 5.5670 | lr 2.00e-02 | gnorm 0.80 | 41847 tok/s | mfu 34.4%
step 140 | loss 5.3600 | lr 1.99e-02 | gnorm 0.76 | 39781 tok/s | mfu 33.5%
step 160 | loss 5.2085 | lr 1.99e-02 | gnorm 1.00 | 39943 tok/s | mfu 32.8%
step 180 | loss 5.0118 | lr 1.98e-02 | gnorm 0.80 | 40514 tok/s | mfu 31.9%
step 200 | loss 4.9736 | lr 1.97e-02 | gnorm 0.79 | 41031 tok/s | mfu 31.6%
step 220 | loss 4.7818 | lr 1.96e-02 | gnorm 0.77 | 39914 tok/s | mfu 32.3%
step 240 | loss 4.7016 | lr 1.95e-02 | gnorm 0.87 | 41648 tok/s | mfu 33.1%
step 260 | loss 4.6261 | lr 1.93e-02 | gnorm 0.93 | 42507 tok/s | mfu 33.5%
step 280 | loss 4.5692 | lr 1.92e-02 | gnorm 0.71 | 41136 tok/s | mfu 34.2%
step 300 | loss 4.5062 | lr 1.90e-02 | gnorm 0.92 | 41734 tok/s | mfu 34.5%
step 320 | loss 4.4249 | lr 1.88e-02 | gnorm 0.94 | 41605 tok/s | mfu 34.0%
step 340 | loss 4.3690 | lr 1.86e-02 | gnorm 1.13 | 41030 tok/s | mfu 33.2%
step 360 | loss 4.2474 | lr 1.84e-02 | gnorm 1.28 | 41540 tok/s | mfu 33.3%
step 380 | loss 4.2129 | lr 1.82e-02 | gnorm 1.06 | 43978 tok/s | mfu 33.6%
step 400 | loss 4.2083 | lr 1.79e-02 | gnorm 1.05 | 43016 tok/s | mfu 34.0%
step 420 | loss 4.1362 | lr 1.77e-02 | gnorm 1.22 | 42257 tok/s | mfu 34.2%
step 440 | loss 4.1898 | lr 1.74e-02 | gnorm 1.14 | 42386 tok/s | mfu 34.2%
step 460 | loss 4.0607 | lr 1.71e-02 | gnorm 1.06 | 45268 tok/s | mfu 34.3%
step 480 | loss 4.0260 | lr 1.68e-02 | gnorm 0.95 | 40331 tok/s | mfu 34.3%
step 500 | loss 4.0340 | lr 1.65e-02 | gnorm 1.12 | 39582 tok/s | mfu 33.2%
step 520 | loss 3.9893 | lr 1.62e-02 | gnorm 1.13 | 37813 tok/s | mfu 31.6%
step 540 | loss 4.0165 | lr 1.59e-02 | gnorm 1.26 | 42983 tok/s | mfu 31.3%
step 560 | loss 3.9790 | lr 1.55e-02 | gnorm 1.16 | 39197 tok/s | mfu 31.8%
step 580 | loss 3.9486 | lr 1.52e-02 | gnorm 1.12 | 41317 tok/s | mfu 32.6%
step 600 | loss 3.9543 | lr 1.48e-02 | gnorm 0.90 | 39782 tok/s | mfu 32.1%
step 620 | loss 3.8925 | lr 1.45e-02 | gnorm 1.08 | 40467 tok/s | mfu 32.3%
step 640 | loss 3.8851 | lr 1.41e-02 | gnorm 1.11 | 40908 tok/s | mfu 32.4%
step 660 | loss 3.8403 | lr 1.38e-02 | gnorm 1.00 | 41973 tok/s | mfu 32.8%
step 680 | loss 3.8651 | lr 1.34e-02 | gnorm 0.97 | 40444 tok/s | mfu 33.2%
step 700 | loss 3.8577 | lr 1.30e-02 | gnorm 0.90 | 41868 tok/s | mfu 33.4%
step 720 | loss 3.8058 | lr 1.26e-02 | gnorm 1.06 | 41874 tok/s | mfu 33.4%
step 740 | loss 3.8135 | lr 1.22e-02 | gnorm 0.73 | 40232 tok/s | mfu 33.3%
step 760 | loss 3.7953 | lr 1.18e-02 | gnorm 0.89 | 41992 tok/s | mfu 33.0%
step 780 | loss 3.7931 | lr 1.14e-02 | gnorm 0.92 | 42224 tok/s | mfu 33.2%
step 800 | loss 3.7886 | lr 1.11e-02 | gnorm 0.86 | 39033 tok/s | mfu 33.4%
step 820 | loss 3.7428 | lr 1.07e-02 | gnorm 0.91 | 38959 tok/s | mfu 32.8%
step 840 | loss 3.7293 | lr 1.03e-02 | gnorm 0.75 | 41662 tok/s | mfu 31.8%
step 860 | loss 3.6887 | lr 9.89e-03 | gnorm 0.68 | 40495 tok/s | mfu 32.0%
step 880 | loss 3.7283 | lr 9.50e-03 | gnorm 0.79 | 41993 tok/s | mfu 32.4%
step 900 | loss 3.7040 | lr 9.12e-03 | gnorm 0.60 | 41455 tok/s | mfu 33.0%
step 920 | loss 3.6809 | lr 8.74e-03 | gnorm 0.77 | 42602 tok/s | mfu 33.9%
step 940 | loss 3.7061 | lr 8.36e-03 | gnorm 0.52 | 43552 tok/s | mfu 34.5%
step 960 | loss 3.6564 | lr 7.99e-03 | gnorm 0.55 | 41086 tok/s | mfu 34.1%
step 980 | loss 3.7209 | lr 7.62e-03 | gnorm 0.49 | 43097 tok/s | mfu 33.7%
step 1000 | loss 3.6588 | lr 7.26e-03 | gnorm 0.57 | 44568 tok/s | mfu 33.8%
step 1020 | loss 3.6149 | lr 6.91e-03 | gnorm 0.46 | 42557 tok/s | mfu 34.3%
step 1040 | loss 3.5919 | lr 6.57e-03 | gnorm 0.37 | 41775 tok/s | mfu 34.1%
step 1060 | loss 3.6848 | lr 6.23e-03 | gnorm 0.44 | 41294 tok/s | mfu 33.6%
step 1080 | loss 3.6564 | lr 5.90e-03 | gnorm 0.45 | 42732 tok/s | mfu 33.7%
step 1100 | loss 3.6598 | lr 5.59e-03 | gnorm 0.40 | 42906 tok/s | mfu 34.1%
step 1120 | loss 3.6289 | lr 5.28e-03 | gnorm 0.38 | 46240 tok/s | mfu 34.8%
step 1140 | loss 3.5832 | lr 4.98e-03 | gnorm 0.47 | 40378 tok/s | mfu 34.6%
step 1160 | loss 3.6692 | lr 4.70e-03 | gnorm 0.26 | 37583 tok/s | mfu 33.1%
step 1180 | loss 3.5746 | lr 4.43e-03 | gnorm 0.29 | 38996 tok/s | mfu 31.2%
step 1200 | loss 3.5696 | lr 4.16e-03 | gnorm 0.31 | 40048 tok/s | mfu 30.7%
step 1220 | loss 3.5924 | lr 3.92e-03 | gnorm 0.33 | 38408 tok/s | mfu 31.0%
step 1240 | loss 3.6104 | lr 3.68e-03 | gnorm 0.31 | 38796 tok/s | mfu 31.0%
step 1260 | loss 3.5086 | lr 3.46e-03 | gnorm 0.31 | 40969 tok/s | mfu 30.9%
step 1280 | loss 3.5568 | lr 3.26e-03 | gnorm 0.24 | 36839 tok/s | mfu 31.1%
step 1300 | loss 3.5852 | lr 3.06e-03 | gnorm 0.29 | 38778 tok/s | mfu 30.9%
step 1320 | loss 3.6331 | lr 2.89e-03 | gnorm 0.30 | 38501 tok/s | mfu 30.8%
step 1340 | loss 3.5862 | lr 2.72e-03 | gnorm 0.27 | 40901 tok/s | mfu 31.4%
step 1360 | loss 3.5454 | lr 2.58e-03 | gnorm 0.29 | 37918 tok/s | mfu 31.7%
step 1380 | loss 3.5468 | lr 2.45e-03 | gnorm 0.21 | 36797 tok/s | mfu 31.1%
step 1400 | loss 3.5933 | lr 2.34e-03 | gnorm 0.33 | 37308 tok/s | mfu 30.2%
step 1420 | loss 3.5502 | lr 2.24e-03 | gnorm 0.28 | 38868 tok/s | mfu 30.0%
step 1440 | loss 3.5684 | lr 2.16e-03 | gnorm 0.24 | 41055 tok/s | mfu 30.6%
step 1460 | loss 3.5451 | lr 2.09e-03 | gnorm 0.25 | 39487 tok/s | mfu 31.3%
step 1480 | loss 3.5535 | lr 2.04e-03 | gnorm 0.27 | 39599 tok/s | mfu 31.6%
step 1500 | loss 3.5598 | lr 2.01e-03 | gnorm 0.25 | 39831 tok/s | mfu 31.5%
step 1520 | loss 3.5581 | lr 2.00e-03 | gnorm 0.24 | 38053 tok/s | mfu 31.5%
wrote /workspace/matilda-mini/docs/ABLATIONS.md and /workspace/matilda-mini/results/ablations/ablations.json
baseline loss=3.7236 mfu=0.3381
muon loss=3.557 mfu=0.313
EXIT_CODE=0
|