Blame for R2/run/train.log

[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1
[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288
[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14)
[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x)
[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024
[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved
[13:59:59] torch.compile enabled
[13:59:59] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05
[14:04:20] step 20/4768 loss 10.3979 lr 2.40e-05 gnorm 2.60 40,239 tok/s peakVRAM 13.7GB
[14:07:31] step 40/4768 loss 9.5831 lr 4.80e-05 gnorm 1.42 54,808 tok/s peakVRAM 13.7GB
[14:10:43] step 60/4768 loss 8.7021 lr 7.20e-05 gnorm 0.76 54,817 tok/s peakVRAM 13.7GB
[14:13:54] step 80/4768 loss 8.0482 lr 9.60e-05 gnorm 1.57 54,813 tok/s peakVRAM 13.7GB
[14:17:05] step 100/4768 loss 7.6036 lr 1.20e-04 gnorm 0.99 54,787 tok/s peakVRAM 13.7GB
[14:20:17] step 120/4768 loss 7.1345 lr 1.44e-04 gnorm 0.74 54,772 tok/s peakVRAM 13.7GB
[14:23:28] step 140/4768 loss 6.7198 lr 1.68e-04 gnorm 0.81 54,761 tok/s peakVRAM 13.7GB
[14:26:40] step 160/4768 loss 6.3578 lr 1.92e-04 gnorm 0.82 54,756 tok/s peakVRAM 13.7GB
[14:29:51] step 180/4768 loss 6.0886 lr 2.16e-04 gnorm 0.72 54,754 tok/s peakVRAM 13.7GB
[14:33:03] step 200/4768 loss 5.8673 lr 2.40e-04 gnorm 0.70 54,750 tok/s peakVRAM 13.7GB
[14:36:14] step 220/4768 loss 5.6795 lr 2.64e-04 gnorm 0.63 54,744 tok/s peakVRAM 13.7GB
[14:39:26] step 240/4768 loss 5.5068 lr 2.88e-04 gnorm 0.73 54,744 tok/s peakVRAM 13.7GB
[14:42:37] step 260/4768 loss 5.3420 lr 3.12e-04 gnorm 0.89 54,745 tok/s peakVRAM 13.7GB
[14:45:49] step 280/4768 loss 5.1838 lr 3.36e-04 gnorm 0.74 54,746 tok/s peakVRAM 13.7GB
[14:49:00] step 300/4768 loss 5.0376 lr 3.60e-04 gnorm 0.77 54,747 tok/s peakVRAM 13.7GB
[14:52:12] step 320/4768 loss 4.9032 lr 3.84e-04 gnorm 0.87 54,747 tok/s peakVRAM 13.7GB
[14:55:24] step 340/4768 loss 4.7531 lr 4.08e-04 gnorm 0.75 54,744 tok/s peakVRAM 13.7GB
[14:58:35] step 360/4768 loss 4.6273 lr 4.32e-04 gnorm 0.59 54,745 tok/s peakVRAM 13.7GB
[15:01:47] step 380/4768 loss 4.5078 lr 4.56e-04 gnorm 0.56 54,744 tok/s peakVRAM 13.7GB
[15:04:58] step 400/4768 loss 4.4062 lr 4.80e-04 gnorm 0.45 54,745 tok/s peakVRAM 13.7GB
[15:08:10] step 420/4768 loss 4.3287 lr 5.04e-04 gnorm 0.46 54,747 tok/s peakVRAM 13.7GB
[15:11:21] step 440/4768 loss 4.2505 lr 5.28e-04 gnorm 0.40 54,744 tok/s peakVRAM 13.7GB
[15:14:33] step 460/4768 loss 4.1806 lr 5.52e-04 gnorm 0.39 54,744 tok/s peakVRAM 13.7GB
[15:17:44] step 480/4768 loss 4.1334 lr 5.76e-04 gnorm 0.35 54,742 tok/s peakVRAM 13.7GB
[15:20:56] step 500/4768 loss 4.0774 lr 6.00e-04 gnorm 0.31 54,736 tok/s peakVRAM 13.7GB
[15:24:07] step 520/4768 loss 4.0397 lr 6.00e-04 gnorm 0.27 54,735 tok/s peakVRAM 13.7GB
[15:27:19] step 540/4768 loss 3.9894 lr 6.00e-04 gnorm 0.22 54,736 tok/s peakVRAM 13.7GB
[15:30:31] step 560/4768 loss 3.9379 lr 6.00e-04 gnorm 0.23 54,734 tok/s peakVRAM 13.7GB
[15:33:42] step 580/4768 loss 3.8968 lr 6.00e-04 gnorm 0.23 54,732 tok/s peakVRAM 13.7GB
[15:36:54] step 600/4768 loss 3.8534 lr 6.00e-04 gnorm 0.21 54,728 tok/s peakVRAM 13.7GB
[15:40:05] step 620/4768 loss 3.8302 lr 6.00e-04 gnorm 0.21 54,730 tok/s peakVRAM 13.7GB
[15:43:17] step 640/4768 loss 3.8048 lr 6.00e-04 gnorm 0.19 54,735 tok/s peakVRAM 13.7GB
[15:46:29] step 660/4768 loss 3.7741 lr 6.00e-04 gnorm 0.17 54,729 tok/s peakVRAM 13.7GB
[15:49:40] step 680/4768 loss 3.7513 lr 6.00e-04 gnorm 0.18 54,735 tok/s peakVRAM 13.7GB
[15:52:52] step 700/4768 loss 3.7199 lr 6.00e-04 gnorm 0.18 54,736 tok/s peakVRAM 13.7GB
[15:56:03] step 720/4768 loss 3.6895 lr 6.00e-04 gnorm 0.17 54,733 tok/s peakVRAM 13.7GB
[15:59:15] step 740/4768 loss 3.6858 lr 6.00e-04 gnorm 0.19 54,732 tok/s peakVRAM 13.7GB
[16:02:26] step 760/4768 loss 3.6607 lr 6.00e-04 gnorm 0.16 54,731 tok/s peakVRAM 13.7GB
[16:05:38] step 780/4768 loss 3.6347 lr 6.00e-04 gnorm 0.16 54,731 tok/s peakVRAM 13.7GB
[16:08:50] step 800/4768 loss 3.6206 lr 6.00e-04 gnorm 0.14 54,732 tok/s peakVRAM 13.7GB
[16:12:01] step 820/4768 loss 3.6040 lr 6.00e-04 gnorm 0.14 54,730 tok/s peakVRAM 13.7GB
[16:15:13] step 840/4768 loss 3.5898 lr 6.00e-04 gnorm 0.15 54,728 tok/s peakVRAM 13.7GB
[16:18:24] step 860/4768 loss 3.5694 lr 6.00e-04 gnorm 0.16 54,730 tok/s peakVRAM 13.7GB
[16:21:36] step 880/4768 loss 3.5635 lr 6.00e-04 gnorm 0.14 54,727 tok/s peakVRAM 13.7GB
[16:24:48] step 900/4768 loss 3.5456 lr 6.00e-04 gnorm 0.14 54,728 tok/s peakVRAM 13.7GB
[16:27:59] step 920/4768 loss 3.5379 lr 6.00e-04 gnorm 0.14 54,732 tok/s peakVRAM 13.7GB
[16:31:11] step 940/4768 loss 3.5332 lr 6.00e-04 gnorm 0.14 54,731 tok/s peakVRAM 13.7GB
[16:34:22] step 960/4768 loss 3.5076 lr 6.00e-04 gnorm 0.13 54,727 tok/s peakVRAM 13.7GB
[16:37:34] step 980/4768 loss 3.5049 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB
[16:40:46] step 1000/4768 loss 3.4945 lr 6.00e-04 gnorm 0.14 54,723 tok/s peakVRAM 13.7GB
[16:43:57] step 1020/4768 loss 3.4891 lr 6.00e-04 gnorm 0.13 54,721 tok/s peakVRAM 13.7GB
[16:47:09] step 1040/4768 loss 3.4625 lr 6.00e-04 gnorm 0.13 54,718 tok/s peakVRAM 13.7GB
[16:50:20] step 1060/4768 loss 3.4738 lr 6.00e-04 gnorm 0.14 54,717 tok/s peakVRAM 13.7GB
[16:53:32] step 1080/4768 loss 3.4629 lr 6.00e-04 gnorm 0.12 54,721 tok/s peakVRAM 13.7GB
[16:56:44] step 1100/4768 loss 3.4473 lr 6.00e-04 gnorm 0.14 54,724 tok/s peakVRAM 13.7GB
[16:59:55] step 1120/4768 loss 3.4297 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB
[17:03:07] step 1140/4768 loss 3.4228 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB
[17:06:19] step 1160/4768 loss 3.4173 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB
[17:09:30] step 1180/4768 loss 3.4195 lr 6.00e-04 gnorm 0.12 54,725 tok/s peakVRAM 13.7GB
[17:12:42] step 1200/4768 loss 3.4018 lr 6.00e-04 gnorm 0.13 54,724 tok/s peakVRAM 13.7GB
[17:15:53] step 1220/4768 loss 3.3993 lr 6.00e-04 gnorm 0.13 54,726 tok/s peakVRAM 13.7GB
[17:19:05] step 1240/4768 loss 3.3916 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB
[17:22:17] step 1260/4768 loss 3.3782 lr 6.00e-04 gnorm 0.12 54,729 tok/s peakVRAM 13.7GB
[17:25:28] step 1280/4768 loss 3.3807 lr 6.00e-04 gnorm 0.12 54,722 tok/s peakVRAM 13.7GB
[17:28:40] step 1300/4768 loss 3.3735 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB
[17:31:51] step 1320/4768 loss 3.3659 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB
[17:35:03] step 1340/4768 loss 3.3664 lr 6.00e-04 gnorm 0.13 54,719 tok/s peakVRAM 13.7GB
[17:38:15] step 1360/4768 loss 3.3575 lr 6.00e-04 gnorm 0.13 54,716 tok/s peakVRAM 13.7GB
[17:41:26] step 1380/4768 loss 3.3491 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB
[17:44:38] step 1400/4768 loss 3.3387 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB
[17:47:50] step 1420/4768 loss 3.3266 lr 6.00e-04 gnorm 0.12 54,725 tok/s peakVRAM 13.7GB
[17:51:01] step 1440/4768 loss 3.3337 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB
[17:54:13] step 1460/4768 loss 3.3312 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB
[17:57:24] step 1480/4768 loss 3.3352 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB
[18:00:36] step 1500/4768 loss 3.3080 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB
[18:03:48] step 1520/4768 loss 3.3154 lr 6.00e-04 gnorm 0.12 54,719 tok/s peakVRAM 13.7GB
[18:06:59] step 1540/4768 loss 3.2992 lr 6.00e-04 gnorm 0.11 54,721 tok/s peakVRAM 13.7GB
[18:10:11] step 1560/4768 loss 3.3079 lr 6.00e-04 gnorm 0.13 54,718 tok/s peakVRAM 13.7GB
[18:13:23] step 1580/4768 loss 3.2967 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB
[18:16:34] step 1600/4768 loss 3.2853 lr 6.00e-04 gnorm 0.11 54,716 tok/s peakVRAM 13.7GB
[18:19:46] step 1620/4768 loss 3.2858 lr 6.00e-04 gnorm 0.12 54,719 tok/s peakVRAM 13.7GB
[18:22:57] step 1640/4768 loss 3.2827 lr 6.00e-04 gnorm 0.12 54,718 tok/s peakVRAM 13.7GB
[18:26:09] step 1660/4768 loss 3.2839 lr 6.00e-04 gnorm 0.11 54,723 tok/s peakVRAM 13.7GB
[18:29:21] step 1680/4768 loss 3.2733 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB
[18:32:32] step 1700/4768 loss 3.2658 lr 6.00e-04 gnorm 0.12 54,718 tok/s peakVRAM 13.7GB
[18:35:44] step 1720/4768 loss 3.2679 lr 6.00e-04 gnorm 0.12 54,712 tok/s peakVRAM 13.7GB
[18:38:56] step 1740/4768 loss 3.2618 lr 6.00e-04 gnorm 0.10 54,710 tok/s peakVRAM 13.7GB
[18:42:07] step 1760/4768 loss 3.2606 lr 6.00e-04 gnorm 0.11 54,712 tok/s peakVRAM 13.7GB
[18:45:19] step 1780/4768 loss 3.2480 lr 6.00e-04 gnorm 0.11 54,715 tok/s peakVRAM 13.7GB
[18:48:31] step 1800/4768 loss 3.2525 lr 6.00e-04 gnorm 0.12 54,716 tok/s peakVRAM 13.7GB
[18:51:42] step 1820/4768 loss 3.2466 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB
[18:54:54] step 1840/4768 loss 3.2411 lr 6.00e-04 gnorm 0.10 54,712 tok/s peakVRAM 13.7GB
[18:58:06] step 1860/4768 loss 3.2404 lr 6.00e-04 gnorm 0.12 54,710 tok/s peakVRAM 13.7GB
[19:01:17] step 1880/4768 loss 3.2358 lr 6.00e-04 gnorm 0.11 54,707 tok/s peakVRAM 13.7GB
[19:04:29] step 1900/4768 loss 3.2309 lr 6.00e-04 gnorm 0.10 54,706 tok/s peakVRAM 13.7GB
[19:07:41] step 1920/4768 loss 3.2304 lr 6.00e-04 gnorm 0.10 54,706 tok/s peakVRAM 13.7GB
[19:10:52] step 1940/4768 loss 3.2230 lr 6.00e-04 gnorm 0.12 54,710 tok/s peakVRAM 13.7GB
[19:14:04] step 1960/4768 loss 3.2270 lr 6.00e-04 gnorm 0.11 54,711 tok/s peakVRAM 13.7GB
[19:17:16] step 1980/4768 loss 3.2220 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB
[19:20:27] step 2000/4768 loss 3.2153 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB
[19:23:39] step 2020/4768 loss 3.2152 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB
[19:26:50] step 2040/4768 loss 3.2077 lr 6.00e-04 gnorm 0.11 54,721 tok/s peakVRAM 13.7GB
[19:30:02] step 2060/4768 loss 3.2078 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[19:33:14] step 2080/4768 loss 3.1967 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[19:36:25] step 2100/4768 loss 3.2051 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB
[19:39:37] step 2120/4768 loss 3.1951 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB
[19:42:49] step 2140/4768 loss 3.2076 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB
[19:46:00] step 2160/4768 loss 3.1865 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB
[19:49:12] step 2180/4768 loss 3.1881 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[19:52:24] step 2200/4768 loss 3.1892 lr 6.00e-04 gnorm 0.10 54,711 tok/s peakVRAM 13.7GB
[19:55:35] step 2220/4768 loss 3.1820 lr 6.00e-04 gnorm 0.10 54,709 tok/s peakVRAM 13.7GB
[19:58:47] step 2240/4768 loss 3.1787 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB
[20:01:58] step 2260/4768 loss 3.1754 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB
[20:05:10] step 2280/4768 loss 3.1711 lr 6.00e-04 gnorm 0.11 54,712 tok/s peakVRAM 13.7GB
[20:08:22] step 2300/4768 loss 3.1747 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB
[20:11:33] step 2320/4768 loss 3.1717 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB
[20:14:45] step 2340/4768 loss 3.1667 lr 6.00e-04 gnorm 0.11 54,715 tok/s peakVRAM 13.7GB
[20:17:57] step 2360/4768 loss 3.1573 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB
[20:21:08] step 2380/4768 loss 3.1559 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB
[20:24:20] step 2400/4768 loss 3.1566 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB
[20:27:32] step 2420/4768 loss 3.1597 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB
[20:30:43] step 2440/4768 loss 3.1548 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB
[20:33:55] step 2460/4768 loss 3.1532 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB
[20:37:06] step 2480/4768 loss 3.1504 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB
[20:40:18] step 2500/4768 loss 3.1447 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB
[20:43:30] step 2520/4768 loss 3.1466 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB
[20:46:41] step 2540/4768 loss 3.1465 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[20:49:53] step 2560/4768 loss 3.1434 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB
[20:53:05] step 2580/4768 loss 3.1385 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB
[20:56:16] step 2600/4768 loss 3.1294 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB
[20:59:28] step 2620/4768 loss 3.1375 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB
[21:02:40] step 2640/4768 loss 3.1389 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB
[21:05:51] step 2660/4768 loss 3.1372 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB
[21:09:03] step 2680/4768 loss 3.1326 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB
[21:12:14] step 2700/4768 loss 3.1352 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB
[21:15:26] step 2720/4768 loss 3.1180 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[21:18:38] step 2740/4768 loss 3.1193 lr 6.00e-04 gnorm 0.10 54,723 tok/s peakVRAM 13.7GB
[21:21:49] step 2760/4768 loss 3.1220 lr 6.00e-04 gnorm 0.10 54,724 tok/s peakVRAM 13.7GB
[21:25:01] step 2780/4768 loss 3.1132 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB
[21:28:13] step 2800/4768 loss 3.1081 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB
[21:31:24] step 2820/4768 loss 3.1058 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB
[21:34:36] step 2840/4768 loss 3.1134 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB
[21:37:47] step 2860/4768 loss 3.1055 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[21:40:59] step 2880/4768 loss 3.1071 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB
[21:44:11] step 2900/4768 loss 3.1088 lr 6.00e-04 gnorm 0.11 54,713 tok/s peakVRAM 13.7GB
[21:47:22] step 2920/4768 loss 3.1159 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB
[21:50:34] step 2940/4768 loss 3.1065 lr 6.00e-04 gnorm 0.08 54,720 tok/s peakVRAM 13.7GB
[21:53:46] step 2960/4768 loss 3.1085 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB
[21:56:57] step 2980/4768 loss 3.0954 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB
[22:00:09] step 3000/4768 loss 3.0957 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB
[22:03:21] step 3020/4768 loss 3.0920 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB
[22:06:32] step 3040/4768 loss 3.0889 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB
[22:09:44] step 3060/4768 loss 3.0923 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB
[22:12:55] step 3080/4768 loss 3.0847 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB
[22:16:07] step 3100/4768 loss 3.0786 lr 6.00e-04 gnorm 0.10 54,725 tok/s peakVRAM 13.7GB
[22:19:19] step 3120/4768 loss 3.0947 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB
[22:22:30] step 3140/4768 loss 3.0798 lr 6.00e-04 gnorm 0.10 54,724 tok/s peakVRAM 13.7GB
[22:25:42] step 3160/4768 loss 3.0761 lr 6.00e-04 gnorm 0.09 54,722 tok/s peakVRAM 13.7GB
[22:28:54] step 3180/4768 loss 3.0823 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB
[22:32:05] step 3200/4768 loss 3.0832 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB
[22:35:17] step 3220/4768 loss 3.0926 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB
[22:38:28] step 3240/4768 loss 3.0746 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB
[22:41:40] step 3260/4768 loss 3.0712 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[22:44:52] step 3280/4768 loss 3.0670 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB
[22:48:03] step 3300/4768 loss 3.0780 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB
[22:51:15] step 3320/4768 loss 3.0718 lr 6.00e-04 gnorm 0.08 54,713 tok/s peakVRAM 13.7GB
[22:54:27] step 3340/4768 loss 3.0722 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB
[22:57:38] step 3360/4768 loss 3.0630 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB
[23:00:50] step 3380/4768 loss 3.0728 lr 6.00e-04 gnorm 0.10 54,715 tok/s peakVRAM 13.7GB
[23:04:02] step 3400/4768 loss 3.0580 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB
[23:07:13] step 3420/4768 loss 3.0660 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[23:10:25] step 3440/4768 loss 3.0576 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB
[23:13:36] step 3460/4768 loss 3.0667 lr 6.00e-04 gnorm 0.09 54,715 tok/s peakVRAM 13.7GB
[23:16:48] step 3480/4768 loss 3.0667 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB
[23:20:00] step 3500/4768 loss 3.0526 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB
[23:23:11] step 3520/4768 loss 3.0552 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB
[23:26:23] step 3540/4768 loss 3.0578 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB
[23:29:35] step 3560/4768 loss 3.0616 lr 6.00e-04 gnorm 0.09 54,717 tok/s peakVRAM 13.7GB
[23:32:46] step 3580/4768 loss 3.0558 lr 6.00e-04 gnorm 0.08 54,722 tok/s peakVRAM 13.7GB
[23:35:58] step 3600/4768 loss 3.0576 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB
[23:39:10] step 3620/4768 loss 3.0497 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB
[23:42:21] step 3640/4768 loss 3.0574 lr 6.00e-04 gnorm 0.09 54,722 tok/s peakVRAM 13.7GB
[23:45:33] step 3660/4768 loss 3.0462 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB
[23:48:44] step 3680/4768 loss 3.0531 lr 6.00e-04 gnorm 0.09 54,724 tok/s peakVRAM 13.7GB
[23:51:56] step 3700/4768 loss 3.0503 lr 6.00e-04 gnorm 0.09 54,726 tok/s peakVRAM 13.7GB
[23:55:08] step 3720/4768 loss 3.0435 lr 6.00e-04 gnorm 0.08 54,724 tok/s peakVRAM 13.7GB
[23:58:19] step 3740/4768 loss 3.0400 lr 6.00e-04 gnorm 0.08 54,727 tok/s peakVRAM 13.7GB
[00:01:31] step 3760/4768 loss 3.0384 lr 6.00e-04 gnorm 0.09 54,725 tok/s peakVRAM 13.7GB
[00:04:42] step 3780/4768 loss 3.0390 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB
[00:07:54] step 3800/4768 loss 3.0400 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB
[00:11:06] step 3820/4768 loss 3.0331 lr 5.61e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB
[00:14:17] step 3840/4768 loss 3.0316 lr 5.13e-04 gnorm 0.08 54,717 tok/s peakVRAM 13.7GB
[00:17:29] step 3860/4768 loss 3.0245 lr 4.83e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB
[00:20:41] step 3880/4768 loss 3.0217 lr 4.59e-04 gnorm 0.08 54,721 tok/s peakVRAM 13.7GB
[00:23:52] step 3900/4768 loss 3.0150 lr 4.39e-04 gnorm 0.08 54,716 tok/s peakVRAM 13.7GB
[00:27:04] step 3920/4768 loss 3.0093 lr 4.21e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB
[00:30:16] step 3940/4768 loss 3.0019 lr 4.05e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB
[00:33:27] step 3960/4768 loss 3.0024 lr 3.89e-04 gnorm 0.08 54,716 tok/s peakVRAM 13.7GB
[00:36:39] step 3980/4768 loss 2.9945 lr 3.75e-04 gnorm 0.07 54,721 tok/s peakVRAM 13.7GB
[00:39:50] step 4000/4768 loss 2.9933 lr 3.62e-04 gnorm 0.07 54,722 tok/s peakVRAM 13.7GB
[00:43:02] step 4020/4768 loss 2.9906 lr 3.50e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB
[00:46:14] step 4040/4768 loss 2.9812 lr 3.38e-04 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB
[00:49:25] step 4060/4768 loss 2.9809 lr 3.26e-04 gnorm 0.07 54,719 tok/s peakVRAM 13.7GB
[00:52:37] step 4080/4768 loss 2.9725 lr 3.15e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB
[00:55:49] step 4100/4768 loss 2.9776 lr 3.05e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB
[00:59:00] step 4120/4768 loss 2.9664 lr 2.95e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB
[01:02:12] step 4140/4768 loss 2.9586 lr 2.85e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB
[01:05:24] step 4160/4768 loss 2.9768 lr 2.75e-04 gnorm 0.07 54,710 tok/s peakVRAM 13.7GB
[01:08:35] step 4180/4768 loss 2.9694 lr 2.66e-04 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB
[01:11:47] step 4200/4768 loss 2.9631 lr 2.57e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB
[01:14:58] step 4220/4768 loss 2.9561 lr 2.48e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB
[01:18:10] step 4240/4768 loss 2.9624 lr 2.40e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB
[01:21:22] step 4260/4768 loss 2.9546 lr 2.31e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB
[01:24:33] step 4280/4768 loss 2.9540 lr 2.23e-04 gnorm 0.07 54,725 tok/s peakVRAM 13.7GB
[01:27:45] step 4300/4768 loss 2.9518 lr 2.15e-04 gnorm 0.07 54,721 tok/s peakVRAM 13.7GB
[01:30:57] step 4320/4768 loss 2.9470 lr 2.07e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB
[01:34:08] step 4340/4768 loss 2.9457 lr 1.99e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB
[01:37:20] step 4360/4768 loss 2.9396 lr 1.92e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB
[01:40:32] step 4380/4768 loss 2.9465 lr 1.84e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB
[01:43:43] step 4400/4768 loss 2.9439 lr 1.77e-04 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB
[01:46:55] step 4420/4768 loss 2.9371 lr 1.70e-04 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB
[01:50:06] step 4440/4768 loss 2.9435 lr 1.63e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB
[01:53:18] step 4460/4768 loss 2.9390 lr 1.56e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB
[01:56:30] step 4480/4768 loss 2.9341 lr 1.49e-04 gnorm 0.07 54,719 tok/s peakVRAM 13.7GB
[01:59:41] step 4500/4768 loss 2.9313 lr 1.42e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB
[02:02:53] step 4520/4768 loss 2.9332 lr 1.36e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB
[02:06:05] step 4540/4768 loss 2.9246 lr 1.29e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB
[02:09:16] step 4560/4768 loss 2.9280 lr 1.23e-04 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB
[02:12:28] step 4580/4768 loss 2.9309 lr 1.16e-04 gnorm 0.07 54,710 tok/s peakVRAM 13.7GB
[02:15:40] step 4600/4768 loss 2.9299 lr 1.10e-04 gnorm 0.07 54,708 tok/s peakVRAM 13.7GB
[02:18:51] step 4620/4768 loss 2.9163 lr 1.04e-04 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB
[02:22:03] step 4640/4768 loss 2.9256 lr 9.78e-05 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB
[02:25:15] step 4660/4768 loss 2.9184 lr 9.18e-05 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB
[02:28:26] step 4680/4768 loss 2.9159 lr 8.58e-05 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB
[02:31:38] step 4700/4768 loss 2.9121 lr 7.99e-05 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB
[02:34:50] step 4720/4768 loss 2.9131 lr 7.41e-05 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB
[02:38:01] step 4740/4768 loss 2.9110 lr 6.83e-05 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB
[02:41:13] step 4760/4768 loss 2.9202 lr 6.26e-05 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB
[02:42:31] ckpt @ 4768
[02:42:31] DONE