File size: 21,904 Bytes
c28a890 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 | [13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1
[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288
[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14)
[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x)
[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024
[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved
[14:00:00] torch.compile enabled
[14:00:00] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05
[14:04:22] step 20/4768 loss 10.3912 lr 2.40e-05 gnorm 2.73 39,938 tok/s peakVRAM 13.7GB
[14:07:35] step 40/4768 loss 9.5518 lr 4.80e-05 gnorm 1.36 54,379 tok/s peakVRAM 13.7GB
[14:10:48] step 60/4768 loss 8.6782 lr 7.20e-05 gnorm 0.73 54,389 tok/s peakVRAM 13.7GB
[14:14:00] step 80/4768 loss 8.0385 lr 9.60e-05 gnorm 1.30 54,384 tok/s peakVRAM 13.7GB
[14:17:13] step 100/4768 loss 7.5974 lr 1.20e-04 gnorm 0.84 54,364 tok/s peakVRAM 13.7GB
[14:20:26] step 120/4768 loss 7.1265 lr 1.44e-04 gnorm 0.78 54,354 tok/s peakVRAM 13.7GB
[14:23:39] step 140/4768 loss 6.7082 lr 1.68e-04 gnorm 0.88 54,350 tok/s peakVRAM 13.7GB
[14:26:52] step 160/4768 loss 6.3537 lr 1.92e-04 gnorm 0.78 54,345 tok/s peakVRAM 13.7GB
[14:30:05] step 180/4768 loss 6.0894 lr 2.16e-04 gnorm 0.86 54,335 tok/s peakVRAM 13.7GB
[14:33:18] step 200/4768 loss 5.8676 lr 2.40e-04 gnorm 0.76 54,330 tok/s peakVRAM 13.7GB
[14:36:31] step 220/4768 loss 5.6845 lr 2.64e-04 gnorm 0.84 54,330 tok/s peakVRAM 13.7GB
[14:39:44] step 240/4768 loss 5.5133 lr 2.88e-04 gnorm 0.76 54,330 tok/s peakVRAM 13.7GB
[14:42:57] step 260/4768 loss 5.3498 lr 3.12e-04 gnorm 0.70 54,329 tok/s peakVRAM 13.7GB
[14:46:10] step 280/4768 loss 5.1918 lr 3.36e-04 gnorm 0.66 54,330 tok/s peakVRAM 13.7GB
[14:49:23] step 300/4768 loss 5.0466 lr 3.60e-04 gnorm 0.66 54,328 tok/s peakVRAM 13.7GB
[14:52:36] step 320/4768 loss 4.9124 lr 3.84e-04 gnorm 0.72 54,329 tok/s peakVRAM 13.7GB
[14:55:49] step 340/4768 loss 4.7663 lr 4.08e-04 gnorm 0.64 54,326 tok/s peakVRAM 13.7GB
[14:59:02] step 360/4768 loss 4.6379 lr 4.32e-04 gnorm 0.60 54,329 tok/s peakVRAM 13.7GB
[15:02:15] step 380/4768 loss 4.5207 lr 4.56e-04 gnorm 0.50 54,328 tok/s peakVRAM 13.7GB
[15:05:28] step 400/4768 loss 4.4147 lr 4.80e-04 gnorm 0.44 54,326 tok/s peakVRAM 13.7GB
[15:08:41] step 420/4768 loss 4.3344 lr 5.04e-04 gnorm 0.48 54,325 tok/s peakVRAM 13.7GB
[15:11:54] step 440/4768 loss 4.2526 lr 5.28e-04 gnorm 0.36 54,324 tok/s peakVRAM 13.7GB
[15:15:07] step 460/4768 loss 4.1822 lr 5.52e-04 gnorm 0.47 54,322 tok/s peakVRAM 13.7GB
[15:18:20] step 480/4768 loss 4.1330 lr 5.76e-04 gnorm 0.38 54,320 tok/s peakVRAM 13.7GB
[15:21:33] step 500/4768 loss 4.0766 lr 6.00e-04 gnorm 0.30 54,320 tok/s peakVRAM 13.7GB
[15:24:46] step 520/4768 loss 4.0389 lr 6.00e-04 gnorm 0.35 54,316 tok/s peakVRAM 13.7GB
[15:27:59] step 540/4768 loss 3.9892 lr 6.00e-04 gnorm 0.26 54,316 tok/s peakVRAM 13.7GB
[15:31:13] step 560/4768 loss 3.9361 lr 6.00e-04 gnorm 0.24 54,314 tok/s peakVRAM 13.7GB
[15:34:26] step 580/4768 loss 3.8941 lr 6.00e-04 gnorm 0.24 54,311 tok/s peakVRAM 13.7GB
[15:37:39] step 600/4768 loss 3.8519 lr 6.00e-04 gnorm 0.22 54,312 tok/s peakVRAM 13.7GB
[15:40:52] step 620/4768 loss 3.8298 lr 6.00e-04 gnorm 0.22 54,314 tok/s peakVRAM 13.7GB
[15:44:05] step 640/4768 loss 3.8032 lr 6.00e-04 gnorm 0.19 54,308 tok/s peakVRAM 13.7GB
[15:47:18] step 660/4768 loss 3.7710 lr 6.00e-04 gnorm 0.18 54,309 tok/s peakVRAM 13.7GB
[15:50:31] step 680/4768 loss 3.7483 lr 6.00e-04 gnorm 0.18 54,309 tok/s peakVRAM 13.7GB
[15:53:44] step 700/4768 loss 3.7176 lr 6.00e-04 gnorm 0.17 54,304 tok/s peakVRAM 13.7GB
[15:56:57] step 720/4768 loss 3.6874 lr 6.00e-04 gnorm 0.18 54,296 tok/s peakVRAM 13.7GB
[16:00:10] step 740/4768 loss 3.6829 lr 6.00e-04 gnorm 0.18 54,301 tok/s peakVRAM 13.7GB
[16:03:23] step 760/4768 loss 3.6580 lr 6.00e-04 gnorm 0.15 54,298 tok/s peakVRAM 13.7GB
[16:06:36] step 780/4768 loss 3.6315 lr 6.00e-04 gnorm 0.16 54,301 tok/s peakVRAM 13.7GB
[16:09:50] step 800/4768 loss 3.6186 lr 6.00e-04 gnorm 0.17 54,303 tok/s peakVRAM 13.7GB
[16:13:03] step 820/4768 loss 3.6022 lr 6.00e-04 gnorm 0.15 54,305 tok/s peakVRAM 13.7GB
[16:16:16] step 840/4768 loss 3.5874 lr 6.00e-04 gnorm 0.17 54,300 tok/s peakVRAM 13.7GB
[16:19:29] step 860/4768 loss 3.5672 lr 6.00e-04 gnorm 0.16 54,296 tok/s peakVRAM 13.7GB
[16:22:42] step 880/4768 loss 3.5612 lr 6.00e-04 gnorm 0.13 54,301 tok/s peakVRAM 13.7GB
[16:25:55] step 900/4768 loss 3.5428 lr 6.00e-04 gnorm 0.14 54,306 tok/s peakVRAM 13.7GB
[16:29:08] step 920/4768 loss 3.5357 lr 6.00e-04 gnorm 0.14 54,304 tok/s peakVRAM 13.7GB
[16:32:21] step 940/4768 loss 3.5316 lr 6.00e-04 gnorm 0.13 54,308 tok/s peakVRAM 13.7GB
[16:35:34] step 960/4768 loss 3.5057 lr 6.00e-04 gnorm 0.13 54,309 tok/s peakVRAM 13.7GB
[16:38:47] step 980/4768 loss 3.5031 lr 6.00e-04 gnorm 0.14 54,307 tok/s peakVRAM 13.7GB
[16:42:01] step 1000/4768 loss 3.4927 lr 6.00e-04 gnorm 0.13 54,302 tok/s peakVRAM 13.7GB
[16:45:14] step 1020/4768 loss 3.4872 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB
[16:48:27] step 1040/4768 loss 3.4604 lr 6.00e-04 gnorm 0.13 54,306 tok/s peakVRAM 13.7GB
[16:51:40] step 1060/4768 loss 3.4721 lr 6.00e-04 gnorm 0.13 54,302 tok/s peakVRAM 13.7GB
[16:54:53] step 1080/4768 loss 3.4612 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB
[16:58:06] step 1100/4768 loss 3.4462 lr 6.00e-04 gnorm 0.14 54,300 tok/s peakVRAM 13.7GB
[17:01:19] step 1120/4768 loss 3.4286 lr 6.00e-04 gnorm 0.13 54,299 tok/s peakVRAM 13.7GB
[17:04:32] step 1140/4768 loss 3.4225 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB
[17:07:45] step 1160/4768 loss 3.4164 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB
[17:10:58] step 1180/4768 loss 3.4192 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB
[17:14:12] step 1200/4768 loss 3.4006 lr 6.00e-04 gnorm 0.12 54,301 tok/s peakVRAM 13.7GB
[17:17:25] step 1220/4768 loss 3.3984 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB
[17:20:38] step 1240/4768 loss 3.3901 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB
[17:23:51] step 1260/4768 loss 3.3763 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB
[17:27:04] step 1280/4768 loss 3.3792 lr 6.00e-04 gnorm 0.12 54,301 tok/s peakVRAM 13.7GB
[17:30:17] step 1300/4768 loss 3.3721 lr 6.00e-04 gnorm 0.13 54,297 tok/s peakVRAM 13.7GB
[17:33:30] step 1320/4768 loss 3.3646 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB
[17:36:43] step 1340/4768 loss 3.3652 lr 6.00e-04 gnorm 0.13 54,297 tok/s peakVRAM 13.7GB
[17:39:57] step 1360/4768 loss 3.3561 lr 6.00e-04 gnorm 0.12 54,297 tok/s peakVRAM 13.7GB
[17:43:10] step 1380/4768 loss 3.3482 lr 6.00e-04 gnorm 0.13 54,301 tok/s peakVRAM 13.7GB
[17:46:23] step 1400/4768 loss 3.3375 lr 6.00e-04 gnorm 0.12 54,298 tok/s peakVRAM 13.7GB
[17:49:36] step 1420/4768 loss 3.3256 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB
[17:52:49] step 1440/4768 loss 3.3326 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB
[17:56:02] step 1460/4768 loss 3.3301 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB
[17:59:15] step 1480/4768 loss 3.3341 lr 6.00e-04 gnorm 0.11 54,299 tok/s peakVRAM 13.7GB
[18:02:28] step 1500/4768 loss 3.3077 lr 6.00e-04 gnorm 0.12 54,299 tok/s peakVRAM 13.7GB
[18:05:41] step 1520/4768 loss 3.3145 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB
[18:08:55] step 1540/4768 loss 3.2987 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB
[18:12:08] step 1560/4768 loss 3.3075 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB
[18:15:21] step 1580/4768 loss 3.2953 lr 6.00e-04 gnorm 0.12 54,292 tok/s peakVRAM 13.7GB
[18:18:34] step 1600/4768 loss 3.2850 lr 6.00e-04 gnorm 0.11 54,290 tok/s peakVRAM 13.7GB
[18:21:47] step 1620/4768 loss 3.2859 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB
[18:25:00] step 1640/4768 loss 3.2823 lr 6.00e-04 gnorm 0.11 54,294 tok/s peakVRAM 13.7GB
[18:28:13] step 1660/4768 loss 3.2832 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB
[18:31:26] step 1680/4768 loss 3.2724 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB
[18:34:40] step 1700/4768 loss 3.2652 lr 6.00e-04 gnorm 0.12 54,297 tok/s peakVRAM 13.7GB
[18:37:53] step 1720/4768 loss 3.2673 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB
[18:41:06] step 1740/4768 loss 3.2613 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB
[18:44:19] step 1760/4768 loss 3.2605 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB
[18:47:32] step 1780/4768 loss 3.2474 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB
[18:50:45] step 1800/4768 loss 3.2512 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB
[18:53:58] step 1820/4768 loss 3.2471 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB
[18:57:11] step 1840/4768 loss 3.2405 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB
[19:00:25] step 1860/4768 loss 3.2397 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB
[19:03:38] step 1880/4768 loss 3.2355 lr 6.00e-04 gnorm 0.10 54,290 tok/s peakVRAM 13.7GB
[19:06:51] step 1900/4768 loss 3.2295 lr 6.00e-04 gnorm 0.11 54,292 tok/s peakVRAM 13.7GB
[19:10:04] step 1920/4768 loss 3.2302 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB
[19:13:17] step 1940/4768 loss 3.2224 lr 6.00e-04 gnorm 0.10 54,290 tok/s peakVRAM 13.7GB
[19:16:30] step 1960/4768 loss 3.2266 lr 6.00e-04 gnorm 0.11 54,292 tok/s peakVRAM 13.7GB
[19:19:43] step 1980/4768 loss 3.2208 lr 6.00e-04 gnorm 0.09 54,293 tok/s peakVRAM 13.7GB
[19:22:56] step 2000/4768 loss 3.2143 lr 6.00e-04 gnorm 0.12 54,302 tok/s peakVRAM 13.7GB
[19:26:10] step 2020/4768 loss 3.2143 lr 6.00e-04 gnorm 0.11 54,303 tok/s peakVRAM 13.7GB
[19:29:23] step 2040/4768 loss 3.2074 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB
[19:32:36] step 2060/4768 loss 3.2075 lr 6.00e-04 gnorm 0.10 54,298 tok/s peakVRAM 13.7GB
[19:35:49] step 2080/4768 loss 3.1964 lr 6.00e-04 gnorm 0.09 54,300 tok/s peakVRAM 13.7GB
[19:39:02] step 2100/4768 loss 3.2047 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB
[19:42:15] step 2120/4768 loss 3.1944 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB
[19:45:28] step 2140/4768 loss 3.2072 lr 6.00e-04 gnorm 0.11 54,298 tok/s peakVRAM 13.7GB
[19:48:41] step 2160/4768 loss 3.1865 lr 6.00e-04 gnorm 0.10 54,300 tok/s peakVRAM 13.7GB
[19:51:54] step 2180/4768 loss 3.1878 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB
[19:55:08] step 2200/4768 loss 3.1888 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB
[19:58:21] step 2220/4768 loss 3.1810 lr 6.00e-04 gnorm 0.10 54,295 tok/s peakVRAM 13.7GB
[20:01:34] step 2240/4768 loss 3.1782 lr 6.00e-04 gnorm 0.11 54,293 tok/s peakVRAM 13.7GB
[20:04:47] step 2260/4768 loss 3.1746 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB
[20:08:00] step 2280/4768 loss 3.1712 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB
[20:11:13] step 2300/4768 loss 3.1743 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB
[20:14:26] step 2320/4768 loss 3.1707 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB
[20:17:39] step 2340/4768 loss 3.1656 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB
[20:20:53] step 2360/4768 loss 3.1564 lr 6.00e-04 gnorm 0.10 54,294 tok/s peakVRAM 13.7GB
[20:24:06] step 2380/4768 loss 3.1555 lr 6.00e-04 gnorm 0.10 54,296 tok/s peakVRAM 13.7GB
[20:27:19] step 2400/4768 loss 3.1551 lr 6.00e-04 gnorm 0.10 54,296 tok/s peakVRAM 13.7GB
[20:30:32] step 2420/4768 loss 3.1583 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB
[20:33:45] step 2440/4768 loss 3.1520 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB
[20:36:58] step 2460/4768 loss 3.1510 lr 6.00e-04 gnorm 0.11 54,303 tok/s peakVRAM 13.7GB
[20:40:11] step 2480/4768 loss 3.1482 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB
[20:43:24] step 2500/4768 loss 3.1426 lr 6.00e-04 gnorm 0.10 54,300 tok/s peakVRAM 13.7GB
[20:46:37] step 2520/4768 loss 3.1444 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB
[20:49:51] step 2540/4768 loss 3.1448 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB
[20:53:04] step 2560/4768 loss 3.1416 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB
[20:56:17] step 2580/4768 loss 3.1369 lr 6.00e-04 gnorm 0.09 54,297 tok/s peakVRAM 13.7GB
[20:59:30] step 2600/4768 loss 3.1276 lr 6.00e-04 gnorm 0.10 54,298 tok/s peakVRAM 13.7GB
[21:02:43] step 2620/4768 loss 3.1362 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB
[21:05:56] step 2640/4768 loss 3.1374 lr 6.00e-04 gnorm 0.10 54,303 tok/s peakVRAM 13.7GB
[21:09:09] step 2660/4768 loss 3.1362 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB
[21:12:22] step 2680/4768 loss 3.1310 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB
[21:15:35] step 2700/4768 loss 3.1338 lr 6.00e-04 gnorm 0.09 54,299 tok/s peakVRAM 13.7GB
[21:18:48] step 2720/4768 loss 3.1165 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB
[21:22:02] step 2740/4768 loss 3.1183 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB
[21:25:15] step 2760/4768 loss 3.1214 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB
[21:28:28] step 2780/4768 loss 3.1122 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB
[21:31:41] step 2800/4768 loss 3.1071 lr 6.00e-04 gnorm 0.11 54,294 tok/s peakVRAM 13.7GB
[21:34:54] step 2820/4768 loss 3.1044 lr 6.00e-04 gnorm 0.09 54,294 tok/s peakVRAM 13.7GB
[21:38:07] step 2840/4768 loss 3.1124 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB
[21:41:20] step 2860/4768 loss 3.1040 lr 6.00e-04 gnorm 0.09 54,296 tok/s peakVRAM 13.7GB
[21:44:33] step 2880/4768 loss 3.1058 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB
[21:47:47] step 2900/4768 loss 3.1077 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB
[21:51:00] step 2920/4768 loss 3.1145 lr 6.00e-04 gnorm 0.09 54,298 tok/s peakVRAM 13.7GB
[21:54:13] step 2940/4768 loss 3.1047 lr 6.00e-04 gnorm 0.08 54,296 tok/s peakVRAM 13.7GB
[21:57:26] step 2960/4768 loss 3.1066 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB
[22:00:39] step 2980/4768 loss 3.0944 lr 6.00e-04 gnorm 0.11 54,300 tok/s peakVRAM 13.7GB
[22:03:52] step 3000/4768 loss 3.0945 lr 6.00e-04 gnorm 0.09 54,295 tok/s peakVRAM 13.7GB
[22:07:05] step 3020/4768 loss 3.0901 lr 6.00e-04 gnorm 0.09 54,296 tok/s peakVRAM 13.7GB
[22:10:18] step 3040/4768 loss 3.0874 lr 6.00e-04 gnorm 0.10 54,295 tok/s peakVRAM 13.7GB
[22:13:31] step 3060/4768 loss 3.0911 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB
[22:16:45] step 3080/4768 loss 3.0834 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB
[22:19:58] step 3100/4768 loss 3.0775 lr 6.00e-04 gnorm 0.10 54,303 tok/s peakVRAM 13.7GB
[22:23:11] step 3120/4768 loss 3.0931 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB
[22:26:24] step 3140/4768 loss 3.0785 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB
[22:29:37] step 3160/4768 loss 3.0747 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB
[22:32:50] step 3180/4768 loss 3.0803 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB
[22:36:03] step 3200/4768 loss 3.0820 lr 6.00e-04 gnorm 0.09 54,306 tok/s peakVRAM 13.7GB
[22:39:16] step 3220/4768 loss 3.0916 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB
[22:42:29] step 3240/4768 loss 3.0735 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB
[22:45:42] step 3260/4768 loss 3.0701 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB
[22:48:56] step 3280/4768 loss 3.0658 lr 6.00e-04 gnorm 0.09 54,301 tok/s peakVRAM 13.7GB
[22:52:09] step 3300/4768 loss 3.0763 lr 6.00e-04 gnorm 0.08 54,302 tok/s peakVRAM 13.7GB
[22:55:22] step 3320/4768 loss 3.0704 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB
[22:58:35] step 3340/4768 loss 3.0707 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB
[23:01:48] step 3360/4768 loss 3.0617 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB
[23:05:01] step 3380/4768 loss 3.0715 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB
[23:08:14] step 3400/4768 loss 3.0564 lr 6.00e-04 gnorm 0.09 54,306 tok/s peakVRAM 13.7GB
[23:11:27] step 3420/4768 loss 3.0647 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB
[23:14:40] step 3440/4768 loss 3.0560 lr 6.00e-04 gnorm 0.10 54,306 tok/s peakVRAM 13.7GB
[23:17:53] step 3460/4768 loss 3.0657 lr 6.00e-04 gnorm 0.09 54,307 tok/s peakVRAM 13.7GB
[23:21:06] step 3480/4768 loss 3.0652 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB
[23:24:20] step 3500/4768 loss 3.0513 lr 6.00e-04 gnorm 0.10 54,308 tok/s peakVRAM 13.7GB
[23:27:33] step 3520/4768 loss 3.0545 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB
[23:30:46] step 3540/4768 loss 3.0561 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB
[23:33:59] step 3560/4768 loss 3.0606 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB
[23:37:12] step 3580/4768 loss 3.0544 lr 6.00e-04 gnorm 0.08 54,307 tok/s peakVRAM 13.7GB
[23:40:25] step 3600/4768 loss 3.0568 lr 6.00e-04 gnorm 0.10 54,309 tok/s peakVRAM 13.7GB
[23:43:38] step 3620/4768 loss 3.0479 lr 6.00e-04 gnorm 0.10 54,307 tok/s peakVRAM 13.7GB
[23:46:51] step 3640/4768 loss 3.0562 lr 6.00e-04 gnorm 0.09 54,308 tok/s peakVRAM 13.7GB
[23:50:04] step 3660/4768 loss 3.0452 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB
[23:53:17] step 3680/4768 loss 3.0520 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB
[23:56:30] step 3700/4768 loss 3.0491 lr 6.00e-04 gnorm 0.10 54,306 tok/s peakVRAM 13.7GB
[23:59:44] step 3720/4768 loss 3.0422 lr 6.00e-04 gnorm 0.10 54,305 tok/s peakVRAM 13.7GB
[00:02:57] step 3740/4768 loss 3.0386 lr 6.00e-04 gnorm 0.09 54,297 tok/s peakVRAM 13.7GB
[00:06:10] step 3760/4768 loss 3.0374 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB
[00:09:23] step 3780/4768 loss 3.0376 lr 6.00e-04 gnorm 0.09 54,300 tok/s peakVRAM 13.7GB
[00:12:36] step 3800/4768 loss 3.0388 lr 6.00e-04 gnorm 0.09 54,301 tok/s peakVRAM 13.7GB
[00:15:49] step 3820/4768 loss 3.0327 lr 5.61e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB
[00:19:02] step 3840/4768 loss 3.0306 lr 5.13e-04 gnorm 0.08 54,308 tok/s peakVRAM 13.7GB
[00:22:15] step 3860/4768 loss 3.0235 lr 4.83e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB
[00:25:28] step 3880/4768 loss 3.0207 lr 4.59e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB
[00:28:41] step 3900/4768 loss 3.0140 lr 4.39e-04 gnorm 0.08 54,308 tok/s peakVRAM 13.7GB
[00:31:54] step 3920/4768 loss 3.0089 lr 4.21e-04 gnorm 0.08 54,310 tok/s peakVRAM 13.7GB
[00:35:08] step 3940/4768 loss 3.0008 lr 4.05e-04 gnorm 0.07 54,312 tok/s peakVRAM 13.7GB
[00:38:21] step 3960/4768 loss 3.0012 lr 3.89e-04 gnorm 0.08 54,309 tok/s peakVRAM 13.7GB
[00:41:34] step 3980/4768 loss 2.9935 lr 3.75e-04 gnorm 0.07 54,310 tok/s peakVRAM 13.7GB
[00:44:47] step 4000/4768 loss 2.9921 lr 3.62e-04 gnorm 0.07 54,306 tok/s peakVRAM 13.7GB
[00:48:00] step 4020/4768 loss 2.9892 lr 3.50e-04 gnorm 0.07 54,309 tok/s peakVRAM 13.7GB
[00:51:13] step 4040/4768 loss 2.9804 lr 3.38e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB
[00:54:26] step 4060/4768 loss 2.9800 lr 3.26e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB
[00:57:39] step 4080/4768 loss 2.9711 lr 3.15e-04 gnorm 0.07 54,306 tok/s peakVRAM 13.7GB
[01:00:52] step 4100/4768 loss 2.9771 lr 3.05e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB
[01:04:05] step 4120/4768 loss 2.9652 lr 2.95e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB
[01:07:18] step 4140/4768 loss 2.9576 lr 2.85e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB
[01:10:32] step 4160/4768 loss 2.9754 lr 2.75e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB
[01:13:45] step 4180/4768 loss 2.9682 lr 2.66e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB
[01:16:58] step 4200/4768 loss 2.9618 lr 2.57e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB
[01:20:11] step 4220/4768 loss 2.9553 lr 2.48e-04 gnorm 0.07 54,302 tok/s peakVRAM 13.7GB
[01:23:24] step 4240/4768 loss 2.9614 lr 2.40e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB
[01:26:37] step 4260/4768 loss 2.9540 lr 2.31e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB
[01:29:50] step 4280/4768 loss 2.9529 lr 2.23e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB
[01:33:03] step 4300/4768 loss 2.9506 lr 2.15e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB
[01:36:16] step 4320/4768 loss 2.9459 lr 2.07e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB
[01:39:30] step 4340/4768 loss 2.9443 lr 1.99e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB
[01:42:43] step 4360/4768 loss 2.9391 lr 1.92e-04 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB
[01:45:56] step 4380/4768 loss 2.9453 lr 1.84e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB
[01:49:09] step 4400/4768 loss 2.9431 lr 1.77e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB
[01:52:22] step 4420/4768 loss 2.9360 lr 1.70e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB
[01:55:35] step 4440/4768 loss 2.9422 lr 1.63e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB
[01:58:48] step 4460/4768 loss 2.9381 lr 1.56e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB
[02:02:01] step 4480/4768 loss 2.9331 lr 1.49e-04 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB
[02:05:15] step 4500/4768 loss 2.9303 lr 1.42e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB
[02:08:28] step 4520/4768 loss 2.9317 lr 1.36e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB
[02:11:41] step 4540/4768 loss 2.9233 lr 1.29e-04 gnorm 0.07 54,293 tok/s peakVRAM 13.7GB
[02:14:54] step 4560/4768 loss 2.9264 lr 1.23e-04 gnorm 0.07 54,295 tok/s peakVRAM 13.7GB
[02:18:07] step 4580/4768 loss 2.9300 lr 1.16e-04 gnorm 0.07 54,293 tok/s peakVRAM 13.7GB
[02:21:20] step 4600/4768 loss 2.9284 lr 1.10e-04 gnorm 0.07 54,295 tok/s peakVRAM 13.7GB
[02:24:33] step 4620/4768 loss 2.9155 lr 1.04e-04 gnorm 0.07 54,290 tok/s peakVRAM 13.7GB
[02:27:46] step 4640/4768 loss 2.9244 lr 9.78e-05 gnorm 0.07 54,291 tok/s peakVRAM 13.7GB
[02:31:00] step 4660/4768 loss 2.9173 lr 9.18e-05 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB
[02:34:13] step 4680/4768 loss 2.9151 lr 8.58e-05 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB
[02:37:26] step 4700/4768 loss 2.9109 lr 7.99e-05 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB
[02:40:39] step 4720/4768 loss 2.9122 lr 7.41e-05 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB
[02:43:52] step 4740/4768 loss 2.9100 lr 6.83e-05 gnorm 0.07 54,301 tok/s peakVRAM 13.7GB
[02:47:05] step 4760/4768 loss 2.9194 lr 6.26e-05 gnorm 0.07 54,313 tok/s peakVRAM 13.7GB
[02:48:24] ckpt @ 4768
[02:48:24] DONE
|