Upload folder using huggingface_hub
Browse files
logs/stdout/proxy1b_20260828_1539.log
CHANGED
|
@@ -1707,3 +1707,450 @@ step 787 | loss: 2.760746 | lr 1.0000e-03 | norm: 0.2322 | dt: 5935.57ms | tok
|
|
| 1707 |
[TIMING] data: 7.4ms | fwd: 1812.2ms | loss: 12.9ms | bwd: 4040.3ms | clip: 0.0ms | optim: 49.9ms
|
| 1708 |
step 788 | loss: 2.770452 | lr 1.0000e-03 | norm: 0.2111 | dt: 5937.72ms | tok/sec: 176595.9 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1709 |
[TIMING] data: 7.4ms | fwd: 1813.3ms | loss: 12.9ms | bwd: 4041.3ms | clip: 0.0ms | optim: 49.9ms
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1707 |
[TIMING] data: 7.4ms | fwd: 1812.2ms | loss: 12.9ms | bwd: 4040.3ms | clip: 0.0ms | optim: 49.9ms
|
| 1708 |
step 788 | loss: 2.770452 | lr 1.0000e-03 | norm: 0.2111 | dt: 5937.72ms | tok/sec: 176595.9 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1709 |
[TIMING] data: 7.4ms | fwd: 1813.3ms | loss: 12.9ms | bwd: 4041.3ms | clip: 0.0ms | optim: 49.9ms
|
| 1710 |
+
step 789 | loss: 2.776759 | lr 1.0000e-03 | norm: 0.2091 | dt: 5955.42ms | tok/sec: 176070.7 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1711 |
+
[TIMING] data: 7.4ms | fwd: 1816.1ms | loss: 12.9ms | bwd: 4056.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1712 |
+
step 790 | loss: 2.744757 | lr 1.0000e-03 | norm: 0.2047 | dt: 5949.19ms | tok/sec: 176255.4 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1713 |
+
[TIMING] data: 7.4ms | fwd: 1814.4ms | loss: 12.9ms | bwd: 4051.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1714 |
+
step 791 | loss: 2.800298 | lr 1.0000e-03 | norm: 0.1973 | dt: 5941.71ms | tok/sec: 176477.1 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1715 |
+
[TIMING] data: 7.4ms | fwd: 1818.8ms | loss: 12.9ms | bwd: 4039.8ms | clip: 0.0ms | optim: 50.0ms
|
| 1716 |
+
step 792 | loss: 2.741198 | lr 1.0000e-03 | norm: 0.1913 | dt: 5938.08ms | tok/sec: 176585.1 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1717 |
+
[TIMING] data: 7.3ms | fwd: 1813.7ms | loss: 12.9ms | bwd: 4041.4ms | clip: 0.0ms | optim: 49.9ms
|
| 1718 |
+
step 793 | loss: 2.769120 | lr 1.0000e-03 | norm: 0.2016 | dt: 5949.52ms | tok/sec: 176245.6 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1719 |
+
[TIMING] data: 7.4ms | fwd: 1809.5ms | loss: 12.9ms | bwd: 4056.7ms | clip: 0.0ms | optim: 50.0ms
|
| 1720 |
+
step 794 | loss: 2.708147 | lr 1.0000e-03 | norm: 0.2087 | dt: 5947.61ms | tok/sec: 176302.0 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1721 |
+
[TIMING] data: 7.4ms | fwd: 1812.4ms | loss: 12.9ms | bwd: 4052.0ms | clip: 0.0ms | optim: 50.0ms
|
| 1722 |
+
step 795 | loss: 2.766753 | lr 1.0000e-03 | norm: 0.2106 | dt: 5956.69ms | tok/sec: 176033.2 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1723 |
+
[TIMING] data: 7.4ms | fwd: 1816.4ms | loss: 12.9ms | bwd: 4057.2ms | clip: 0.0ms | optim: 49.9ms
|
| 1724 |
+
step 796 | loss: 2.777637 | lr 1.0000e-03 | norm: 0.1937 | dt: 5958.09ms | tok/sec: 175991.9 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1725 |
+
[TIMING] data: 7.4ms | fwd: 1815.0ms | loss: 12.9ms | bwd: 4060.1ms | clip: 0.0ms | optim: 49.8ms
|
| 1726 |
+
step 797 | loss: 2.785909 | lr 1.0000e-03 | norm: 0.1869 | dt: 5953.33ms | tok/sec: 176132.8 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1727 |
+
[TIMING] data: 7.4ms | fwd: 1817.4ms | loss: 12.9ms | bwd: 4052.7ms | clip: 0.0ms | optim: 50.1ms
|
| 1728 |
+
step 798 | loss: 2.737992 | lr 1.0000e-03 | norm: 0.1851 | dt: 5954.09ms | tok/sec: 176110.3 | mem: 95437MB | Time Left: 00 hrs : 20 mins
|
| 1729 |
+
[TIMING] data: 7.4ms | fwd: 1815.4ms | loss: 12.9ms | bwd: 4055.5ms | clip: 0.0ms | optim: 50.0ms
|
| 1730 |
+
step 799 | loss: 2.741153 | lr 1.0000e-03 | norm: 0.1915 | dt: 5953.37ms | tok/sec: 176131.5 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1731 |
+
[TIMING] data: 7.3ms | fwd: 1816.1ms | loss: 12.9ms | bwd: 4054.3ms | clip: 0.0ms | optim: 49.8ms
|
| 1732 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_800.ckpt (safely)
|
| 1733 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_750.ckpt
|
| 1734 |
+
step 800 | loss: 2.787261 | lr 1.0000e-03 | norm: 0.1837 | dt: 5895.17ms | tok/sec: 177870.3 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1735 |
+
[TIMING] data: 7.4ms | fwd: 1798.5ms | loss: 12.9ms | bwd: 4013.5ms | clip: 0.0ms | optim: 49.8ms
|
| 1736 |
+
step 801 | loss: 2.733080 | lr 9.9994e-04 | norm: 0.1834 | dt: 5924.03ms | tok/sec: 177003.7 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1737 |
+
[TIMING] data: 7.3ms | fwd: 1809.6ms | loss: 12.9ms | bwd: 4031.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1738 |
+
step 802 | loss: 2.765313 | lr 9.9978e-04 | norm: 0.2010 | dt: 5917.94ms | tok/sec: 177186.0 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1739 |
+
[TIMING] data: 7.4ms | fwd: 1806.5ms | loss: 12.9ms | bwd: 4028.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1740 |
+
step 803 | loss: 2.801133 | lr 9.9950e-04 | norm: 0.2103 | dt: 5938.80ms | tok/sec: 176563.6 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1741 |
+
[TIMING] data: 7.4ms | fwd: 1809.8ms | loss: 12.9ms | bwd: 4045.9ms | clip: 0.0ms | optim: 50.0ms
|
| 1742 |
+
step 804 | loss: 2.702775 | lr 9.9911e-04 | norm: 0.2131 | dt: 5920.28ms | tok/sec: 177116.0 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1743 |
+
[TIMING] data: 7.5ms | fwd: 1810.5ms | loss: 12.9ms | bwd: 4026.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1744 |
+
step 805 | loss: 2.681854 | lr 9.9861e-04 | norm: 0.2145 | dt: 5954.05ms | tok/sec: 176111.4 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1745 |
+
[TIMING] data: 7.7ms | fwd: 1812.1ms | loss: 12.9ms | bwd: 4057.5ms | clip: 0.0ms | optim: 50.0ms
|
| 1746 |
+
step 806 | loss: 2.755555 | lr 9.9800e-04 | norm: 0.2164 | dt: 5943.06ms | tok/sec: 176437.1 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1747 |
+
[TIMING] data: 7.4ms | fwd: 1812.3ms | loss: 12.9ms | bwd: 4047.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1748 |
+
step 807 | loss: 2.967290 | lr 9.9728e-04 | norm: 0.2373 | dt: 5938.36ms | tok/sec: 176576.6 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1749 |
+
[TIMING] data: 7.4ms | fwd: 1807.8ms | loss: 12.9ms | bwd: 4047.4ms | clip: 0.0ms | optim: 49.9ms
|
| 1750 |
+
step 808 | loss: 2.861712 | lr 9.9645e-04 | norm: 0.2266 | dt: 5945.41ms | tok/sec: 176367.2 | mem: 95437MB | Time Left: 00 hrs : 19 mins
|
| 1751 |
+
[TIMING] data: 7.3ms | fwd: 1809.4ms | loss: 12.9ms | bwd: 4052.9ms | clip: 0.0ms | optim: 50.0ms
|
| 1752 |
+
step 809 | loss: 2.802719 | lr 9.9551e-04 | norm: 0.2373 | dt: 5957.38ms | tok/sec: 176012.8 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1753 |
+
[TIMING] data: 7.3ms | fwd: 1814.5ms | loss: 12.9ms | bwd: 4059.8ms | clip: 0.0ms | optim: 49.9ms
|
| 1754 |
+
step 810 | loss: 2.682600 | lr 9.9446e-04 | norm: 0.2117 | dt: 5924.76ms | tok/sec: 176982.2 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1755 |
+
[TIMING] data: 7.3ms | fwd: 1808.4ms | loss: 12.9ms | bwd: 4033.4ms | clip: 0.0ms | optim: 49.8ms
|
| 1756 |
+
step 811 | loss: 2.853862 | lr 9.9330e-04 | norm: 0.2195 | dt: 5940.73ms | tok/sec: 176506.2 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1757 |
+
[TIMING] data: 7.3ms | fwd: 1813.1ms | loss: 12.9ms | bwd: 4044.6ms | clip: 0.0ms | optim: 49.9ms
|
| 1758 |
+
step 812 | loss: 2.667439 | lr 9.9203e-04 | norm: 0.1868 | dt: 5941.55ms | tok/sec: 176481.8 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1759 |
+
[TIMING] data: 7.3ms | fwd: 1812.2ms | loss: 12.9ms | bwd: 4046.3ms | clip: 0.0ms | optim: 49.9ms
|
| 1760 |
+
step 813 | loss: 2.720612 | lr 9.9065e-04 | norm: 0.1986 | dt: 5953.56ms | tok/sec: 176125.8 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1761 |
+
[TIMING] data: 7.4ms | fwd: 1815.8ms | loss: 12.9ms | bwd: 4054.7ms | clip: 0.0ms | optim: 49.9ms
|
| 1762 |
+
step 814 | loss: 2.735756 | lr 9.8916e-04 | norm: 0.2035 | dt: 5952.67ms | tok/sec: 176152.3 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1763 |
+
[TIMING] data: 7.4ms | fwd: 1817.4ms | loss: 12.9ms | bwd: 4052.1ms | clip: 0.0ms | optim: 49.9ms
|
| 1764 |
+
step 815 | loss: 2.755608 | lr 9.8757e-04 | norm: 0.1732 | dt: 5963.34ms | tok/sec: 175837.0 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1765 |
+
[TIMING] data: 7.3ms | fwd: 1815.3ms | loss: 12.9ms | bwd: 4064.8ms | clip: 0.0ms | optim: 50.0ms
|
| 1766 |
+
step 816 | loss: 2.777060 | lr 9.8586e-04 | norm: 0.1759 | dt: 5954.59ms | tok/sec: 176095.3 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1767 |
+
[TIMING] data: 7.3ms | fwd: 1814.1ms | loss: 12.9ms | bwd: 4057.4ms | clip: 0.0ms | optim: 49.9ms
|
| 1768 |
+
step 817 | loss: 2.674704 | lr 9.8405e-04 | norm: 0.1797 | dt: 5946.20ms | tok/sec: 176343.8 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1769 |
+
[TIMING] data: 7.4ms | fwd: 1817.3ms | loss: 13.0ms | bwd: 4045.6ms | clip: 0.0ms | optim: 49.9ms
|
| 1770 |
+
step 818 | loss: 2.701495 | lr 9.8213e-04 | norm: 0.1953 | dt: 5950.62ms | tok/sec: 176213.0 | mem: 95437MB | Time Left: 00 hrs : 18 mins
|
| 1771 |
+
[TIMING] data: 7.6ms | fwd: 1812.0ms | loss: 13.2ms | bwd: 4054.8ms | clip: 0.0ms | optim: 50.0ms
|
| 1772 |
+
step 819 | loss: 2.783086 | lr 9.8011e-04 | norm: 0.2514 | dt: 5950.96ms | tok/sec: 176202.9 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1773 |
+
[TIMING] data: 7.6ms | fwd: 1812.5ms | loss: 13.2ms | bwd: 4054.6ms | clip: 0.0ms | optim: 50.0ms
|
| 1774 |
+
step 820 | loss: 2.724981 | lr 9.7798e-04 | norm: 0.2041 | dt: 5957.20ms | tok/sec: 176018.4 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1775 |
+
[TIMING] data: 7.6ms | fwd: 1816.6ms | loss: 13.2ms | bwd: 4056.7ms | clip: 0.0ms | optim: 50.0ms
|
| 1776 |
+
step 821 | loss: 2.725283 | lr 9.7574e-04 | norm: 0.2311 | dt: 5960.68ms | tok/sec: 175915.4 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1777 |
+
[TIMING] data: 7.5ms | fwd: 1816.5ms | loss: 13.2ms | bwd: 4059.0ms | clip: 0.0ms | optim: 51.4ms
|
| 1778 |
+
step 822 | loss: 2.721730 | lr 9.7340e-04 | norm: 0.2195 | dt: 5954.61ms | tok/sec: 176094.9 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1779 |
+
[TIMING] data: 7.5ms | fwd: 1813.5ms | loss: 13.2ms | bwd: 4056.6ms | clip: 0.0ms | optim: 50.7ms
|
| 1780 |
+
step 823 | loss: 2.706247 | lr 9.7095e-04 | norm: 0.1954 | dt: 5951.26ms | tok/sec: 176194.1 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1781 |
+
[TIMING] data: 7.6ms | fwd: 1816.0ms | loss: 13.2ms | bwd: 4050.9ms | clip: 0.0ms | optim: 50.6ms
|
| 1782 |
+
step 824 | loss: 2.788187 | lr 9.6840e-04 | norm: 0.1873 | dt: 5945.18ms | tok/sec: 176374.1 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1783 |
+
[TIMING] data: 7.5ms | fwd: 1820.0ms | loss: 13.2ms | bwd: 4041.1ms | clip: 0.0ms | optim: 49.9ms
|
| 1784 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_825.ckpt (safely)
|
| 1785 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_775.ckpt
|
| 1786 |
+
step 825 | loss: 2.741726 | lr 9.6575e-04 | norm: 0.1833 | dt: 5900.06ms | tok/sec: 177723.0 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1787 |
+
[TIMING] data: 7.6ms | fwd: 1798.6ms | loss: 13.2ms | bwd: 4017.6ms | clip: 0.0ms | optim: 49.9ms
|
| 1788 |
+
step 826 | loss: 2.855867 | lr 9.6299e-04 | norm: 0.2057 | dt: 5915.41ms | tok/sec: 177261.8 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1789 |
+
[TIMING] data: 7.6ms | fwd: 1808.5ms | loss: 13.1ms | bwd: 4023.1ms | clip: 0.0ms | optim: 49.9ms
|
| 1790 |
+
step 827 | loss: 2.719985 | lr 9.6013e-04 | norm: 0.2032 | dt: 5919.10ms | tok/sec: 177151.1 | mem: 95437MB | Time Left: 00 hrs : 17 mins
|
| 1791 |
+
[TIMING] data: 7.6ms | fwd: 1806.8ms | loss: 13.2ms | bwd: 4028.6ms | clip: 0.0ms | optim: 50.0ms
|
| 1792 |
+
step 828 | loss: 2.723501 | lr 9.5717e-04 | norm: 0.1788 | dt: 5928.41ms | tok/sec: 176873.2 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1793 |
+
[TIMING] data: 7.7ms | fwd: 1808.2ms | loss: 13.2ms | bwd: 4036.3ms | clip: 0.0ms | optim: 49.9ms
|
| 1794 |
+
step 829 | loss: 2.741814 | lr 9.5411e-04 | norm: 0.1709 | dt: 5931.51ms | tok/sec: 176780.6 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1795 |
+
[TIMING] data: 7.6ms | fwd: 1810.8ms | loss: 13.2ms | bwd: 4036.8ms | clip: 0.0ms | optim: 49.9ms
|
| 1796 |
+
step 830 | loss: 2.722056 | lr 9.5095e-04 | norm: 0.1595 | dt: 5946.20ms | tok/sec: 176344.0 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1797 |
+
[TIMING] data: 7.7ms | fwd: 1814.3ms | loss: 13.2ms | bwd: 4047.9ms | clip: 0.0ms | optim: 50.0ms
|
| 1798 |
+
step 831 | loss: 2.745772 | lr 9.4769e-04 | norm: 0.1551 | dt: 5951.37ms | tok/sec: 176190.6 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1799 |
+
[TIMING] data: 7.6ms | fwd: 1815.4ms | loss: 13.2ms | bwd: 4052.1ms | clip: 0.0ms | optim: 49.8ms
|
| 1800 |
+
step 832 | loss: 2.765649 | lr 9.4434e-04 | norm: 0.1658 | dt: 5942.53ms | tok/sec: 176452.8 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1801 |
+
[TIMING] data: 7.6ms | fwd: 1812.2ms | loss: 13.2ms | bwd: 4046.6ms | clip: 0.0ms | optim: 50.0ms
|
| 1802 |
+
step 833 | loss: 2.809209 | lr 9.4088e-04 | norm: 0.1836 | dt: 5955.92ms | tok/sec: 176056.2 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1803 |
+
[TIMING] data: 7.6ms | fwd: 1813.9ms | loss: 13.2ms | bwd: 4058.3ms | clip: 0.0ms | optim: 49.9ms
|
| 1804 |
+
step 834 | loss: 2.834930 | lr 9.3733e-04 | norm: 0.1841 | dt: 5962.15ms | tok/sec: 175872.2 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1805 |
+
[TIMING] data: 8.5ms | fwd: 1817.2ms | loss: 13.2ms | bwd: 4060.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1806 |
+
step 835 | loss: 2.805997 | lr 9.3369e-04 | norm: 0.2164 | dt: 5937.67ms | tok/sec: 176597.3 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1807 |
+
[TIMING] data: 7.6ms | fwd: 1812.0ms | loss: 13.2ms | bwd: 4041.8ms | clip: 0.0ms | optim: 50.0ms
|
| 1808 |
+
step 836 | loss: 2.751029 | lr 9.2995e-04 | norm: 0.2148 | dt: 5952.36ms | tok/sec: 176161.3 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1809 |
+
[TIMING] data: 7.6ms | fwd: 1816.9ms | loss: 13.2ms | bwd: 4051.6ms | clip: 0.0ms | optim: 49.9ms
|
| 1810 |
+
step 837 | loss: 2.801919 | lr 9.2611e-04 | norm: 0.2055 | dt: 5950.26ms | tok/sec: 176223.7 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1811 |
+
[TIMING] data: 7.6ms | fwd: 1811.8ms | loss: 13.2ms | bwd: 4054.8ms | clip: 0.0ms | optim: 49.8ms
|
| 1812 |
+
step 838 | loss: 2.842059 | lr 9.2219e-04 | norm: 0.3696 | dt: 5946.57ms | tok/sec: 176332.9 | mem: 95437MB | Time Left: 00 hrs : 16 mins
|
| 1813 |
+
[TIMING] data: 7.7ms | fwd: 1819.2ms | loss: 13.2ms | bwd: 4043.7ms | clip: 0.0ms | optim: 49.8ms
|
| 1814 |
+
step 839 | loss: 2.724386 | lr 9.1817e-04 | norm: 0.1721 | dt: 5954.80ms | tok/sec: 176089.1 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1815 |
+
[TIMING] data: 7.6ms | fwd: 1814.5ms | loss: 13.2ms | bwd: 4056.4ms | clip: 0.0ms | optim: 50.0ms
|
| 1816 |
+
step 840 | loss: 2.785441 | lr 9.1406e-04 | norm: 0.1564 | dt: 5950.19ms | tok/sec: 176225.7 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1817 |
+
[TIMING] data: 7.9ms | fwd: 1817.4ms | loss: 13.2ms | bwd: 4048.7ms | clip: 0.0ms | optim: 50.0ms
|
| 1818 |
+
step 841 | loss: 2.743148 | lr 9.0986e-04 | norm: 0.1751 | dt: 5946.80ms | tok/sec: 176326.2 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1819 |
+
[TIMING] data: 7.6ms | fwd: 1814.6ms | loss: 13.2ms | bwd: 4048.3ms | clip: 0.0ms | optim: 50.1ms
|
| 1820 |
+
step 842 | loss: 2.741702 | lr 9.0557e-04 | norm: 0.1780 | dt: 5948.44ms | tok/sec: 176277.4 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1821 |
+
[TIMING] data: 7.6ms | fwd: 1817.0ms | loss: 13.2ms | bwd: 4047.5ms | clip: 0.0ms | optim: 50.0ms
|
| 1822 |
+
step 843 | loss: 2.755732 | lr 9.0119e-04 | norm: 0.1834 | dt: 5945.70ms | tok/sec: 176358.8 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1823 |
+
[TIMING] data: 7.7ms | fwd: 1815.4ms | loss: 13.2ms | bwd: 4046.4ms | clip: 0.0ms | optim: 49.8ms
|
| 1824 |
+
step 844 | loss: 2.752215 | lr 8.9673e-04 | norm: 0.1719 | dt: 5952.95ms | tok/sec: 176143.8 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1825 |
+
[TIMING] data: 7.7ms | fwd: 1814.6ms | loss: 13.2ms | bwd: 4054.5ms | clip: 0.0ms | optim: 50.0ms
|
| 1826 |
+
step 845 | loss: 2.820490 | lr 8.9218e-04 | norm: 0.1808 | dt: 5947.11ms | tok/sec: 176316.8 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1827 |
+
[TIMING] data: 7.6ms | fwd: 1815.2ms | loss: 13.2ms | bwd: 4046.9ms | clip: 0.0ms | optim: 50.5ms
|
| 1828 |
+
step 846 | loss: 2.790235 | lr 8.8755e-04 | norm: 0.1713 | dt: 5958.75ms | tok/sec: 175972.4 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1829 |
+
[TIMING] data: 7.7ms | fwd: 1816.1ms | loss: 13.2ms | bwd: 4058.8ms | clip: 0.0ms | optim: 50.0ms
|
| 1830 |
+
step 847 | loss: 2.829271 | lr 8.8283e-04 | norm: 0.1678 | dt: 5949.57ms | tok/sec: 176244.1 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1831 |
+
[TIMING] data: 7.5ms | fwd: 1813.7ms | loss: 13.2ms | bwd: 4052.0ms | clip: 0.0ms | optim: 50.0ms
|
| 1832 |
+
step 848 | loss: 2.834256 | lr 8.7804e-04 | norm: 0.1876 | dt: 5962.57ms | tok/sec: 175859.6 | mem: 95437MB | Time Left: 00 hrs : 15 mins
|
| 1833 |
+
[TIMING] data: 7.7ms | fwd: 1817.5ms | loss: 13.2ms | bwd: 4061.0ms | clip: 0.0ms | optim: 50.1ms
|
| 1834 |
+
step 849 | loss: 2.803730 | lr 8.7316e-04 | norm: 0.1557 | dt: 5960.05ms | tok/sec: 175934.2 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1835 |
+
[TIMING] data: 7.5ms | fwd: 1815.8ms | loss: 13.2ms | bwd: 4060.2ms | clip: 0.0ms | optim: 50.2ms
|
| 1836 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_850.ckpt (safely)
|
| 1837 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_800.ckpt
|
| 1838 |
+
step 850 | loss: 2.739548 | lr 8.6820e-04 | norm: 0.1432 | dt: 5900.28ms | tok/sec: 177716.3 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1839 |
+
[TIMING] data: 7.6ms | fwd: 1800.8ms | loss: 13.1ms | bwd: 4015.7ms | clip: 0.0ms | optim: 49.9ms
|
| 1840 |
+
step 851 | loss: 2.712695 | lr 8.6316e-04 | norm: 0.1689 | dt: 5927.52ms | tok/sec: 176899.7 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1841 |
+
[TIMING] data: 7.5ms | fwd: 1810.2ms | loss: 13.2ms | bwd: 4033.6ms | clip: 0.0ms | optim: 50.0ms
|
| 1842 |
+
step 852 | loss: 2.741832 | lr 8.5805e-04 | norm: 0.1790 | dt: 5933.82ms | tok/sec: 176711.8 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1843 |
+
[TIMING] data: 7.8ms | fwd: 1809.3ms | loss: 13.2ms | bwd: 4040.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1844 |
+
step 853 | loss: 2.713052 | lr 8.5286e-04 | norm: 0.1701 | dt: 5929.51ms | tok/sec: 176840.3 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1845 |
+
[TIMING] data: 7.5ms | fwd: 1812.4ms | loss: 13.2ms | bwd: 4033.4ms | clip: 0.0ms | optim: 49.8ms
|
| 1846 |
+
step 854 | loss: 2.742614 | lr 8.4759e-04 | norm: 0.1625 | dt: 5939.28ms | tok/sec: 176549.4 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1847 |
+
[TIMING] data: 7.6ms | fwd: 1814.5ms | loss: 13.2ms | bwd: 4040.9ms | clip: 0.0ms | optim: 49.9ms
|
| 1848 |
+
step 855 | loss: 2.781667 | lr 8.4225e-04 | norm: 0.1640 | dt: 5941.98ms | tok/sec: 176469.2 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1849 |
+
[TIMING] data: 7.7ms | fwd: 1817.6ms | loss: 13.2ms | bwd: 4040.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1850 |
+
step 856 | loss: 2.709754 | lr 8.3684e-04 | norm: 0.1702 | dt: 5939.19ms | tok/sec: 176552.1 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1851 |
+
[TIMING] data: 7.6ms | fwd: 1812.4ms | loss: 13.2ms | bwd: 4042.7ms | clip: 0.0ms | optim: 50.3ms
|
| 1852 |
+
Current shard: /root/.cache/huggingface/hub/datasets--navingv--flux/snapshots/5232b3d133d2eb71905725f288ff6b56dc0df773/flux_train_000010.bin
|
| 1853 |
+
100000000
|
| 1854 |
+
step 857 | loss: 2.686680 | lr 8.3136e-04 | norm: 0.1749 | dt: 5942.86ms | tok/sec: 176442.9 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1855 |
+
[TIMING] data: 10.7ms | fwd: 1810.4ms | loss: 13.2ms | bwd: 4045.6ms | clip: 0.0ms | optim: 49.8ms
|
| 1856 |
+
step 858 | loss: 2.785247 | lr 8.2581e-04 | norm: 0.1908 | dt: 5944.69ms | tok/sec: 176388.8 | mem: 95437MB | Time Left: 00 hrs : 14 mins
|
| 1857 |
+
[TIMING] data: 7.6ms | fwd: 1812.4ms | loss: 13.2ms | bwd: 4048.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1858 |
+
step 859 | loss: 2.766020 | lr 8.2019e-04 | norm: 0.1597 | dt: 5946.41ms | tok/sec: 176337.7 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1859 |
+
[TIMING] data: 7.6ms | fwd: 1813.4ms | loss: 13.2ms | bwd: 4049.3ms | clip: 0.0ms | optim: 50.0ms
|
| 1860 |
+
step 860 | loss: 2.677573 | lr 8.1450e-04 | norm: 0.1735 | dt: 5949.05ms | tok/sec: 176259.4 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1861 |
+
[TIMING] data: 7.5ms | fwd: 1810.8ms | loss: 13.2ms | bwd: 4054.4ms | clip: 0.0ms | optim: 50.0ms
|
| 1862 |
+
step 861 | loss: 2.716818 | lr 8.0875e-04 | norm: 0.1708 | dt: 5949.05ms | tok/sec: 176259.5 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1863 |
+
[TIMING] data: 7.6ms | fwd: 1816.1ms | loss: 13.2ms | bwd: 4049.2ms | clip: 0.0ms | optim: 49.9ms
|
| 1864 |
+
step 862 | loss: 2.786133 | lr 8.0294e-04 | norm: 0.1578 | dt: 5953.01ms | tok/sec: 176142.3 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1865 |
+
[TIMING] data: 7.6ms | fwd: 1816.0ms | loss: 13.1ms | bwd: 4053.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1866 |
+
step 863 | loss: 2.790838 | lr 7.9706e-04 | norm: 0.1477 | dt: 5954.12ms | tok/sec: 176109.3 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1867 |
+
[TIMING] data: 7.6ms | fwd: 1820.8ms | loss: 13.2ms | bwd: 4049.3ms | clip: 0.0ms | optim: 50.2ms
|
| 1868 |
+
step 864 | loss: 2.752659 | lr 7.9112e-04 | norm: 0.1474 | dt: 5960.40ms | tok/sec: 175923.8 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1869 |
+
[TIMING] data: 7.6ms | fwd: 1816.2ms | loss: 13.1ms | bwd: 4060.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1870 |
+
step 865 | loss: 2.717798 | lr 7.8512e-04 | norm: 0.1443 | dt: 5956.03ms | tok/sec: 176052.9 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1871 |
+
[TIMING] data: 7.6ms | fwd: 1816.7ms | loss: 13.2ms | bwd: 4055.4ms | clip: 0.0ms | optim: 50.0ms
|
| 1872 |
+
step 866 | loss: 2.714525 | lr 7.7907e-04 | norm: 0.1482 | dt: 5956.26ms | tok/sec: 176046.2 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1873 |
+
[TIMING] data: 7.6ms | fwd: 1815.1ms | loss: 13.2ms | bwd: 4057.3ms | clip: 0.0ms | optim: 49.9ms
|
| 1874 |
+
step 867 | loss: 2.828845 | lr 7.7296e-04 | norm: 0.1501 | dt: 5963.38ms | tok/sec: 175835.7 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1875 |
+
[TIMING] data: 7.6ms | fwd: 1818.3ms | loss: 13.2ms | bwd: 4061.3ms | clip: 0.0ms | optim: 49.8ms
|
| 1876 |
+
step 868 | loss: 2.704845 | lr 7.6679e-04 | norm: 0.1408 | dt: 5948.66ms | tok/sec: 176271.0 | mem: 95437MB | Time Left: 00 hrs : 13 mins
|
| 1877 |
+
[TIMING] data: 7.6ms | fwd: 1814.5ms | loss: 13.2ms | bwd: 4050.0ms | clip: 0.0ms | optim: 50.0ms
|
| 1878 |
+
step 869 | loss: 2.721409 | lr 7.6057e-04 | norm: 0.1376 | dt: 5952.24ms | tok/sec: 176164.9 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1879 |
+
[TIMING] data: 7.7ms | fwd: 1815.1ms | loss: 13.2ms | bwd: 4053.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1880 |
+
step 870 | loss: 2.608265 | lr 7.5430e-04 | norm: 0.1354 | dt: 5964.68ms | tok/sec: 175797.4 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1881 |
+
[TIMING] data: 7.5ms | fwd: 1817.0ms | loss: 13.2ms | bwd: 4064.0ms | clip: 0.0ms | optim: 49.9ms
|
| 1882 |
+
step 871 | loss: 2.698054 | lr 7.4797e-04 | norm: 0.1605 | dt: 5947.62ms | tok/sec: 176301.9 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1883 |
+
[TIMING] data: 7.6ms | fwd: 1816.0ms | loss: 13.2ms | bwd: 4047.8ms | clip: 0.0ms | optim: 50.0ms
|
| 1884 |
+
step 872 | loss: 2.700965 | lr 7.4160e-04 | norm: 0.1497 | dt: 5964.66ms | tok/sec: 175798.1 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1885 |
+
[TIMING] data: 7.7ms | fwd: 1816.7ms | loss: 13.2ms | bwd: 4064.1ms | clip: 0.0ms | optim: 49.9ms
|
| 1886 |
+
step 873 | loss: 2.816442 | lr 7.3518e-04 | norm: 0.1478 | dt: 5957.33ms | tok/sec: 176014.3 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1887 |
+
[TIMING] data: 7.6ms | fwd: 1814.5ms | loss: 13.2ms | bwd: 4059.0ms | clip: 0.0ms | optim: 50.0ms
|
| 1888 |
+
step 874 | loss: 2.724926 | lr 7.2872e-04 | norm: 0.1503 | dt: 5963.89ms | tok/sec: 175820.7 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1889 |
+
[TIMING] data: 7.6ms | fwd: 1817.5ms | loss: 13.2ms | bwd: 4061.7ms | clip: 0.0ms | optim: 50.8ms
|
| 1890 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_875.ckpt (safely)
|
| 1891 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_825.ckpt
|
| 1892 |
+
step 875 | loss: 2.726554 | lr 7.2221e-04 | norm: 0.1294 | dt: 5888.63ms | tok/sec: 178067.9 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1893 |
+
[TIMING] data: 7.4ms | fwd: 1796.0ms | loss: 13.1ms | bwd: 4009.2ms | clip: 0.0ms | optim: 49.9ms
|
| 1894 |
+
step 876 | loss: 2.638878 | lr 7.1566e-04 | norm: 0.1293 | dt: 5913.94ms | tok/sec: 177305.9 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1895 |
+
[TIMING] data: 7.6ms | fwd: 1809.3ms | loss: 13.1ms | bwd: 4021.0ms | clip: 0.0ms | optim: 49.9ms
|
| 1896 |
+
step 877 | loss: 2.694532 | lr 7.0906e-04 | norm: 0.1330 | dt: 5932.08ms | tok/sec: 176763.8 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1897 |
+
[TIMING] data: 7.6ms | fwd: 1807.4ms | loss: 13.1ms | bwd: 4040.9ms | clip: 0.0ms | optim: 50.0ms
|
| 1898 |
+
step 878 | loss: 2.668688 | lr 7.0243e-04 | norm: 0.1384 | dt: 5929.32ms | tok/sec: 176845.9 | mem: 95437MB | Time Left: 00 hrs : 12 mins
|
| 1899 |
+
[TIMING] data: 7.5ms | fwd: 1805.4ms | loss: 13.1ms | bwd: 4040.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1900 |
+
step 879 | loss: 2.698247 | lr 6.9576e-04 | norm: 0.1417 | dt: 5937.31ms | tok/sec: 176607.9 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1901 |
+
[TIMING] data: 7.6ms | fwd: 1813.1ms | loss: 13.2ms | bwd: 4040.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1902 |
+
step 880 | loss: 2.724090 | lr 6.8906e-04 | norm: 0.1467 | dt: 5940.46ms | tok/sec: 176514.1 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1903 |
+
[TIMING] data: 7.7ms | fwd: 1808.6ms | loss: 13.2ms | bwd: 4047.7ms | clip: 0.0ms | optim: 50.0ms
|
| 1904 |
+
step 881 | loss: 2.647027 | lr 6.8232e-04 | norm: 0.1327 | dt: 5953.25ms | tok/sec: 176135.1 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1905 |
+
[TIMING] data: 7.6ms | fwd: 1811.9ms | loss: 13.2ms | bwd: 4057.5ms | clip: 0.0ms | optim: 50.0ms
|
| 1906 |
+
step 882 | loss: 2.741259 | lr 6.7555e-04 | norm: 0.1389 | dt: 5945.21ms | tok/sec: 176373.3 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1907 |
+
[TIMING] data: 7.7ms | fwd: 1817.1ms | loss: 13.2ms | bwd: 4044.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1908 |
+
step 883 | loss: 2.676762 | lr 6.6874e-04 | norm: 0.1356 | dt: 5948.23ms | tok/sec: 176283.8 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1909 |
+
[TIMING] data: 7.6ms | fwd: 1816.5ms | loss: 13.2ms | bwd: 4047.9ms | clip: 0.0ms | optim: 50.0ms
|
| 1910 |
+
step 884 | loss: 2.662589 | lr 6.6191e-04 | norm: 0.1311 | dt: 5950.15ms | tok/sec: 176226.8 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1911 |
+
[TIMING] data: 7.6ms | fwd: 1813.8ms | loss: 13.2ms | bwd: 4052.5ms | clip: 0.0ms | optim: 50.1ms
|
| 1912 |
+
step 885 | loss: 2.716739 | lr 6.5505e-04 | norm: 0.1246 | dt: 5945.11ms | tok/sec: 176376.1 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1913 |
+
[TIMING] data: 7.6ms | fwd: 1813.1ms | loss: 13.2ms | bwd: 4048.0ms | clip: 0.0ms | optim: 50.2ms
|
| 1914 |
+
step 886 | loss: 2.650722 | lr 6.4816e-04 | norm: 0.1221 | dt: 5941.01ms | tok/sec: 176498.0 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1915 |
+
[TIMING] data: 7.5ms | fwd: 1813.8ms | loss: 13.1ms | bwd: 4043.4ms | clip: 0.0ms | optim: 50.1ms
|
| 1916 |
+
step 887 | loss: 2.681291 | lr 6.4125e-04 | norm: 0.1199 | dt: 5957.93ms | tok/sec: 175996.7 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1917 |
+
[TIMING] data: 7.6ms | fwd: 1819.4ms | loss: 13.2ms | bwd: 4054.0ms | clip: 0.0ms | optim: 50.6ms
|
| 1918 |
+
step 888 | loss: 2.679257 | lr 6.3432e-04 | norm: 0.1108 | dt: 5956.29ms | tok/sec: 176045.1 | mem: 95437MB | Time Left: 00 hrs : 11 mins
|
| 1919 |
+
[TIMING] data: 7.6ms | fwd: 1815.7ms | loss: 13.2ms | bwd: 4056.6ms | clip: 0.0ms | optim: 49.9ms
|
| 1920 |
+
step 889 | loss: 2.730773 | lr 6.2737e-04 | norm: 0.1155 | dt: 5941.54ms | tok/sec: 176482.3 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1921 |
+
[TIMING] data: 7.6ms | fwd: 1808.6ms | loss: 13.1ms | bwd: 4048.5ms | clip: 0.0ms | optim: 50.7ms
|
| 1922 |
+
step 890 | loss: 2.747416 | lr 6.2040e-04 | norm: 0.1134 | dt: 5949.67ms | tok/sec: 176241.1 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1923 |
+
[TIMING] data: 7.6ms | fwd: 1813.9ms | loss: 13.2ms | bwd: 4052.1ms | clip: 0.0ms | optim: 49.9ms
|
| 1924 |
+
step 891 | loss: 2.767974 | lr 6.1341e-04 | norm: 0.1167 | dt: 5946.82ms | tok/sec: 176325.6 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1925 |
+
[TIMING] data: 7.6ms | fwd: 1811.6ms | loss: 13.2ms | bwd: 4051.3ms | clip: 0.0ms | optim: 50.1ms
|
| 1926 |
+
step 892 | loss: 2.712966 | lr 6.0640e-04 | norm: 0.1284 | dt: 5947.70ms | tok/sec: 176299.4 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1927 |
+
[TIMING] data: 7.6ms | fwd: 1815.2ms | loss: 13.2ms | bwd: 4048.7ms | clip: 0.0ms | optim: 50.0ms
|
| 1928 |
+
step 893 | loss: 2.688213 | lr 5.9938e-04 | norm: 0.1370 | dt: 5958.31ms | tok/sec: 175985.4 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1929 |
+
[TIMING] data: 7.6ms | fwd: 1814.4ms | loss: 13.2ms | bwd: 4060.1ms | clip: 0.0ms | optim: 50.0ms
|
| 1930 |
+
step 894 | loss: 2.678957 | lr 5.9235e-04 | norm: 0.1364 | dt: 5962.11ms | tok/sec: 175873.4 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1931 |
+
[TIMING] data: 7.6ms | fwd: 1814.7ms | loss: 13.2ms | bwd: 4059.8ms | clip: 0.0ms | optim: 53.7ms
|
| 1932 |
+
step 895 | loss: 2.652299 | lr 5.8531e-04 | norm: 0.1195 | dt: 5947.45ms | tok/sec: 176306.9 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1933 |
+
[TIMING] data: 7.6ms | fwd: 1813.1ms | loss: 13.2ms | bwd: 4050.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1934 |
+
step 896 | loss: 2.647521 | lr 5.7826e-04 | norm: 0.1103 | dt: 5961.59ms | tok/sec: 175888.6 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1935 |
+
[TIMING] data: 7.5ms | fwd: 1816.5ms | loss: 13.2ms | bwd: 4061.2ms | clip: 0.0ms | optim: 50.1ms
|
| 1936 |
+
step 897 | loss: 2.694245 | lr 5.7120e-04 | norm: 0.1161 | dt: 5956.57ms | tok/sec: 176036.8 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1937 |
+
[TIMING] data: 7.5ms | fwd: 1814.8ms | loss: 13.2ms | bwd: 4058.1ms | clip: 0.0ms | optim: 49.9ms
|
| 1938 |
+
step 898 | loss: 2.683159 | lr 5.6413e-04 | norm: 0.1078 | dt: 5953.71ms | tok/sec: 176121.5 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1939 |
+
[TIMING] data: 7.7ms | fwd: 1815.2ms | loss: 13.2ms | bwd: 4054.7ms | clip: 0.0ms | optim: 50.0ms
|
| 1940 |
+
step 899 | loss: 2.668508 | lr 5.5707e-04 | norm: 0.1058 | dt: 5955.17ms | tok/sec: 176078.3 | mem: 95437MB | Time Left: 00 hrs : 10 mins
|
| 1941 |
+
[TIMING] data: 7.6ms | fwd: 1817.3ms | loss: 13.2ms | bwd: 4053.0ms | clip: 0.0ms | optim: 50.0ms
|
| 1942 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_900.ckpt (safely)
|
| 1943 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_850.ckpt
|
| 1944 |
+
step 900 | loss: 2.664789 | lr 5.5000e-04 | norm: 0.1057 | dt: 5882.53ms | tok/sec: 178252.5 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1945 |
+
[TIMING] data: 7.6ms | fwd: 1795.5ms | loss: 13.2ms | bwd: 4003.4ms | clip: 0.0ms | optim: 49.9ms
|
| 1946 |
+
step 901 | loss: 2.650917 | lr 5.4293e-04 | norm: 0.1015 | dt: 5929.83ms | tok/sec: 176830.7 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1947 |
+
[TIMING] data: 7.6ms | fwd: 1810.6ms | loss: 13.1ms | bwd: 4035.5ms | clip: 0.0ms | optim: 50.0ms
|
| 1948 |
+
step 902 | loss: 2.640871 | lr 5.3587e-04 | norm: 0.1027 | dt: 5929.50ms | tok/sec: 176840.7 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1949 |
+
[TIMING] data: 7.5ms | fwd: 1810.3ms | loss: 13.2ms | bwd: 4035.5ms | clip: 0.0ms | optim: 49.9ms
|
| 1950 |
+
step 903 | loss: 2.666461 | lr 5.2880e-04 | norm: 0.1005 | dt: 5925.34ms | tok/sec: 176964.8 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1951 |
+
[TIMING] data: 7.6ms | fwd: 1807.6ms | loss: 13.2ms | bwd: 4034.0ms | clip: 0.0ms | optim: 49.9ms
|
| 1952 |
+
step 904 | loss: 2.598424 | lr 5.2174e-04 | norm: 0.0992 | dt: 5937.47ms | tok/sec: 176603.2 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1953 |
+
[TIMING] data: 7.6ms | fwd: 1813.6ms | loss: 13.1ms | bwd: 4040.1ms | clip: 0.0ms | optim: 50.0ms
|
| 1954 |
+
step 905 | loss: 2.664531 | lr 5.1469e-04 | norm: 0.1027 | dt: 5949.61ms | tok/sec: 176242.9 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1955 |
+
[TIMING] data: 7.7ms | fwd: 1810.5ms | loss: 13.2ms | bwd: 4054.5ms | clip: 0.0ms | optim: 50.1ms
|
| 1956 |
+
step 906 | loss: 2.598361 | lr 5.0765e-04 | norm: 0.0996 | dt: 5948.84ms | tok/sec: 176265.8 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1957 |
+
[TIMING] data: 7.6ms | fwd: 1813.3ms | loss: 13.2ms | bwd: 4051.4ms | clip: 0.0ms | optim: 50.0ms
|
| 1958 |
+
step 907 | loss: 2.670407 | lr 5.0062e-04 | norm: 0.0998 | dt: 5953.57ms | tok/sec: 176125.6 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1959 |
+
[TIMING] data: 7.7ms | fwd: 1813.0ms | loss: 13.2ms | bwd: 4056.7ms | clip: 0.0ms | optim: 50.0ms
|
| 1960 |
+
step 908 | loss: 2.615990 | lr 4.9360e-04 | norm: 0.1060 | dt: 5959.63ms | tok/sec: 175946.5 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1961 |
+
[TIMING] data: 7.5ms | fwd: 1814.8ms | loss: 13.2ms | bwd: 4061.1ms | clip: 0.0ms | optim: 50.0ms
|
| 1962 |
+
step 909 | loss: 2.686617 | lr 4.8659e-04 | norm: 0.0986 | dt: 5946.64ms | tok/sec: 176330.8 | mem: 95437MB | Time Left: 00 hrs : 09 mins
|
| 1963 |
+
[TIMING] data: 7.6ms | fwd: 1815.4ms | loss: 13.2ms | bwd: 4047.1ms | clip: 0.0ms | optim: 50.2ms
|
| 1964 |
+
step 910 | loss: 2.685745 | lr 4.7960e-04 | norm: 0.0933 | dt: 5953.15ms | tok/sec: 176138.1 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1965 |
+
[TIMING] data: 7.6ms | fwd: 1817.7ms | loss: 13.2ms | bwd: 4051.6ms | clip: 0.0ms | optim: 49.9ms
|
| 1966 |
+
step 911 | loss: 2.627549 | lr 4.7263e-04 | norm: 0.0944 | dt: 5960.51ms | tok/sec: 175920.4 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1967 |
+
[TIMING] data: 7.6ms | fwd: 1814.6ms | loss: 13.2ms | bwd: 4062.0ms | clip: 0.0ms | optim: 50.1ms
|
| 1968 |
+
step 912 | loss: 2.653412 | lr 4.6568e-04 | norm: 0.0951 | dt: 5946.17ms | tok/sec: 176344.8 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1969 |
+
[TIMING] data: 7.6ms | fwd: 1812.7ms | loss: 13.2ms | bwd: 4049.1ms | clip: 0.0ms | optim: 50.0ms
|
| 1970 |
+
step 913 | loss: 2.612473 | lr 4.5875e-04 | norm: 0.0918 | dt: 5949.24ms | tok/sec: 176253.9 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1971 |
+
[TIMING] data: 7.5ms | fwd: 1814.9ms | loss: 13.1ms | bwd: 4050.6ms | clip: 0.0ms | optim: 50.0ms
|
| 1972 |
+
step 914 | loss: 2.612973 | lr 4.5184e-04 | norm: 0.0933 | dt: 5956.10ms | tok/sec: 176050.7 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1973 |
+
[TIMING] data: 7.5ms | fwd: 1818.7ms | loss: 12.9ms | bwd: 4053.7ms | clip: 0.0ms | optim: 50.1ms
|
| 1974 |
+
step 915 | loss: 2.619504 | lr 4.4495e-04 | norm: 0.0981 | dt: 5951.57ms | tok/sec: 176184.7 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1975 |
+
[TIMING] data: 7.4ms | fwd: 1817.4ms | loss: 12.9ms | bwd: 4050.8ms | clip: 0.0ms | optim: 49.9ms
|
| 1976 |
+
step 916 | loss: 2.639683 | lr 4.3809e-04 | norm: 0.0944 | dt: 5951.54ms | tok/sec: 176185.8 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1977 |
+
[TIMING] data: 7.5ms | fwd: 1815.5ms | loss: 12.9ms | bwd: 4052.5ms | clip: 0.0ms | optim: 50.1ms
|
| 1978 |
+
step 917 | loss: 2.680348 | lr 4.3126e-04 | norm: 0.0939 | dt: 5956.32ms | tok/sec: 176044.1 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1979 |
+
[TIMING] data: 7.5ms | fwd: 1814.9ms | loss: 12.9ms | bwd: 4057.9ms | clip: 0.0ms | optim: 50.0ms
|
| 1980 |
+
step 918 | loss: 2.669458 | lr 4.2445e-04 | norm: 0.0978 | dt: 5951.98ms | tok/sec: 176172.6 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1981 |
+
[TIMING] data: 7.4ms | fwd: 1815.4ms | loss: 12.9ms | bwd: 4053.1ms | clip: 0.0ms | optim: 50.0ms
|
| 1982 |
+
step 919 | loss: 2.701620 | lr 4.1768e-04 | norm: 0.1002 | dt: 5955.54ms | tok/sec: 176067.3 | mem: 95437MB | Time Left: 00 hrs : 08 mins
|
| 1983 |
+
[TIMING] data: 7.4ms | fwd: 1814.2ms | loss: 12.9ms | bwd: 4058.0ms | clip: 0.0ms | optim: 50.0ms
|
| 1984 |
+
step 920 | loss: 2.726842 | lr 4.1094e-04 | norm: 0.0955 | dt: 5950.71ms | tok/sec: 176210.1 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 1985 |
+
[TIMING] data: 7.4ms | fwd: 1816.0ms | loss: 13.0ms | bwd: 4051.2ms | clip: 0.0ms | optim: 50.1ms
|
| 1986 |
+
step 921 | loss: 2.703319 | lr 4.0424e-04 | norm: 0.1031 | dt: 5952.19ms | tok/sec: 176166.3 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 1987 |
+
[TIMING] data: 7.5ms | fwd: 1815.0ms | loss: 12.9ms | bwd: 4053.8ms | clip: 0.0ms | optim: 50.0ms
|
| 1988 |
+
step 922 | loss: 2.682874 | lr 3.9757e-04 | norm: 0.0937 | dt: 5959.25ms | tok/sec: 175957.8 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 1989 |
+
[TIMING] data: 7.4ms | fwd: 1819.2ms | loss: 12.9ms | bwd: 4056.6ms | clip: 0.0ms | optim: 50.0ms
|
| 1990 |
+
step 923 | loss: 2.661068 | lr 3.9094e-04 | norm: 0.0949 | dt: 5950.91ms | tok/sec: 176204.2 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 1991 |
+
[TIMING] data: 7.4ms | fwd: 1821.3ms | loss: 12.9ms | bwd: 4046.2ms | clip: 0.0ms | optim: 50.0ms
|
| 1992 |
+
step 924 | loss: 2.570555 | lr 3.8434e-04 | norm: 0.0952 | dt: 5944.92ms | tok/sec: 176381.9 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 1993 |
+
[TIMING] data: 7.4ms | fwd: 1815.1ms | loss: 12.9ms | bwd: 4046.1ms | clip: 0.0ms | optim: 50.1ms
|
| 1994 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_925.ckpt (safely)
|
| 1995 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_875.ckpt
|
| 1996 |
+
step 925 | loss: 2.700989 | lr 3.7779e-04 | norm: 0.0926 | dt: 5963.47ms | tok/sec: 175833.3 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 1997 |
+
[TIMING] data: 7.3ms | fwd: 1797.2ms | loss: 13.0ms | bwd: 4018.6ms | clip: 0.0ms | optim: 49.9ms
|
| 1998 |
+
step 926 | loss: 2.650601 | lr 3.7128e-04 | norm: 0.0929 | dt: 5917.54ms | tok/sec: 177198.0 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 1999 |
+
[TIMING] data: 7.4ms | fwd: 1804.8ms | loss: 13.0ms | bwd: 4029.4ms | clip: 0.0ms | optim: 49.9ms
|
| 2000 |
+
step 927 | loss: 2.638551 | lr 3.6482e-04 | norm: 0.0902 | dt: 5919.45ms | tok/sec: 177140.7 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 2001 |
+
[TIMING] data: 7.4ms | fwd: 1810.3ms | loss: 12.9ms | bwd: 4025.8ms | clip: 0.0ms | optim: 49.9ms
|
| 2002 |
+
step 928 | loss: 2.647935 | lr 3.5840e-04 | norm: 0.0933 | dt: 5933.51ms | tok/sec: 176721.2 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 2003 |
+
[TIMING] data: 7.6ms | fwd: 1811.5ms | loss: 12.9ms | bwd: 4038.5ms | clip: 0.0ms | optim: 49.9ms
|
| 2004 |
+
step 929 | loss: 2.729273 | lr 3.5203e-04 | norm: 0.0898 | dt: 5933.37ms | tok/sec: 176725.3 | mem: 95437MB | Time Left: 00 hrs : 07 mins
|
| 2005 |
+
[TIMING] data: 7.6ms | fwd: 1810.3ms | loss: 13.0ms | bwd: 4039.3ms | clip: 0.0ms | optim: 50.0ms
|
| 2006 |
+
step 930 | loss: 2.663484 | lr 3.4570e-04 | norm: 0.1073 | dt: 5938.49ms | tok/sec: 176572.7 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2007 |
+
[TIMING] data: 7.5ms | fwd: 1812.1ms | loss: 12.9ms | bwd: 4042.7ms | clip: 0.0ms | optim: 50.0ms
|
| 2008 |
+
step 931 | loss: 2.610590 | lr 3.3943e-04 | norm: 0.0949 | dt: 5939.70ms | tok/sec: 176536.8 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2009 |
+
[TIMING] data: 7.5ms | fwd: 1816.0ms | loss: 12.9ms | bwd: 4040.3ms | clip: 0.0ms | optim: 49.9ms
|
| 2010 |
+
step 932 | loss: 2.578015 | lr 3.3321e-04 | norm: 0.0922 | dt: 5933.52ms | tok/sec: 176720.8 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2011 |
+
[TIMING] data: 7.4ms | fwd: 1813.6ms | loss: 12.9ms | bwd: 4036.7ms | clip: 0.0ms | optim: 49.9ms
|
| 2012 |
+
step 933 | loss: 2.631655 | lr 3.2704e-04 | norm: 0.0968 | dt: 5947.12ms | tok/sec: 176316.6 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2013 |
+
[TIMING] data: 7.5ms | fwd: 1816.7ms | loss: 12.9ms | bwd: 4047.0ms | clip: 0.0ms | optim: 50.0ms
|
| 2014 |
+
step 934 | loss: 2.691433 | lr 3.2093e-04 | norm: 0.0906 | dt: 5952.82ms | tok/sec: 176147.7 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2015 |
+
[TIMING] data: 7.4ms | fwd: 1814.4ms | loss: 12.9ms | bwd: 4055.2ms | clip: 0.0ms | optim: 49.9ms
|
| 2016 |
+
step 935 | loss: 2.621213 | lr 3.1488e-04 | norm: 0.0911 | dt: 5955.14ms | tok/sec: 176079.2 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2017 |
+
[TIMING] data: 7.5ms | fwd: 1813.4ms | loss: 13.0ms | bwd: 4058.3ms | clip: 0.0ms | optim: 50.0ms
|
| 2018 |
+
step 936 | loss: 2.640804 | lr 3.0888e-04 | norm: 0.0934 | dt: 5947.37ms | tok/sec: 176309.2 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2019 |
+
[TIMING] data: 7.4ms | fwd: 1813.7ms | loss: 12.9ms | bwd: 4050.5ms | clip: 0.0ms | optim: 50.0ms
|
| 2020 |
+
step 937 | loss: 2.636666 | lr 3.0294e-04 | norm: 0.0918 | dt: 5953.37ms | tok/sec: 176131.6 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2021 |
+
[TIMING] data: 7.4ms | fwd: 1817.3ms | loss: 12.9ms | bwd: 4052.8ms | clip: 0.0ms | optim: 49.9ms
|
| 2022 |
+
step 938 | loss: 2.627810 | lr 2.9706e-04 | norm: 0.0900 | dt: 5955.29ms | tok/sec: 176074.7 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2023 |
+
[TIMING] data: 7.4ms | fwd: 1813.2ms | loss: 12.9ms | bwd: 4058.7ms | clip: 0.0ms | optim: 50.1ms
|
| 2024 |
+
step 939 | loss: 2.699911 | lr 2.9125e-04 | norm: 0.0961 | dt: 5943.61ms | tok/sec: 176420.7 | mem: 95437MB | Time Left: 00 hrs : 06 mins
|
| 2025 |
+
[TIMING] data: 7.4ms | fwd: 1814.0ms | loss: 13.0ms | bwd: 4046.1ms | clip: 0.0ms | optim: 49.9ms
|
| 2026 |
+
step 940 | loss: 2.640586 | lr 2.8550e-04 | norm: 0.0911 | dt: 5956.48ms | tok/sec: 176039.5 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2027 |
+
[TIMING] data: 7.4ms | fwd: 1819.0ms | loss: 12.9ms | bwd: 4054.2ms | clip: 0.0ms | optim: 49.9ms
|
| 2028 |
+
step 941 | loss: 2.598037 | lr 2.7981e-04 | norm: 0.0868 | dt: 5956.92ms | tok/sec: 176026.6 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2029 |
+
[TIMING] data: 7.4ms | fwd: 1812.5ms | loss: 12.9ms | bwd: 4061.1ms | clip: 0.0ms | optim: 50.0ms
|
| 2030 |
+
step 942 | loss: 2.670021 | lr 2.7419e-04 | norm: 0.0879 | dt: 5958.25ms | tok/sec: 175987.3 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2031 |
+
[TIMING] data: 7.4ms | fwd: 1815.6ms | loss: 12.9ms | bwd: 4059.1ms | clip: 0.0ms | optim: 50.0ms
|
| 2032 |
+
step 943 | loss: 2.566868 | lr 2.6864e-04 | norm: 0.0913 | dt: 5954.66ms | tok/sec: 176093.3 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2033 |
+
[TIMING] data: 7.4ms | fwd: 1814.4ms | loss: 12.9ms | bwd: 4056.9ms | clip: 0.0ms | optim: 49.9ms
|
| 2034 |
+
step 944 | loss: 2.649738 | lr 2.6316e-04 | norm: 0.0970 | dt: 5959.55ms | tok/sec: 175948.9 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2035 |
+
[TIMING] data: 7.4ms | fwd: 1820.9ms | loss: 13.0ms | bwd: 4055.3ms | clip: 0.0ms | optim: 50.0ms
|
| 2036 |
+
step 945 | loss: 2.594111 | lr 2.5775e-04 | norm: 0.0909 | dt: 5961.93ms | tok/sec: 175878.6 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2037 |
+
[TIMING] data: 7.4ms | fwd: 1813.9ms | loss: 12.9ms | bwd: 4064.9ms | clip: 0.0ms | optim: 49.9ms
|
| 2038 |
+
step 946 | loss: 2.605718 | lr 2.5241e-04 | norm: 0.1452 | dt: 5959.44ms | tok/sec: 175952.2 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2039 |
+
[TIMING] data: 7.4ms | fwd: 1813.7ms | loss: 12.9ms | bwd: 4062.5ms | clip: 0.0ms | optim: 49.9ms
|
| 2040 |
+
step 947 | loss: 2.552562 | lr 2.4714e-04 | norm: 0.0892 | dt: 5957.18ms | tok/sec: 176018.9 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2041 |
+
[TIMING] data: 7.4ms | fwd: 1809.9ms | loss: 12.9ms | bwd: 4063.8ms | clip: 0.0ms | optim: 50.0ms
|
| 2042 |
+
step 948 | loss: 2.608181 | lr 2.4195e-04 | norm: 0.0871 | dt: 5953.17ms | tok/sec: 176137.5 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2043 |
+
[TIMING] data: 7.4ms | fwd: 1818.6ms | loss: 12.9ms | bwd: 4051.3ms | clip: 0.0ms | optim: 50.0ms
|
| 2044 |
+
step 949 | loss: 2.624990 | lr 2.3684e-04 | norm: 0.0898 | dt: 5956.42ms | tok/sec: 176041.3 | mem: 95437MB | Time Left: 00 hrs : 05 mins
|
| 2045 |
+
[TIMING] data: 7.4ms | fwd: 1817.7ms | loss: 13.0ms | bwd: 4055.4ms | clip: 0.0ms | optim: 50.0ms
|
| 2046 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_950.ckpt (safely)
|
| 2047 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_900.ckpt
|
| 2048 |
+
step 950 | loss: 2.645914 | lr 2.3180e-04 | norm: 0.0929 | dt: 5904.70ms | tok/sec: 177583.3 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2049 |
+
[TIMING] data: 7.4ms | fwd: 1800.6ms | loss: 12.9ms | bwd: 4020.8ms | clip: 0.0ms | optim: 50.0ms
|
| 2050 |
+
step 951 | loss: 2.558727 | lr 2.2684e-04 | norm: 0.0849 | dt: 5936.29ms | tok/sec: 176638.4 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2051 |
+
[TIMING] data: 7.3ms | fwd: 1808.0ms | loss: 12.9ms | bwd: 4045.1ms | clip: 0.0ms | optim: 49.9ms
|
| 2052 |
+
Current shard: /root/.cache/huggingface/hub/datasets--navingv--flux/snapshots/5232b3d133d2eb71905725f288ff6b56dc0df773/flux_train_000011.bin
|
| 2053 |
+
100000000
|
| 2054 |
+
step 952 | loss: 2.652134 | lr 2.2196e-04 | norm: 0.1023 | dt: 5935.33ms | tok/sec: 176667.0 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2055 |
+
[TIMING] data: 10.5ms | fwd: 1810.4ms | loss: 12.9ms | bwd: 4038.5ms | clip: 0.0ms | optim: 50.0ms
|
| 2056 |
+
step 953 | loss: 2.672426 | lr 2.1717e-04 | norm: 0.0958 | dt: 5936.77ms | tok/sec: 176623.9 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2057 |
+
[TIMING] data: 7.4ms | fwd: 1811.2ms | loss: 12.9ms | bwd: 4042.3ms | clip: 0.0ms | optim: 49.9ms
|
| 2058 |
+
step 954 | loss: 2.657829 | lr 2.1245e-04 | norm: 0.0941 | dt: 5947.32ms | tok/sec: 176310.7 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2059 |
+
[TIMING] data: 7.5ms | fwd: 1810.7ms | loss: 12.9ms | bwd: 4053.2ms | clip: 0.0ms | optim: 50.1ms
|
| 2060 |
+
step 955 | loss: 2.892116 | lr 2.0782e-04 | norm: 0.1836 | dt: 5939.09ms | tok/sec: 176555.0 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2061 |
+
[TIMING] data: 7.5ms | fwd: 1812.0ms | loss: 12.9ms | bwd: 4043.6ms | clip: 0.0ms | optim: 50.0ms
|
| 2062 |
+
step 956 | loss: 2.542175 | lr 2.0327e-04 | norm: 0.0981 | dt: 5940.34ms | tok/sec: 176518.0 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2063 |
+
[TIMING] data: 7.6ms | fwd: 1813.5ms | loss: 12.9ms | bwd: 4043.2ms | clip: 0.0ms | optim: 49.9ms
|
| 2064 |
+
step 957 | loss: 2.618965 | lr 1.9881e-04 | norm: 0.0906 | dt: 5958.59ms | tok/sec: 175977.1 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2065 |
+
[TIMING] data: 7.4ms | fwd: 1815.2ms | loss: 12.9ms | bwd: 4060.1ms | clip: 0.0ms | optim: 50.0ms
|
| 2066 |
+
step 958 | loss: 2.630756 | lr 1.9443e-04 | norm: 0.0891 | dt: 5946.62ms | tok/sec: 176331.4 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2067 |
+
[TIMING] data: 7.4ms | fwd: 1811.9ms | loss: 12.9ms | bwd: 4051.4ms | clip: 0.0ms | optim: 50.0ms
|
| 2068 |
+
step 959 | loss: 2.612083 | lr 1.9014e-04 | norm: 0.0862 | dt: 5947.68ms | tok/sec: 176300.0 | mem: 95437MB | Time Left: 00 hrs : 04 mins
|
| 2069 |
+
[TIMING] data: 7.4ms | fwd: 1813.7ms | loss: 12.9ms | bwd: 4050.6ms | clip: 0.0ms | optim: 50.0ms
|
| 2070 |
+
step 960 | loss: 2.655385 | lr 1.8594e-04 | norm: 0.0875 | dt: 5939.15ms | tok/sec: 176553.3 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2071 |
+
[TIMING] data: 7.5ms | fwd: 1812.0ms | loss: 12.9ms | bwd: 4043.6ms | clip: 0.0ms | optim: 50.1ms
|
| 2072 |
+
step 961 | loss: 2.651754 | lr 1.8183e-04 | norm: 0.0984 | dt: 5957.66ms | tok/sec: 176004.7 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2073 |
+
[TIMING] data: 7.4ms | fwd: 1815.0ms | loss: 12.9ms | bwd: 4059.3ms | clip: 0.0ms | optim: 50.0ms
|
| 2074 |
+
step 962 | loss: 2.620201 | lr 1.7781e-04 | norm: 0.0870 | dt: 5952.99ms | tok/sec: 176142.8 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2075 |
+
[TIMING] data: 7.4ms | fwd: 1816.4ms | loss: 12.9ms | bwd: 4053.3ms | clip: 0.0ms | optim: 49.9ms
|
| 2076 |
+
step 963 | loss: 2.628344 | lr 1.7389e-04 | norm: 0.0866 | dt: 5939.74ms | tok/sec: 176535.8 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2077 |
+
[TIMING] data: 7.4ms | fwd: 1816.7ms | loss: 12.9ms | bwd: 4039.8ms | clip: 0.0ms | optim: 49.8ms
|
| 2078 |
+
step 964 | loss: 2.542377 | lr 1.7005e-04 | norm: 0.0880 | dt: 5944.77ms | tok/sec: 176386.2 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2079 |
+
[TIMING] data: 7.5ms | fwd: 1812.8ms | loss: 12.9ms | bwd: 4048.5ms | clip: 0.0ms | optim: 50.0ms
|
| 2080 |
+
step 965 | loss: 2.614022 | lr 1.6631e-04 | norm: 0.0888 | dt: 5959.78ms | tok/sec: 175942.1 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2081 |
+
[TIMING] data: 7.5ms | fwd: 1814.6ms | loss: 12.9ms | bwd: 4061.8ms | clip: 0.0ms | optim: 49.9ms
|
| 2082 |
+
step 966 | loss: 2.584664 | lr 1.6267e-04 | norm: 0.0853 | dt: 5946.96ms | tok/sec: 176321.4 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2083 |
+
[TIMING] data: 7.5ms | fwd: 1816.7ms | loss: 12.9ms | bwd: 4046.9ms | clip: 0.0ms | optim: 49.9ms
|
| 2084 |
+
step 967 | loss: 2.542435 | lr 1.5912e-04 | norm: 0.0827 | dt: 5960.51ms | tok/sec: 175920.5 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2085 |
+
[TIMING] data: 7.5ms | fwd: 1815.6ms | loss: 12.9ms | bwd: 4061.5ms | clip: 0.0ms | optim: 50.0ms
|
| 2086 |
+
step 968 | loss: 2.577236 | lr 1.5566e-04 | norm: 0.0861 | dt: 5948.88ms | tok/sec: 176264.4 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2087 |
+
[TIMING] data: 7.5ms | fwd: 1816.1ms | loss: 12.9ms | bwd: 4049.4ms | clip: 0.0ms | optim: 50.0ms
|
| 2088 |
+
step 969 | loss: 2.535849 | lr 1.5231e-04 | norm: 0.0831 | dt: 5953.61ms | tok/sec: 176124.4 | mem: 95437MB | Time Left: 00 hrs : 03 mins
|
| 2089 |
+
[TIMING] data: 7.4ms | fwd: 1814.4ms | loss: 12.9ms | bwd: 4056.0ms | clip: 0.0ms | optim: 49.9ms
|
| 2090 |
+
step 970 | loss: 2.614408 | lr 1.4905e-04 | norm: 0.0874 | dt: 5947.18ms | tok/sec: 176314.9 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2091 |
+
[TIMING] data: 7.4ms | fwd: 1815.4ms | loss: 12.9ms | bwd: 4048.1ms | clip: 0.0ms | optim: 50.1ms
|
| 2092 |
+
step 971 | loss: 2.638258 | lr 1.4589e-04 | norm: 0.0837 | dt: 5967.20ms | tok/sec: 175723.2 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2093 |
+
[TIMING] data: 7.4ms | fwd: 1817.6ms | loss: 12.9ms | bwd: 4066.4ms | clip: 0.0ms | optim: 50.0ms
|
| 2094 |
+
step 972 | loss: 2.529380 | lr 1.4283e-04 | norm: 0.0813 | dt: 5941.32ms | tok/sec: 176488.8 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2095 |
+
[TIMING] data: 7.4ms | fwd: 1813.7ms | loss: 12.9ms | bwd: 4044.3ms | clip: 0.0ms | optim: 50.0ms
|
| 2096 |
+
step 973 | loss: 2.657314 | lr 1.3987e-04 | norm: 0.0884 | dt: 5965.05ms | tok/sec: 175786.5 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2097 |
+
[TIMING] data: 7.4ms | fwd: 1816.2ms | loss: 12.9ms | bwd: 4065.5ms | clip: 0.0ms | optim: 50.0ms
|
| 2098 |
+
step 974 | loss: 2.745756 | lr 1.3701e-04 | norm: 0.1004 | dt: 5947.38ms | tok/sec: 176308.9 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2099 |
+
[TIMING] data: 7.4ms | fwd: 1818.5ms | loss: 12.9ms | bwd: 4045.7ms | clip: 0.0ms | optim: 49.9ms
|
| 2100 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_975.ckpt (safely)
|
| 2101 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_925.ckpt
|
| 2102 |
+
step 975 | loss: 2.788818 | lr 1.3425e-04 | norm: 0.1041 | dt: 5906.06ms | tok/sec: 177542.5 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2103 |
+
[TIMING] data: 7.2ms | fwd: 1803.3ms | loss: 13.0ms | bwd: 4019.7ms | clip: 0.0ms | optim: 50.0ms
|
| 2104 |
+
step 976 | loss: 2.660657 | lr 1.3160e-04 | norm: 0.0925 | dt: 5921.58ms | tok/sec: 177077.0 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2105 |
+
[TIMING] data: 7.4ms | fwd: 1808.0ms | loss: 12.9ms | bwd: 4029.7ms | clip: 0.0ms | optim: 50.9ms
|
| 2106 |
+
step 977 | loss: 2.754989 | lr 1.2905e-04 | norm: 0.0980 | dt: 5926.21ms | tok/sec: 176938.6 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2107 |
+
[TIMING] data: 7.3ms | fwd: 1809.7ms | loss: 12.9ms | bwd: 4032.7ms | clip: 0.0ms | optim: 50.8ms
|
| 2108 |
+
step 978 | loss: 2.782961 | lr 1.2660e-04 | norm: 0.0882 | dt: 5938.60ms | tok/sec: 176569.4 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2109 |
+
[TIMING] data: 7.3ms | fwd: 1812.4ms | loss: 12.9ms | bwd: 4042.1ms | clip: 0.0ms | optim: 51.0ms
|
| 2110 |
+
step 979 | loss: 2.649026 | lr 1.2426e-04 | norm: 0.0877 | dt: 5934.15ms | tok/sec: 176701.9 | mem: 95437MB | Time Left: 00 hrs : 02 mins
|
| 2111 |
+
[TIMING] data: 7.3ms | fwd: 1814.3ms | loss: 12.9ms | bwd: 4036.1ms | clip: 0.0ms | optim: 50.8ms
|
| 2112 |
+
step 980 | loss: 2.794215 | lr 1.2202e-04 | norm: 0.0962 | dt: 5943.67ms | tok/sec: 176418.9 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2113 |
+
[TIMING] data: 7.3ms | fwd: 1813.8ms | loss: 12.9ms | bwd: 4046.6ms | clip: 0.0ms | optim: 50.1ms
|
| 2114 |
+
step 981 | loss: 2.708078 | lr 1.1989e-04 | norm: 0.0884 | dt: 5939.28ms | tok/sec: 176549.4 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2115 |
+
[TIMING] data: 7.4ms | fwd: 1813.9ms | loss: 12.9ms | bwd: 4041.5ms | clip: 0.0ms | optim: 50.7ms
|
| 2116 |
+
step 982 | loss: 2.647987 | lr 1.1787e-04 | norm: 0.0911 | dt: 5947.67ms | tok/sec: 176300.3 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2117 |
+
[TIMING] data: 7.3ms | fwd: 1809.5ms | loss: 12.9ms | bwd: 4054.4ms | clip: 0.0ms | optim: 50.8ms
|
| 2118 |
+
step 983 | loss: 2.673492 | lr 1.1595e-04 | norm: 0.0890 | dt: 5949.33ms | tok/sec: 176251.1 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2119 |
+
[TIMING] data: 7.3ms | fwd: 1812.5ms | loss: 12.9ms | bwd: 4052.8ms | clip: 0.0ms | optim: 51.1ms
|
| 2120 |
+
step 984 | loss: 2.737326 | lr 1.1414e-04 | norm: 0.0887 | dt: 5951.18ms | tok/sec: 176196.3 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2121 |
+
[TIMING] data: 7.3ms | fwd: 1817.7ms | loss: 12.9ms | bwd: 4050.5ms | clip: 0.0ms | optim: 50.0ms
|
| 2122 |
+
step 985 | loss: 2.697475 | lr 1.1243e-04 | norm: 0.0861 | dt: 5952.46ms | tok/sec: 176158.4 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2123 |
+
[TIMING] data: 7.3ms | fwd: 1814.4ms | loss: 12.9ms | bwd: 4055.2ms | clip: 0.0ms | optim: 49.9ms
|
| 2124 |
+
step 986 | loss: 2.673871 | lr 1.1084e-04 | norm: 0.0860 | dt: 5947.86ms | tok/sec: 176294.5 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2125 |
+
[TIMING] data: 7.3ms | fwd: 1816.9ms | loss: 12.9ms | bwd: 4047.9ms | clip: 0.0ms | optim: 50.0ms
|
| 2126 |
+
step 987 | loss: 2.765230 | lr 1.0935e-04 | norm: 0.0846 | dt: 5946.70ms | tok/sec: 176329.2 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2127 |
+
[TIMING] data: 7.3ms | fwd: 1817.8ms | loss: 12.9ms | bwd: 4046.0ms | clip: 0.0ms | optim: 49.9ms
|
| 2128 |
+
step 988 | loss: 2.672553 | lr 1.0797e-04 | norm: 0.0862 | dt: 5945.65ms | tok/sec: 176360.2 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2129 |
+
[TIMING] data: 7.4ms | fwd: 1813.2ms | loss: 12.9ms | bwd: 4049.5ms | clip: 0.0ms | optim: 49.9ms
|
| 2130 |
+
step 989 | loss: 2.674335 | lr 1.0670e-04 | norm: 0.0849 | dt: 5947.73ms | tok/sec: 176298.4 | mem: 95437MB | Time Left: 00 hrs : 01 mins
|
| 2131 |
+
[TIMING] data: 7.3ms | fwd: 1814.3ms | loss: 12.9ms | bwd: 4050.5ms | clip: 0.0ms | optim: 49.9ms
|
| 2132 |
+
step 990 | loss: 2.692748 | lr 1.0554e-04 | norm: 0.0867 | dt: 5951.44ms | tok/sec: 176188.7 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2133 |
+
[TIMING] data: 7.4ms | fwd: 1807.2ms | loss: 12.9ms | bwd: 4061.2ms | clip: 0.0ms | optim: 49.9ms
|
| 2134 |
+
step 991 | loss: 2.712992 | lr 1.0449e-04 | norm: 0.0839 | dt: 5953.31ms | tok/sec: 176133.4 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2135 |
+
[TIMING] data: 7.4ms | fwd: 1816.4ms | loss: 12.9ms | bwd: 4053.9ms | clip: 0.0ms | optim: 50.0ms
|
| 2136 |
+
step 992 | loss: 2.743290 | lr 1.0355e-04 | norm: 0.0857 | dt: 5957.27ms | tok/sec: 176016.2 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2137 |
+
[TIMING] data: 7.4ms | fwd: 1816.4ms | loss: 12.9ms | bwd: 4058.0ms | clip: 0.0ms | optim: 49.8ms
|
| 2138 |
+
step 993 | loss: 2.844353 | lr 1.0272e-04 | norm: 0.1009 | dt: 5949.89ms | tok/sec: 176234.7 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2139 |
+
[TIMING] data: 7.4ms | fwd: 1816.9ms | loss: 12.9ms | bwd: 4050.0ms | clip: 0.0ms | optim: 49.9ms
|
| 2140 |
+
step 994 | loss: 2.689448 | lr 1.0200e-04 | norm: 0.0823 | dt: 5939.13ms | tok/sec: 176553.9 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2141 |
+
[TIMING] data: 7.3ms | fwd: 1809.1ms | loss: 12.9ms | bwd: 4047.0ms | clip: 0.0ms | optim: 49.9ms
|
| 2142 |
+
step 995 | loss: 2.710656 | lr 1.0139e-04 | norm: 0.0859 | dt: 5951.06ms | tok/sec: 176199.8 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2143 |
+
[TIMING] data: 7.3ms | fwd: 1815.1ms | loss: 12.9ms | bwd: 4053.1ms | clip: 0.0ms | optim: 49.9ms
|
| 2144 |
+
step 996 | loss: 2.718275 | lr 1.0089e-04 | norm: 0.0891 | dt: 5950.01ms | tok/sec: 176231.0 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2145 |
+
[TIMING] data: 7.3ms | fwd: 1818.1ms | loss: 12.9ms | bwd: 4048.5ms | clip: 0.0ms | optim: 50.4ms
|
| 2146 |
+
step 997 | loss: 2.700864 | lr 1.0050e-04 | norm: 0.1134 | dt: 5952.79ms | tok/sec: 176148.7 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2147 |
+
[TIMING] data: 7.3ms | fwd: 1815.8ms | loss: 12.9ms | bwd: 4054.1ms | clip: 0.0ms | optim: 49.9ms
|
| 2148 |
+
step 998 | loss: 2.747742 | lr 1.0022e-04 | norm: 0.0857 | dt: 5945.29ms | tok/sec: 176371.0 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2149 |
+
[TIMING] data: 7.3ms | fwd: 1813.9ms | loss: 12.9ms | bwd: 4047.9ms | clip: 0.0ms | optim: 50.5ms
|
| 2150 |
+
validation loss: 2.8172
|
| 2151 |
+
step 999 | loss: 2.662259 | lr 1.0006e-04 | norm: 0.0920 | dt: 8284.76ms | tok/sec: 126566.9 | mem: 95437MB | Time Left: 00 hrs : 00 mins
|
| 2152 |
+
[TIMING] data: 7.3ms | fwd: 1815.8ms | loss: 12.9ms | bwd: 4065.7ms | clip: 0.0ms | optim: 50.0ms
|
| 2153 |
+
[CheckpointManager] Saved: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_1000.ckpt (safely)
|
| 2154 |
+
[CheckpointManager] Deleted old checkpoint: checkpoints_bluscriptcp/rank_0/blumodelcp_run8_step_950.ckpt
|
| 2155 |
+
|
| 2156 |
+
=== Training complete ===
|