Commit ·
a72006f
1
Parent(s): bb1cb2a
Update AGILLM4 training log tail
Browse files
training/agillm4_floor_v47_v4pro/logs/train_tail.log
CHANGED
|
@@ -1,5 +1,3 @@
|
|
| 1 |
-
[6.1207%] 3,648,672,768/59,611,622,510 tok | 3398.60 tok/s | loss=6.913 B=6 L=1024 step=89868 eta=190d14h1m elapsed=1d21h7m
|
| 2 |
-
[dblock] step=89875 block=3 obj=sat layers=[21, 22, 23, 24, 25, 26, 27] loss=8.314 weighted=34.573 ar=0.000 sat=34.573 nat=0.000 raw_sum=8.314 edm_w=4.158 counts=[19233,15663,15715,39264] ema=[18.10,20.75,21.58,29.02] peak_alloc=16.11GB peak_reserved=18.20GB
|
| 3 |
[heartbeat] phase=pretrain pid=1389025 step=1483387 seen_tok=3648752640 loss=6.060 B=6 L=1024 dblock=True structured_masks=True gpu_alloc=4.91GB gpu_reserved=18.20GB gpu_peak=16.16GB
|
| 4 |
[dblock] step=89900 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=9.841 weighted=40.285 ar=40.285 sat=0.000 nat=0.000 raw_sum=9.841 edm_w=4.093 counts=[19233,15664,15715,39288] ema=[18.10,21.15,21.58,29.50] peak_alloc=16.11GB peak_reserved=18.20GB
|
| 5 |
[6.1211%] 3,648,881,664/59,611,622,510 tok | 3398.61 tok/s | loss=10.046 B=6 L=1024 step=89902 eta=190d13h59m elapsed=1d21h8m
|
|
@@ -4998,3 +4996,5 @@ Retrying in 8s [Retry 5/5].
|
|
| 4998 |
[dblock] step=152650 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.834 weighted=31.459 ar=31.459 sat=0.000 nat=0.000 raw_sum=7.834 edm_w=4.016 counts=[33977,29718,30231,58724] ema=[21.13,24.88,24.41,27.91] peak_alloc=16.01GB peak_reserved=18.20GB
|
| 4999 |
[6.7679%] 4,034,472,960/59,611,622,510 tok | 3386.26 tok/s | loss=6.571 B=6 L=1024 step=152661 eta=189d23h2m elapsed=3d4h56m
|
| 5000 |
[dblock] step=152675 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.901 weighted=33.047 ar=33.047 sat=0.000 nat=0.000 raw_sum=7.901 edm_w=4.183 counts=[33977,29719,30231,58748] ema=[21.13,24.95,24.41,28.39] peak_alloc=16.06GB peak_reserved=18.20GB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
[heartbeat] phase=pretrain pid=1389025 step=1483387 seen_tok=3648752640 loss=6.060 B=6 L=1024 dblock=True structured_masks=True gpu_alloc=4.91GB gpu_reserved=18.20GB gpu_peak=16.16GB
|
| 2 |
[dblock] step=89900 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=9.841 weighted=40.285 ar=40.285 sat=0.000 nat=0.000 raw_sum=9.841 edm_w=4.093 counts=[19233,15664,15715,39288] ema=[18.10,21.15,21.58,29.50] peak_alloc=16.11GB peak_reserved=18.20GB
|
| 3 |
[6.1211%] 3,648,881,664/59,611,622,510 tok | 3398.61 tok/s | loss=10.046 B=6 L=1024 step=89902 eta=190d13h59m elapsed=1d21h8m
|
|
|
|
| 4996 |
[dblock] step=152650 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.834 weighted=31.459 ar=31.459 sat=0.000 nat=0.000 raw_sum=7.834 edm_w=4.016 counts=[33977,29718,30231,58724] ema=[21.13,24.88,24.41,27.91] peak_alloc=16.01GB peak_reserved=18.20GB
|
| 4997 |
[6.7679%] 4,034,472,960/59,611,622,510 tok | 3386.26 tok/s | loss=6.571 B=6 L=1024 step=152661 eta=189d23h2m elapsed=3d4h56m
|
| 4998 |
[dblock] step=152675 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.901 weighted=33.047 ar=33.047 sat=0.000 nat=0.000 raw_sum=7.901 edm_w=4.183 counts=[33977,29719,30231,58748] ema=[21.13,24.95,24.41,28.39] peak_alloc=16.06GB peak_reserved=18.20GB
|
| 4999 |
+
[6.7683%] 4,034,681,856/59,611,622,510 tok | 3386.26 tok/s | loss=7.684 B=6 L=1024 step=152695 eta=189d23h1m elapsed=3d4h57m
|
| 5000 |
+
[dblock] step=152700 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.737 weighted=32.306 ar=32.306 sat=0.000 nat=0.000 raw_sum=7.737 edm_w=4.176 counts=[33977,29721,30231,58771] ema=[21.13,24.44,24.41,29.96] peak_alloc=16.01GB peak_reserved=18.20GB
|