OpenTransformer commited on
Commit
2ccba19
·
1 Parent(s): dc4249a

Update AGILLM4 training log tail

Browse files
training/agillm4_floor_v47_v4pro/logs/train_tail.log CHANGED
@@ -1,7 +1,3 @@
1
- [dblock] step=89575 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.200 weighted=28.972 ar=28.972 sat=0.000 nat=0.000 raw_sum=7.200 edm_w=4.024 counts=[19233,15642,15715,38985] ema=[18.10,21.94,21.58,26.96] peak_alloc=16.11GB peak_reserved=18.20GB
2
- [6.1180%] 3,647,013,888/59,611,622,510 tok | 3398.56 tok/s | loss=7.670 B=6 L=1024 step=89598 eta=190d14h12m elapsed=1d20h59m
3
- [dblock] step=89600 block=1 obj=ar layers=[7, 8, 9, 10, 11, 12, 13] loss=5.380 weighted=26.898 ar=26.898 sat=0.000 nat=0.000 raw_sum=5.380 edm_w=5.000 counts=[19233,15645,15715,39007] ema=[18.10,23.16,21.58,28.60] peak_alloc=16.06GB peak_reserved=18.20GB
4
- [dblock] step=89625 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=4.553 weighted=18.335 ar=18.335 sat=0.000 nat=0.000 raw_sum=4.553 edm_w=4.027 counts=[19233,15645,15715,39032] ema=[18.10,23.16,21.58,26.92] peak_alloc=16.11GB peak_reserved=18.20GB
5
  [6.1183%] 3,647,216,640/59,611,622,510 tok | 3398.55 tok/s | loss=5.961 B=6 L=1024 step=89631 eta=190d14h12m elapsed=1d21h0m
6
  [dblock] step=89650 block=1 obj=ar layers=[7, 8, 9, 10, 11, 12, 13] loss=6.944 weighted=34.722 ar=34.722 sat=0.000 nat=0.000 raw_sum=6.944 edm_w=5.000 counts=[19233,15647,15715,39055] ema=[18.10,23.23,21.58,26.66] peak_alloc=16.06GB peak_reserved=18.20GB
7
  [6.1186%] 3,647,419,392/59,611,622,510 tok | 3398.53 tok/s | loss=8.824 B=6 L=1024 step=89664 eta=190d14h12m elapsed=1d21h1m
@@ -4998,3 +4994,7 @@ Retrying in 8s [Retry 5/5].
4998
  [dblock] step=152375 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=5.844 weighted=29.218 ar=29.218 sat=0.000 nat=0.000 raw_sum=5.844 edm_w=5.000 counts=[33977,29703,29984,58711] ema=[21.13,24.53,26.93,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4999
  [6.7652%] 4,032,826,368/59,611,622,510 tok | 3386.23 tok/s | loss=6.112 B=6 L=1024 step=152393 eta=189d23h12m elapsed=3d4h48m
5000
  [dblock] step=152400 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.268 weighted=31.338 ar=31.338 sat=0.000 nat=0.000 raw_sum=6.268 edm_w=5.000 counts=[33977,29703,30009,58711] ema=[21.13,24.53,27.51,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
 
 
 
 
 
 
 
 
 
1
  [6.1183%] 3,647,216,640/59,611,622,510 tok | 3398.55 tok/s | loss=5.961 B=6 L=1024 step=89631 eta=190d14h12m elapsed=1d21h0m
2
  [dblock] step=89650 block=1 obj=ar layers=[7, 8, 9, 10, 11, 12, 13] loss=6.944 weighted=34.722 ar=34.722 sat=0.000 nat=0.000 raw_sum=6.944 edm_w=5.000 counts=[19233,15647,15715,39055] ema=[18.10,23.23,21.58,26.66] peak_alloc=16.06GB peak_reserved=18.20GB
3
  [6.1186%] 3,647,419,392/59,611,622,510 tok | 3398.53 tok/s | loss=8.824 B=6 L=1024 step=89664 eta=190d14h12m elapsed=1d21h1m
 
4994
  [dblock] step=152375 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=5.844 weighted=29.218 ar=29.218 sat=0.000 nat=0.000 raw_sum=5.844 edm_w=5.000 counts=[33977,29703,29984,58711] ema=[21.13,24.53,26.93,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4995
  [6.7652%] 4,032,826,368/59,611,622,510 tok | 3386.23 tok/s | loss=6.112 B=6 L=1024 step=152393 eta=189d23h12m elapsed=3d4h48m
4996
  [dblock] step=152400 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.268 weighted=31.338 ar=31.338 sat=0.000 nat=0.000 raw_sum=6.268 edm_w=5.000 counts=[33977,29703,30009,58711] ema=[21.13,24.53,27.51,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4997
+ [dblock] step=152425 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.355 weighted=31.774 ar=31.774 sat=0.000 nat=0.000 raw_sum=6.355 edm_w=5.000 counts=[33977,29704,30033,58711] ema=[21.13,24.59,26.86,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4998
+ [6.7655%] 4,033,035,264/59,611,622,510 tok | 3386.25 tok/s | loss=6.281 B=6 L=1024 step=152427 eta=189d23h10m elapsed=3d4h49m
4999
+ [dblock] step=152450 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.066 weighted=30.332 ar=30.332 sat=0.000 nat=0.000 raw_sum=6.066 edm_w=5.000 counts=[33977,29708,30054,58711] ema=[21.13,23.75,26.29,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
5000
+ [6.7659%] 4,033,238,016/59,611,622,510 tok | 3386.25 tok/s | loss=6.013 B=6 L=1024 step=152460 eta=189d23h9m elapsed=3d4h50m