OpenTransformer commited on
Commit
f07a335
·
1 Parent(s): 2f4ccb3

Update AGILLM4 training log tail

Browse files
training/agillm4_floor_v47_v4pro/logs/train_tail.log CHANGED
@@ -1,6 +1,3 @@
1
- [dblock] step=89400 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=6.944 weighted=30.270 ar=30.270 sat=0.000 nat=0.000 raw_sum=6.944 edm_w=4.359 counts=[19233,15638,15715,38814] ema=[18.10,21.30,21.58,22.63] peak_alloc=16.16GB peak_reserved=18.20GB
2
- [dblock] step=89425 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.830 weighted=31.660 ar=31.660 sat=0.000 nat=0.000 raw_sum=7.830 edm_w=4.044 counts=[19233,15638,15715,38839] ema=[18.10,21.30,21.58,27.27] peak_alloc=16.16GB peak_reserved=18.20GB
3
- [6.1162%] 3,645,975,552/59,611,622,510 tok | 3398.51 tok/s | loss=10.673 B=6 L=1024 step=89429 eta=190d14h21m elapsed=1d20h54m
4
  [dblock] step=89450 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.392 weighted=31.899 ar=31.899 sat=0.000 nat=0.000 raw_sum=7.392 edm_w=4.316 counts=[19233,15638,15715,38864] ema=[18.10,21.30,21.58,27.45] peak_alloc=16.16GB peak_reserved=18.20GB
5
  [6.1166%] 3,646,184,448/59,611,622,510 tok | 3398.53 tok/s | loss=5.928 B=6 L=1024 step=89463 eta=190d14h19m elapsed=1d20h55m
6
  [dblock] step=89475 block=3 obj=sat layers=[21, 22, 23, 24, 25, 26, 27] loss=7.835 weighted=32.800 ar=0.000 sat=32.800 nat=0.000 raw_sum=7.835 edm_w=4.187 counts=[19233,15639,15715,38888] ema=[18.10,21.66,21.58,27.18] peak_alloc=16.16GB peak_reserved=18.20GB
@@ -4998,3 +4995,6 @@ Retrying in 8s [Retry 5/5].
4998
  [dblock] step=152200 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=5.857 weighted=29.283 ar=29.283 sat=0.000 nat=0.000 raw_sum=5.857 edm_w=5.000 counts=[33977,29697,29815,58711] ema=[21.13,25.20,27.40,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4999
  [dblock] step=152225 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=5.918 weighted=29.591 ar=29.591 sat=0.000 nat=0.000 raw_sum=5.918 edm_w=5.000 counts=[33977,29697,29840,58711] ema=[21.13,25.20,26.48,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
5000
  [6.7634%] 4,031,794,176/59,611,622,510 tok | 3386.22 tok/s | loss=5.918 B=6 L=1024 step=152225 eta=189d23h18m elapsed=3d4h43m
 
 
 
 
 
 
 
1
  [dblock] step=89450 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.392 weighted=31.899 ar=31.899 sat=0.000 nat=0.000 raw_sum=7.392 edm_w=4.316 counts=[19233,15638,15715,38864] ema=[18.10,21.30,21.58,27.45] peak_alloc=16.16GB peak_reserved=18.20GB
2
  [6.1166%] 3,646,184,448/59,611,622,510 tok | 3398.53 tok/s | loss=5.928 B=6 L=1024 step=89463 eta=190d14h19m elapsed=1d20h55m
3
  [dblock] step=89475 block=3 obj=sat layers=[21, 22, 23, 24, 25, 26, 27] loss=7.835 weighted=32.800 ar=0.000 sat=32.800 nat=0.000 raw_sum=7.835 edm_w=4.187 counts=[19233,15639,15715,38888] ema=[18.10,21.66,21.58,27.18] peak_alloc=16.16GB peak_reserved=18.20GB
 
4995
  [dblock] step=152200 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=5.857 weighted=29.283 ar=29.283 sat=0.000 nat=0.000 raw_sum=5.857 edm_w=5.000 counts=[33977,29697,29815,58711] ema=[21.13,25.20,27.40,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4996
  [dblock] step=152225 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=5.918 weighted=29.591 ar=29.591 sat=0.000 nat=0.000 raw_sum=5.918 edm_w=5.000 counts=[33977,29697,29840,58711] ema=[21.13,25.20,26.48,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4997
  [6.7634%] 4,031,794,176/59,611,622,510 tok | 3386.22 tok/s | loss=5.918 B=6 L=1024 step=152225 eta=189d23h18m elapsed=3d4h43m
4998
+ [dblock] step=152250 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.591 weighted=32.956 ar=32.956 sat=0.000 nat=0.000 raw_sum=6.591 edm_w=5.000 counts=[33977,29697,29865,58711] ema=[21.13,25.20,28.11,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
4999
+ [6.7638%] 4,031,996,928/59,611,622,510 tok | 3386.21 tok/s | loss=6.534 B=6 L=1024 step=152258 eta=189d23h18m elapsed=3d4h44m
5000
+ [dblock] step=152275 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.099 weighted=30.493 ar=30.493 sat=0.000 nat=0.000 raw_sum=6.099 edm_w=5.000 counts=[33977,29698,29889,58711] ema=[21.13,25.42,26.56,24.63] peak_alloc=16.11GB peak_reserved=18.20GB