Commit ·
cb8a8f6
1
Parent(s): f2dfb00
Update AGILLM4 training log tail
Browse files
training/agillm4_floor_v47_v4pro/logs/train_tail.log
CHANGED
|
@@ -1,6 +1,3 @@
|
|
| 1 |
-
[6.1169%] 3,646,393,344/59,611,622,510 tok | 3398.55 tok/s | loss=7.112 B=6 L=1024 step=89497 eta=190d14h16m elapsed=1d20h56m
|
| 2 |
-
[dblock] step=89500 block=3 obj=sat layers=[21, 22, 23, 24, 25, 26, 27] loss=9.557 weighted=38.692 ar=0.000 sat=38.692 nat=0.000 raw_sum=9.557 edm_w=4.049 counts=[19233,15639,15715,38913] ema=[18.10,21.66,21.58,25.15] peak_alloc=16.10GB peak_reserved=18.20GB
|
| 3 |
-
[dblock] step=89525 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=8.372 weighted=35.499 ar=35.499 sat=0.000 nat=0.000 raw_sum=8.372 edm_w=4.240 counts=[19233,15641,15715,38936] ema=[18.10,21.61,21.58,27.24] peak_alloc=16.16GB peak_reserved=18.20GB
|
| 4 |
[6.1173%] 3,646,602,240/59,611,622,510 tok | 3398.55 tok/s | loss=6.547 B=6 L=1024 step=89531 eta=190d14h15m elapsed=1d20h57m
|
| 5 |
[heartbeat] phase=pretrain pid=1389025 step=1483052 seen_tok=3646694400 loss=9.173 B=6 L=1024 dblock=True structured_masks=True gpu_alloc=4.91GB gpu_reserved=18.20GB gpu_peak=16.16GB
|
| 6 |
[dblock] step=89550 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.822 weighted=33.079 ar=33.079 sat=0.000 nat=0.000 raw_sum=7.822 edm_w=4.229 counts=[19233,15641,15715,38961] ema=[18.10,21.61,21.58,27.99] peak_alloc=16.16GB peak_reserved=18.20GB
|
|
@@ -4998,3 +4995,6 @@ Retrying in 8s [Retry 5/5].
|
|
| 4998 |
[6.7641%] 4,032,205,824/59,611,622,510 tok | 3386.23 tok/s | loss=6.056 B=6 L=1024 step=152292 eta=189d23h16m elapsed=3d4h45m
|
| 4999 |
[dblock] step=152300 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.493 weighted=32.466 ar=32.466 sat=0.000 nat=0.000 raw_sum=6.493 edm_w=5.000 counts=[33977,29700,29912,58711] ema=[21.13,25.77,26.90,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
|
| 5000 |
[heartbeat] phase=pretrain pid=1389025 step=1545820 seen_tok=4032340992 loss=6.280 B=6 L=1024 dblock=True structured_masks=True gpu_alloc=4.91GB gpu_reserved=18.20GB gpu_peak=16.11GB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
[6.1173%] 3,646,602,240/59,611,622,510 tok | 3398.55 tok/s | loss=6.547 B=6 L=1024 step=89531 eta=190d14h15m elapsed=1d20h57m
|
| 2 |
[heartbeat] phase=pretrain pid=1389025 step=1483052 seen_tok=3646694400 loss=9.173 B=6 L=1024 dblock=True structured_masks=True gpu_alloc=4.91GB gpu_reserved=18.20GB gpu_peak=16.16GB
|
| 3 |
[dblock] step=89550 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.822 weighted=33.079 ar=33.079 sat=0.000 nat=0.000 raw_sum=7.822 edm_w=4.229 counts=[19233,15641,15715,38961] ema=[18.10,21.61,21.58,27.99] peak_alloc=16.16GB peak_reserved=18.20GB
|
|
|
|
| 4995 |
[6.7641%] 4,032,205,824/59,611,622,510 tok | 3386.23 tok/s | loss=6.056 B=6 L=1024 step=152292 eta=189d23h16m elapsed=3d4h45m
|
| 4996 |
[dblock] step=152300 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.493 weighted=32.466 ar=32.466 sat=0.000 nat=0.000 raw_sum=6.493 edm_w=5.000 counts=[33977,29700,29912,58711] ema=[21.13,25.77,26.90,24.63] peak_alloc=16.11GB peak_reserved=18.20GB
|
| 4997 |
[heartbeat] phase=pretrain pid=1389025 step=1545820 seen_tok=4032340992 loss=6.280 B=6 L=1024 dblock=True structured_masks=True gpu_alloc=4.91GB gpu_reserved=18.20GB gpu_peak=16.11GB
|
| 4998 |
+
[dblock] step=152325 block=2 obj=nat layers=[14, 15, 16, 17, 18, 19, 20] loss=6.478 weighted=6.478 ar=0.000 sat=0.000 nat=6.478 raw_sum=6.478 edm_w=5.000 counts=[33977,29700,29937,58711] ema=[21.13,25.77,26.98,24.63] peak_alloc=13.95GB peak_reserved=18.20GB
|
| 4999 |
+
[6.7645%] 4,032,408,576/59,611,622,510 tok | 3386.23 tok/s | loss=6.478 B=6 L=1024 step=152325 eta=189d23h15m elapsed=3d4h46m
|
| 5000 |
+
[dblock] step=152350 block=2 obj=ar layers=[14, 15, 16, 17, 18, 19, 20] loss=6.448 weighted=32.240 ar=32.240 sat=0.000 nat=0.000 raw_sum=6.448 edm_w=5.000 counts=[33977,29701,29961,58711] ema=[21.13,25.01,27.70,24.63] peak_alloc=16.16GB peak_reserved=18.20GB
|