Commit ·
899e89d
1
Parent(s): 0e7e8bb
Update AGILLM4 training status
Browse files
training/agillm4_floor_v47_v4pro/status/status.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
-
"checked_at":
|
| 3 |
-
"checked_at_iso": "2026-06-09T10:
|
| 4 |
"delta_checkpoint": {
|
| 5 |
"name": "pretrain_delta_step01418505.pt",
|
| 6 |
"raw_line": "[delta] saved pretrain_delta_step01418505.pt (5852126cbb73...)",
|
|
@@ -20,9 +20,9 @@
|
|
| 20 |
"memory_total_mib": 24564,
|
| 21 |
"memory_used_mib": 20532,
|
| 22 |
"name": "NVIDIA GeForce RTX 4090",
|
| 23 |
-
"power_draw_w":
|
| 24 |
-
"temperature_c":
|
| 25 |
-
"utilization_gpu":
|
| 26 |
},
|
| 27 |
"latest_full_checkpoint": {
|
| 28 |
"checkpoint_exists": true,
|
|
@@ -40,13 +40,13 @@
|
|
| 40 |
"full": []
|
| 41 |
},
|
| 42 |
"log": {
|
| 43 |
-
"age_human": "
|
| 44 |
-
"age_seconds":
|
| 45 |
"exists": true,
|
| 46 |
-
"mtime":
|
| 47 |
-
"mtime_iso": "2026-06-09T10:
|
| 48 |
"path": "/workspace/agillm41_master_train.log",
|
| 49 |
-
"size_bytes":
|
| 50 |
},
|
| 51 |
"process": {
|
| 52 |
"args": [
|
|
@@ -158,19 +158,19 @@
|
|
| 158 |
"cmdline": "python -u agillm41.py train --preset agillm4_floor --resume /workspace/agillm4_4090_ckpts/pretrain_step01393506.pt --dblock --dblock_blocks 4 --dblock_schedule loss_balanced --dblock_warmup_steps 16 --dblock_sigma_curriculum_steps 2000 --dblock_log_every 25 --dblock_objective_mode stochastic --dblock_ar_prob 0.70 --dblock_sat_prob 0.15 --dblock_nat_prob 0.15 --dblock_ar_loss_tokens 512 --dblock_sat_loss_tokens 0 --dblock_nat_loss_tokens 512 --moe_ffn --moe_experts 2 --moe_top_k 1 --moe_mlp_mult 4 --tie_weights --batch_size 6 --block 1024 --amp --attn_backend sublinear --sublinear_window 128 --sublinear_stride 128 --sublinear_max_anchors 128 --sublinear_chunk 128 --sublinear_sinks 4 --sublinear_recent_anchors 64 --no-sublinear_pooled_landmarks --grad_checkpoint --dblock_checkpoint_stride 1 --optimizer paged_adamw8bit --sat_every 4 --nat_every 4 --nat_max_tokens 768 --nat_mask_ratio 0.5 --token_param_ratio 55 --save_dir /workspace/agillm4_4090_ckpts --save_every_sec 86400 --heartbeat_every_sec 300 --empty_cache_every_steps 0 --delta_every_steps 25000 --delta_max_keep 1 --max_ckpts 2 --async_update_dir /workspace/agillm41_side_updates/incoming --async_update_every_steps 100 --async_update_alpha 0.05 --async_update_max_per_check 2 --async_update_max_age_sec 86400 --async_update_accepted_dir /workspace/agillm41_side_updates/accepted --async_update_rejected_dir /workspace/agillm41_side_updates/rejected",
|
| 159 |
"cwd": "/workspace/agillm41-mainline",
|
| 160 |
"pid": 1389025,
|
| 161 |
-
"uptime_human": "3d 4h
|
| 162 |
-
"uptime_seconds":
|
| 163 |
},
|
| 164 |
"progress": {
|
| 165 |
"batch": 6,
|
| 166 |
"block": 1024,
|
| 167 |
-
"elapsed": "
|
| 168 |
-
"eta": "
|
| 169 |
-
"loss": 6.
|
| 170 |
-
"percent": 6.
|
| 171 |
-
"raw_line": "[6.
|
| 172 |
-
"seen_tokens":
|
| 173 |
-
"step":
|
| 174 |
"target_tokens": 59611622510,
|
| 175 |
"tok_per_sec": 3386.25
|
| 176 |
},
|
|
|
|
| 1 |
{
|
| 2 |
+
"checked_at": 1781002483.5202007,
|
| 3 |
+
"checked_at_iso": "2026-06-09T10:54:43+00:00",
|
| 4 |
"delta_checkpoint": {
|
| 5 |
"name": "pretrain_delta_step01418505.pt",
|
| 6 |
"raw_line": "[delta] saved pretrain_delta_step01418505.pt (5852126cbb73...)",
|
|
|
|
| 20 |
"memory_total_mib": 24564,
|
| 21 |
"memory_used_mib": 20532,
|
| 22 |
"name": "NVIDIA GeForce RTX 4090",
|
| 23 |
+
"power_draw_w": 343.73,
|
| 24 |
+
"temperature_c": 63,
|
| 25 |
+
"utilization_gpu": 98
|
| 26 |
},
|
| 27 |
"latest_full_checkpoint": {
|
| 28 |
"checkpoint_exists": true,
|
|
|
|
| 40 |
"full": []
|
| 41 |
},
|
| 42 |
"log": {
|
| 43 |
+
"age_human": "3s",
|
| 44 |
+
"age_seconds": 3.609,
|
| 45 |
"exists": true,
|
| 46 |
+
"mtime": 1781002479.912141,
|
| 47 |
+
"mtime_iso": "2026-06-09T10:54:39+00:00",
|
| 48 |
"path": "/workspace/agillm41_master_train.log",
|
| 49 |
+
"size_bytes": 4450316
|
| 50 |
},
|
| 51 |
"process": {
|
| 52 |
"args": [
|
|
|
|
| 158 |
"cmdline": "python -u agillm41.py train --preset agillm4_floor --resume /workspace/agillm4_4090_ckpts/pretrain_step01393506.pt --dblock --dblock_blocks 4 --dblock_schedule loss_balanced --dblock_warmup_steps 16 --dblock_sigma_curriculum_steps 2000 --dblock_log_every 25 --dblock_objective_mode stochastic --dblock_ar_prob 0.70 --dblock_sat_prob 0.15 --dblock_nat_prob 0.15 --dblock_ar_loss_tokens 512 --dblock_sat_loss_tokens 0 --dblock_nat_loss_tokens 512 --moe_ffn --moe_experts 2 --moe_top_k 1 --moe_mlp_mult 4 --tie_weights --batch_size 6 --block 1024 --amp --attn_backend sublinear --sublinear_window 128 --sublinear_stride 128 --sublinear_max_anchors 128 --sublinear_chunk 128 --sublinear_sinks 4 --sublinear_recent_anchors 64 --no-sublinear_pooled_landmarks --grad_checkpoint --dblock_checkpoint_stride 1 --optimizer paged_adamw8bit --sat_every 4 --nat_every 4 --nat_max_tokens 768 --nat_mask_ratio 0.5 --token_param_ratio 55 --save_dir /workspace/agillm4_4090_ckpts --save_every_sec 86400 --heartbeat_every_sec 300 --empty_cache_every_steps 0 --delta_every_steps 25000 --delta_max_keep 1 --max_ckpts 2 --async_update_dir /workspace/agillm41_side_updates/incoming --async_update_every_steps 100 --async_update_alpha 0.05 --async_update_max_per_check 2 --async_update_max_age_sec 86400 --async_update_accepted_dir /workspace/agillm41_side_updates/accepted --async_update_rejected_dir /workspace/agillm41_side_updates/rejected",
|
| 159 |
"cwd": "/workspace/agillm41-mainline",
|
| 160 |
"pid": 1389025,
|
| 161 |
+
"uptime_human": "3d 4h 52m 4s",
|
| 162 |
+
"uptime_seconds": 276724.82
|
| 163 |
},
|
| 164 |
"progress": {
|
| 165 |
"batch": 6,
|
| 166 |
"block": 1024,
|
| 167 |
+
"elapsed": "3d4h51m",
|
| 168 |
+
"eta": "189d23h8m",
|
| 169 |
+
"loss": 6.487,
|
| 170 |
+
"percent": 6.7662,
|
| 171 |
+
"raw_line": "[6.7662%] 4,033,446,912/59,611,622,510 tok | 3386.25 tok/s | loss=6.487 B=6 L=1024 step=152494 eta=189d23h8m elapsed=3d4h51m",
|
| 172 |
+
"seen_tokens": 4033446912,
|
| 173 |
+
"step": 152494,
|
| 174 |
"target_tokens": 59611622510,
|
| 175 |
"tok_per_sec": 3386.25
|
| 176 |
},
|