OpenTransformer commited on
Commit
64ca92a
·
1 Parent(s): c3e7126

Update AGILLM4 training status

Browse files
training/agillm4_floor_v47_v4pro/status/status.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
- "checked_at": 1781002548.5987675,
3
- "checked_at_iso": "2026-06-09T10:55:48+00:00",
4
  "delta_checkpoint": {
5
  "name": "pretrain_delta_step01418505.pt",
6
  "raw_line": "[delta] saved pretrain_delta_step01418505.pt (5852126cbb73...)",
@@ -20,7 +20,7 @@
20
  "memory_total_mib": 24564,
21
  "memory_used_mib": 20532,
22
  "name": "NVIDIA GeForce RTX 4090",
23
- "power_draw_w": 349.1,
24
  "temperature_c": 64,
25
  "utilization_gpu": 98
26
  },
@@ -40,13 +40,13 @@
40
  "full": []
41
  },
42
  "log": {
43
- "age_human": "8s",
44
- "age_seconds": 8.642,
45
  "exists": true,
46
- "mtime": 1781002539.9576654,
47
- "mtime_iso": "2026-06-09T10:55:39+00:00",
48
  "path": "/workspace/agillm41_master_train.log",
49
- "size_bytes": 4450959
50
  },
51
  "process": {
52
  "args": [
@@ -158,21 +158,21 @@
158
  "cmdline": "python -u agillm41.py train --preset agillm4_floor --resume /workspace/agillm4_4090_ckpts/pretrain_step01393506.pt --dblock --dblock_blocks 4 --dblock_schedule loss_balanced --dblock_warmup_steps 16 --dblock_sigma_curriculum_steps 2000 --dblock_log_every 25 --dblock_objective_mode stochastic --dblock_ar_prob 0.70 --dblock_sat_prob 0.15 --dblock_nat_prob 0.15 --dblock_ar_loss_tokens 512 --dblock_sat_loss_tokens 0 --dblock_nat_loss_tokens 512 --moe_ffn --moe_experts 2 --moe_top_k 1 --moe_mlp_mult 4 --tie_weights --batch_size 6 --block 1024 --amp --attn_backend sublinear --sublinear_window 128 --sublinear_stride 128 --sublinear_max_anchors 128 --sublinear_chunk 128 --sublinear_sinks 4 --sublinear_recent_anchors 64 --no-sublinear_pooled_landmarks --grad_checkpoint --dblock_checkpoint_stride 1 --optimizer paged_adamw8bit --sat_every 4 --nat_every 4 --nat_max_tokens 768 --nat_mask_ratio 0.5 --token_param_ratio 55 --save_dir /workspace/agillm4_4090_ckpts --save_every_sec 86400 --heartbeat_every_sec 300 --empty_cache_every_steps 0 --delta_every_steps 25000 --delta_max_keep 1 --max_ckpts 2 --async_update_dir /workspace/agillm41_side_updates/incoming --async_update_every_steps 100 --async_update_alpha 0.05 --async_update_max_per_check 2 --async_update_max_age_sec 86400 --async_update_accepted_dir /workspace/agillm41_side_updates/accepted --async_update_rejected_dir /workspace/agillm41_side_updates/rejected",
159
  "cwd": "/workspace/agillm41-mainline",
160
  "pid": 1389025,
161
- "uptime_human": "3d 4h 53m 9s",
162
- "uptime_seconds": 276789.9
163
  },
164
  "progress": {
165
  "batch": 6,
166
  "block": 1024,
167
- "elapsed": "3d4h52m",
168
- "eta": "189d23h7m",
169
- "loss": 7.226,
170
- "percent": 6.7665,
171
- "raw_line": "[6.7665%] 4,033,649,664/59,611,622,510 tok | 3386.25 tok/s | loss=7.226 B=6 L=1024 step=152527 eta=189d23h7m elapsed=3d4h52m",
172
- "seen_tokens": 4033649664,
173
- "step": 152527,
174
  "target_tokens": 59611622510,
175
- "tok_per_sec": 3386.25
176
  },
177
  "running": true,
178
  "save_dir": {
 
1
  {
2
+ "checked_at": 1781002613.987012,
3
+ "checked_at_iso": "2026-06-09T10:56:53+00:00",
4
  "delta_checkpoint": {
5
  "name": "pretrain_delta_step01418505.pt",
6
  "raw_line": "[delta] saved pretrain_delta_step01418505.pt (5852126cbb73...)",
 
20
  "memory_total_mib": 24564,
21
  "memory_used_mib": 20532,
22
  "name": "NVIDIA GeForce RTX 4090",
23
+ "power_draw_w": 343.6,
24
  "temperature_c": 64,
25
  "utilization_gpu": 98
26
  },
 
40
  "full": []
41
  },
42
  "log": {
43
+ "age_human": "13s",
44
+ "age_seconds": 13.611,
45
  "exists": true,
46
+ "mtime": 1781002600.3768916,
47
+ "mtime_iso": "2026-06-09T10:56:40+00:00",
48
  "path": "/workspace/agillm41_master_train.log",
49
+ "size_bytes": 4451343
50
  },
51
  "process": {
52
  "args": [
 
158
  "cmdline": "python -u agillm41.py train --preset agillm4_floor --resume /workspace/agillm4_4090_ckpts/pretrain_step01393506.pt --dblock --dblock_blocks 4 --dblock_schedule loss_balanced --dblock_warmup_steps 16 --dblock_sigma_curriculum_steps 2000 --dblock_log_every 25 --dblock_objective_mode stochastic --dblock_ar_prob 0.70 --dblock_sat_prob 0.15 --dblock_nat_prob 0.15 --dblock_ar_loss_tokens 512 --dblock_sat_loss_tokens 0 --dblock_nat_loss_tokens 512 --moe_ffn --moe_experts 2 --moe_top_k 1 --moe_mlp_mult 4 --tie_weights --batch_size 6 --block 1024 --amp --attn_backend sublinear --sublinear_window 128 --sublinear_stride 128 --sublinear_max_anchors 128 --sublinear_chunk 128 --sublinear_sinks 4 --sublinear_recent_anchors 64 --no-sublinear_pooled_landmarks --grad_checkpoint --dblock_checkpoint_stride 1 --optimizer paged_adamw8bit --sat_every 4 --nat_every 4 --nat_max_tokens 768 --nat_mask_ratio 0.5 --token_param_ratio 55 --save_dir /workspace/agillm4_4090_ckpts --save_every_sec 86400 --heartbeat_every_sec 300 --empty_cache_every_steps 0 --delta_every_steps 25000 --delta_max_keep 1 --max_ckpts 2 --async_update_dir /workspace/agillm41_side_updates/incoming --async_update_every_steps 100 --async_update_alpha 0.05 --async_update_max_per_check 2 --async_update_max_age_sec 86400 --async_update_accepted_dir /workspace/agillm41_side_updates/accepted --async_update_rejected_dir /workspace/agillm41_side_updates/rejected",
159
  "cwd": "/workspace/agillm41-mainline",
160
  "pid": 1389025,
161
+ "uptime_human": "3d 4h 54m 15s",
162
+ "uptime_seconds": 276855.29
163
  },
164
  "progress": {
165
  "batch": 6,
166
  "block": 1024,
167
+ "elapsed": "3d4h53m",
168
+ "eta": "189d23h6m",
169
+ "loss": 6.263,
170
+ "percent": 6.7669,
171
+ "raw_line": "[6.7669%] 4,033,852,416/59,611,622,510 tok | 3386.24 tok/s | loss=6.263 B=6 L=1024 step=152560 eta=189d23h6m elapsed=3d4h53m",
172
+ "seen_tokens": 4033852416,
173
+ "step": 152560,
174
  "target_tokens": 59611622510,
175
+ "tok_per_sec": 3386.24
176
  },
177
  "running": true,
178
  "save_dir": {