OpenTransformer commited on
Commit
899e89d
·
1 Parent(s): 0e7e8bb

Update AGILLM4 training status

Browse files
training/agillm4_floor_v47_v4pro/status/status.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
- "checked_at": 1781002418.5726044,
3
- "checked_at_iso": "2026-06-09T10:53:38+00:00",
4
  "delta_checkpoint": {
5
  "name": "pretrain_delta_step01418505.pt",
6
  "raw_line": "[delta] saved pretrain_delta_step01418505.pt (5852126cbb73...)",
@@ -20,9 +20,9 @@
20
  "memory_total_mib": 24564,
21
  "memory_used_mib": 20532,
22
  "name": "NVIDIA GeForce RTX 4090",
23
- "power_draw_w": 342.43,
24
- "temperature_c": 62,
25
- "utilization_gpu": 94
26
  },
27
  "latest_full_checkpoint": {
28
  "checkpoint_exists": true,
@@ -40,13 +40,13 @@
40
  "full": []
41
  },
42
  "log": {
43
- "age_human": "0s",
44
- "age_seconds": 0.011,
45
  "exists": true,
46
- "mtime": 1781002418.5625825,
47
- "mtime_iso": "2026-06-09T10:53:38+00:00",
48
  "path": "/workspace/agillm41_master_train.log",
49
- "size_bytes": 4449749
50
  },
51
  "process": {
52
  "args": [
@@ -158,19 +158,19 @@
158
  "cmdline": "python -u agillm41.py train --preset agillm4_floor --resume /workspace/agillm4_4090_ckpts/pretrain_step01393506.pt --dblock --dblock_blocks 4 --dblock_schedule loss_balanced --dblock_warmup_steps 16 --dblock_sigma_curriculum_steps 2000 --dblock_log_every 25 --dblock_objective_mode stochastic --dblock_ar_prob 0.70 --dblock_sat_prob 0.15 --dblock_nat_prob 0.15 --dblock_ar_loss_tokens 512 --dblock_sat_loss_tokens 0 --dblock_nat_loss_tokens 512 --moe_ffn --moe_experts 2 --moe_top_k 1 --moe_mlp_mult 4 --tie_weights --batch_size 6 --block 1024 --amp --attn_backend sublinear --sublinear_window 128 --sublinear_stride 128 --sublinear_max_anchors 128 --sublinear_chunk 128 --sublinear_sinks 4 --sublinear_recent_anchors 64 --no-sublinear_pooled_landmarks --grad_checkpoint --dblock_checkpoint_stride 1 --optimizer paged_adamw8bit --sat_every 4 --nat_every 4 --nat_max_tokens 768 --nat_mask_ratio 0.5 --token_param_ratio 55 --save_dir /workspace/agillm4_4090_ckpts --save_every_sec 86400 --heartbeat_every_sec 300 --empty_cache_every_steps 0 --delta_every_steps 25000 --delta_max_keep 1 --max_ckpts 2 --async_update_dir /workspace/agillm41_side_updates/incoming --async_update_every_steps 100 --async_update_alpha 0.05 --async_update_max_per_check 2 --async_update_max_age_sec 86400 --async_update_accepted_dir /workspace/agillm41_side_updates/accepted --async_update_rejected_dir /workspace/agillm41_side_updates/rejected",
159
  "cwd": "/workspace/agillm41-mainline",
160
  "pid": 1389025,
161
- "uptime_human": "3d 4h 50m 59s",
162
- "uptime_seconds": 276659.87
163
  },
164
  "progress": {
165
  "batch": 6,
166
  "block": 1024,
167
- "elapsed": "3d4h50m",
168
- "eta": "189d23h9m",
169
- "loss": 6.013,
170
- "percent": 6.7659,
171
- "raw_line": "[6.7659%] 4,033,238,016/59,611,622,510 tok | 3386.25 tok/s | loss=6.013 B=6 L=1024 step=152460 eta=189d23h9m elapsed=3d4h50m",
172
- "seen_tokens": 4033238016,
173
- "step": 152460,
174
  "target_tokens": 59611622510,
175
  "tok_per_sec": 3386.25
176
  },
 
1
  {
2
+ "checked_at": 1781002483.5202007,
3
+ "checked_at_iso": "2026-06-09T10:54:43+00:00",
4
  "delta_checkpoint": {
5
  "name": "pretrain_delta_step01418505.pt",
6
  "raw_line": "[delta] saved pretrain_delta_step01418505.pt (5852126cbb73...)",
 
20
  "memory_total_mib": 24564,
21
  "memory_used_mib": 20532,
22
  "name": "NVIDIA GeForce RTX 4090",
23
+ "power_draw_w": 343.73,
24
+ "temperature_c": 63,
25
+ "utilization_gpu": 98
26
  },
27
  "latest_full_checkpoint": {
28
  "checkpoint_exists": true,
 
40
  "full": []
41
  },
42
  "log": {
43
+ "age_human": "3s",
44
+ "age_seconds": 3.609,
45
  "exists": true,
46
+ "mtime": 1781002479.912141,
47
+ "mtime_iso": "2026-06-09T10:54:39+00:00",
48
  "path": "/workspace/agillm41_master_train.log",
49
+ "size_bytes": 4450316
50
  },
51
  "process": {
52
  "args": [
 
158
  "cmdline": "python -u agillm41.py train --preset agillm4_floor --resume /workspace/agillm4_4090_ckpts/pretrain_step01393506.pt --dblock --dblock_blocks 4 --dblock_schedule loss_balanced --dblock_warmup_steps 16 --dblock_sigma_curriculum_steps 2000 --dblock_log_every 25 --dblock_objective_mode stochastic --dblock_ar_prob 0.70 --dblock_sat_prob 0.15 --dblock_nat_prob 0.15 --dblock_ar_loss_tokens 512 --dblock_sat_loss_tokens 0 --dblock_nat_loss_tokens 512 --moe_ffn --moe_experts 2 --moe_top_k 1 --moe_mlp_mult 4 --tie_weights --batch_size 6 --block 1024 --amp --attn_backend sublinear --sublinear_window 128 --sublinear_stride 128 --sublinear_max_anchors 128 --sublinear_chunk 128 --sublinear_sinks 4 --sublinear_recent_anchors 64 --no-sublinear_pooled_landmarks --grad_checkpoint --dblock_checkpoint_stride 1 --optimizer paged_adamw8bit --sat_every 4 --nat_every 4 --nat_max_tokens 768 --nat_mask_ratio 0.5 --token_param_ratio 55 --save_dir /workspace/agillm4_4090_ckpts --save_every_sec 86400 --heartbeat_every_sec 300 --empty_cache_every_steps 0 --delta_every_steps 25000 --delta_max_keep 1 --max_ckpts 2 --async_update_dir /workspace/agillm41_side_updates/incoming --async_update_every_steps 100 --async_update_alpha 0.05 --async_update_max_per_check 2 --async_update_max_age_sec 86400 --async_update_accepted_dir /workspace/agillm41_side_updates/accepted --async_update_rejected_dir /workspace/agillm41_side_updates/rejected",
159
  "cwd": "/workspace/agillm41-mainline",
160
  "pid": 1389025,
161
+ "uptime_human": "3d 4h 52m 4s",
162
+ "uptime_seconds": 276724.82
163
  },
164
  "progress": {
165
  "batch": 6,
166
  "block": 1024,
167
+ "elapsed": "3d4h51m",
168
+ "eta": "189d23h8m",
169
+ "loss": 6.487,
170
+ "percent": 6.7662,
171
+ "raw_line": "[6.7662%] 4,033,446,912/59,611,622,510 tok | 3386.25 tok/s | loss=6.487 B=6 L=1024 step=152494 eta=189d23h8m elapsed=3d4h51m",
172
+ "seen_tokens": 4033446912,
173
+ "step": 152494,
174
  "target_tokens": 59611622510,
175
  "tok_per_sec": 3386.25
176
  },