JessieLoki commited on
Commit
02ac035
·
verified ·
1 Parent(s): 5789387

Upload folder using huggingface_hub

Browse files
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt CHANGED
@@ -1 +1 @@
1
- iter_000000900.pt
 
1
+ iter_000001000.pt
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:909451a251cd8d5b652328f5188d9ab732dd0df6354d87698274d7bbb6352dae
3
+ size 161817668
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:394b930f162584fd95b17f8bcf9538ac96b328f09c8f9a306cf768aeccf2168a
3
+ size 647025631
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:47a6bae9e822d2753aee6b0b62d6f98bb46ade9ec4e41da26490e0201d09c172
3
+ size 1602
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42a8eb54f575f232d4f03d7c9594005cb06ae9ffb4bc249977d1154493cfbf75
3
+ size 892
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/stdout.log CHANGED
@@ -570,3 +570,23 @@
570
  [05-20 16:57:24|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000000900.pt
571
  [05-20 16:57:24|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
572
  [05-20 16:57:24|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
570
  [05-20 16:57:24|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000000900.pt
571
  [05-20 16:57:24|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
572
  [05-20 16:57:24|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
573
+ [05-20 17:30:45|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
574
+ Avg Max Min
575
+ cpu_mem_gb 78.291315 78.918194 77.664436
576
+ peak_gpu_mem_gb 18.602952 18.611254 18.594649
577
+ peak_gpu_mem_reserved_gb 28.088867 28.109375 28.068359
578
+ nvml_used_gpu_mem_gb 29.594238 29.614746 29.573730
579
+ nvml_free_gpu_mem_gb 50.053223 50.073730 50.032715
580
+ temp 68.000000 69.000000 67.000000
581
+ power 340003.000000 346189.000000 333817.000000
582
+ util 100.000000 100.000000 100.000000
583
+ clock 1642.500000 1695.000000 1590.000000
584
+ [05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt
585
+ [05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1509 seconds
586
+ [05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt
587
+ [05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5755 seconds
588
+ [05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt
589
+ [05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
590
+ [05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt
591
+ [05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0011 seconds
592
+ [05-20 17:30:47|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1000...