Upload folder using huggingface_hub
Browse files- action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt +1 -1
- action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/stdout.log +20 -0
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
iter_000001000.pt
|
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:909451a251cd8d5b652328f5188d9ab732dd0df6354d87698274d7bbb6352dae
|
| 3 |
+
size 161817668
|
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:394b930f162584fd95b17f8bcf9538ac96b328f09c8f9a306cf768aeccf2168a
|
| 3 |
+
size 647025631
|
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47a6bae9e822d2753aee6b0b62d6f98bb46ade9ec4e41da26490e0201d09c172
|
| 3 |
+
size 1602
|
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:42a8eb54f575f232d4f03d7c9594005cb06ae9ffb4bc249977d1154493cfbf75
|
| 3 |
+
size 892
|
action_decoder/chain/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/stdout.log
CHANGED
|
@@ -570,3 +570,23 @@
|
|
| 570 |
[05-20 16:57:24|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000000900.pt
|
| 571 |
[05-20 16:57:24|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
|
| 572 |
[05-20 16:57:24|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 570 |
[05-20 16:57:24|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000000900.pt
|
| 571 |
[05-20 16:57:24|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
|
| 572 |
[05-20 16:57:24|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
|
| 573 |
+
[05-20 17:30:45|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
|
| 574 |
+
Avg Max Min
|
| 575 |
+
cpu_mem_gb 78.291315 78.918194 77.664436
|
| 576 |
+
peak_gpu_mem_gb 18.602952 18.611254 18.594649
|
| 577 |
+
peak_gpu_mem_reserved_gb 28.088867 28.109375 28.068359
|
| 578 |
+
nvml_used_gpu_mem_gb 29.594238 29.614746 29.573730
|
| 579 |
+
nvml_free_gpu_mem_gb 50.053223 50.073730 50.032715
|
| 580 |
+
temp 68.000000 69.000000 67.000000
|
| 581 |
+
power 340003.000000 346189.000000 333817.000000
|
| 582 |
+
util 100.000000 100.000000 100.000000
|
| 583 |
+
clock 1642.500000 1695.000000 1590.000000
|
| 584 |
+
[05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt
|
| 585 |
+
[05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1509 seconds
|
| 586 |
+
[05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt
|
| 587 |
+
[05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5755 seconds
|
| 588 |
+
[05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt
|
| 589 |
+
[05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
|
| 590 |
+
[05-20 17:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task1_slow_100ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt
|
| 591 |
+
[05-20 17:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0011 seconds
|
| 592 |
+
[05-20 17:30:47|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1000...
|