Upload folder using huggingface_hub
Browse files- action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt +1 -1
- action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt +3 -0
- action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/stdout.log +20 -0
action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
iter_000001000.pt
|
action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:affc918f081420215fbb0ef7c3e74cd57392ed7e5f7a9737abd67b9df3f38b13
|
| 3 |
+
size 161817668
|
action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6ed12416b2402440ecc54029bc376e0cc2277c09ee589fc16f8dce147d338a6f
|
| 3 |
+
size 647025631
|
action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47a6bae9e822d2753aee6b0b62d6f98bb46ade9ec4e41da26490e0201d09c172
|
| 3 |
+
size 1602
|
action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:42a8eb54f575f232d4f03d7c9594005cb06ae9ffb4bc249977d1154493cfbf75
|
| 3 |
+
size 892
|
action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/stdout.log
CHANGED
|
@@ -182,3 +182,23 @@
|
|
| 182 |
[05-20 23:11:04|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000000900.pt
|
| 183 |
[05-20 23:11:04|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0032 seconds
|
| 184 |
[05-20 23:11:04|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
[05-20 23:11:04|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000000900.pt
|
| 183 |
[05-20 23:11:04|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0032 seconds
|
| 184 |
[05-20 23:11:04|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
|
| 185 |
+
[05-20 23:41:04|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
|
| 186 |
+
Avg Max Min
|
| 187 |
+
cpu_mem_gb 66.989233 67.019054 66.959412
|
| 188 |
+
peak_gpu_mem_gb 18.604917 18.609647 18.600186
|
| 189 |
+
peak_gpu_mem_reserved_gb 28.161133 28.279297 28.042969
|
| 190 |
+
nvml_used_gpu_mem_gb 30.919250 31.037415 30.801086
|
| 191 |
+
nvml_free_gpu_mem_gb 62.664734 62.782898 62.546570
|
| 192 |
+
temp 74.500000 76.000000 73.000000
|
| 193 |
+
power 385577.500000 390633.000000 380522.000000
|
| 194 |
+
util 93.000000 100.000000 86.000000
|
| 195 |
+
clock 1687.500000 1740.000000 1635.000000
|
| 196 |
+
[05-20 23:41:04|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt
|
| 197 |
+
[05-20 23:41:04|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1226 seconds
|
| 198 |
+
[05-20 23:41:05|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt
|
| 199 |
+
[05-20 23:41:05|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6005 seconds
|
| 200 |
+
[05-20 23:41:05|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt
|
| 201 |
+
[05-20 23:41:05|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
|
| 202 |
+
[05-20 23:41:05|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt
|
| 203 |
+
[05-20 23:41:05|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0023 seconds
|
| 204 |
+
[05-20 23:41:05|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1000...
|