Upload folder using huggingface_hub
Browse files
action_decoder/chain/w2a_slow_so101_combined_lora32_iter2500_75m_2gpu_1s_chunk/stdout.log
CHANGED
|
@@ -93,3 +93,11 @@
|
|
| 93 |
[05-20 01:30:00|INFO|imaginaire/trainer.py:187:train] Starting training...
|
| 94 |
[05-20 01:30:00|INFO|cosmos_predict2/callbacks/device_monitor.py:92:on_train_start] DeviceMonitor callback: local_dir: checkpoints/vam/slow_so101/w2a_slow_so101_combined_lora32_iter2500_75m_2gpu_1s_chunk/DeviceMonitor
|
| 95 |
[05-20 01:30:03|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 0...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
[05-20 01:30:00|INFO|imaginaire/trainer.py:187:train] Starting training...
|
| 94 |
[05-20 01:30:00|INFO|cosmos_predict2/callbacks/device_monitor.py:92:on_train_start] DeviceMonitor callback: local_dir: checkpoints/vam/slow_so101/w2a_slow_so101_combined_lora32_iter2500_75m_2gpu_1s_chunk/DeviceMonitor
|
| 95 |
[05-20 01:30:03|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 0...
|
| 96 |
+
[05-20 01:30:44|INFO|imaginaire/trainer.py:193:train] Initial validation done.
|
| 97 |
+
[05-20 01:30:50|WARNING|imaginaire/utils/distributed.py:284:ddp_sync_grad] DDP static_graph=True is incompatible with sync_grad(). Performance will be reduced.
|
| 98 |
+
[05-20 01:31:25|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 1: Hit counter: 1/5 | Loss: 21.9094 | Time: 93.57s
|
| 99 |
+
[05-20 01:31:45|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 2: Hit counter: 2/5 | Loss: 28.8553 | Time: 19.53s
|
| 100 |
+
[05-20 01:32:04|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 3: Hit counter: 3/5 | Loss: 37.6186 | Time: 19.52s
|
| 101 |
+
[05-20 01:32:24|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 4: Hit counter: 4/5 | Loss: 36.8048 | Time: 19.59s
|
| 102 |
+
[05-20 01:32:44|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 5: Hit counter: 5/5 | Loss: 39.5356 | Time: 19.62s
|
| 103 |
+
[05-20 01:39:18|CRITICAL|imaginaire/callbacks/manual_gc.py:48:every_n_impl] Garbage collection disabled
|