Upload folder using huggingface_hub
Browse files- action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt +1 -1
- action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002400.pt +3 -0
- action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002400.pt +3 -0
- action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000002400.pt +0 -0
- action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002400.pt +0 -0
- action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/stdout.log +97 -0
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
iter_000002400.pt
|
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002400.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6ce70b842d2ab47c5a1901cb2a5dd73d5a66649bcb16bb737bd71893c1512e6f
|
| 3 |
+
size 161817668
|
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002400.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47b953120827902385ba93044b20e186319e33c37bddfb9f2c680b3cb4988b72
|
| 3 |
+
size 204902400
|
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000002400.pt
ADDED
|
File without changes
|
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002400.pt
ADDED
|
File without changes
|
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/stdout.log
CHANGED
|
@@ -331,3 +331,100 @@ clock 1455.000000 1500.000000 1410.000000
|
|
| 331 |
[05-20 15:57:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002300.pt
|
| 332 |
[05-20 15:57:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0008 seconds
|
| 333 |
[05-20 15:57:13|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2300...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
[05-20 15:57:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002300.pt
|
| 332 |
[05-20 15:57:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0008 seconds
|
| 333 |
[05-20 15:57:13|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2300...
|
| 334 |
+
[05-20 16:30:57|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002400.pt
|
| 335 |
+
[05-20 16:30:57|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1518 seconds
|
| 336 |
+
[05-20 16:30:59|ERROR|cosmos_predict2/checkpointer.py:143:_save_worker_local] Checkpoint failed to save (local): [enforce fail at inline_container.cc:626] . unexpected pos 201024832 vs 201024720
|
| 337 |
+
Traceback (most recent call last):
|
| 338 |
+
|
| 339 |
+
File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 944, in save
|
| 340 |
+
_save(
|
| 341 |
+
β <function _save at 0x70440e394430>
|
| 342 |
+
File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 1216, in _save
|
| 343 |
+
zip_file.write_record(name, storage, num_bytes)
|
| 344 |
+
β β β β β 4194304
|
| 345 |
+
β β β β 97
|
| 346 |
+
β β β 253
|
| 347 |
+
β β β 17
|
| 348 |
+
β β β 52
|
| 349 |
+
β β β 168
|
| 350 |
+
β β β 176
|
| 351 |
+
β β β 13
|
| 352 |
+
β β β 50
|
| 353 |
+
β β β 61
|
| 354 |
+
β β β 29
|
| 355 |
+
β β β 9
|
| 356 |
+
β β β 51
|
| 357 |
+
β β β 255
|
| 358 |
+
β β β 75
|
| 359 |
+
β β β 198
|
| 360 |
+
β β β 49
|
| 361 |
+
β β β 167
|
| 362 |
+
β β β 101
|
| 363 |
+
β β β 203
|
| 364 |
+
β β β 51
|
| 365 |
+
β β β 70
|
| 366 |
+
β β β 36
|
| 367 |
+
β β β 99
|
| 368 |
+
β β β 51
|
| 369 |
+
β β β 24
|
| 370 |
+
β β β 115
|
| 371 |
+
β β β 15
|
| 372 |
+
β β β 48
|
| 373 |
+
β β β 161
|
| 374 |
+
β β β ...
|
| 375 |
+
β β β 'data/221'
|
| 376 |
+
β β <instancemethod write_record at 0x70442f389750>
|
| 377 |
+
β <torch.PyTorchFileWriter object at 0x70409423aa70>
|
| 378 |
+
|
| 379 |
+
RuntimeError: [enforce fail at inline_container.cc:815] . PytorchStreamWriter failed writing file data/221: file write failed
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
During handling of the above exception, another exception occurred:
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Traceback (most recent call last):
|
| 386 |
+
|
| 387 |
+
File "/usr/lib/python3.10/threading.py", line 973, in _bootstrap
|
| 388 |
+
self._bootstrap_inner()
|
| 389 |
+
β β <function Thread._bootstrap_inner at 0x70448271d750>
|
| 390 |
+
β <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
|
| 391 |
+
File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
|
| 392 |
+
self.run()
|
| 393 |
+
β β <function Thread.run at 0x70448271d480>
|
| 394 |
+
β <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
|
| 395 |
+
File "/usr/lib/python3.10/threading.py", line 953, in run
|
| 396 |
+
self._target(*self._args, **self._kwargs)
|
| 397 |
+
β β β β β β {}
|
| 398 |
+
β β β β β <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
|
| 399 |
+
β β β β ({'state': {0: {'exp_avg': tensor([[[ 5.3950e-05, 3.5805e-04, -4.2178e-04, -1.2693e-03, 1.5098e-04,
|
| 400 |
+
β β β 1.2700e-03, ...
|
| 401 |
+
β β β <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
|
| 402 |
+
β β <bound method Checkpointer._save_worker_local of <cosmos_predict2.checkpointer.Checkpointer object at 0x704131e58eb0>>
|
| 403 |
+
β <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
|
| 404 |
+
|
| 405 |
+
File "/home/shadeform/pdt-mimic/mimic-video/model/imaginaire/utils/misc.py", line 261, in wrapper
|
| 406 |
+
result = func(*args, **kwargs)
|
| 407 |
+
β β β {}
|
| 408 |
+
β β (<cosmos_predict2.checkpointer.Checkpointer object at 0x704131e58eb0>, {'state': {0: {'exp_avg': tensor([[[ 5.3950e-05, 3.58...
|
| 409 |
+
β <function Checkpointer._save_worker_local at 0x7041511623b0>
|
| 410 |
+
|
| 411 |
+
> File "/home/shadeform/pdt-mimic/mimic-video/model/cosmos_predict2/checkpointer.py", line 136, in _save_worker_local
|
| 412 |
+
torch.save(state_dict, checkpoint_path)
|
| 413 |
+
β β β β 'checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002400.pt'
|
| 414 |
+
β β β {'state': {0: {'exp_avg': tensor([[[ 5.3950e-05, 3.5805e-04, -4.2178e-04, -1.2693e-03, 1.5098e-04,
|
| 415 |
+
β β 1.2700e-03, -...
|
| 416 |
+
β β <function save at 0x70440e394310>
|
| 417 |
+
β <module 'torch' from '/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/__init__.py'>
|
| 418 |
+
|
| 419 |
+
File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 943, in save
|
| 420 |
+
with _open_zipfile_writer(f) as opened_zipfile:
|
| 421 |
+
β β β <torch.PyTorchFileWriter object at 0x70409423aa70>
|
| 422 |
+
β β 'checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002400.pt'
|
| 423 |
+
β <function _open_zipfile_writer at 0x70440e387c70>
|
| 424 |
+
File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 784, in __exit__
|
| 425 |
+
self.file_like.write_end_of_file()
|
| 426 |
+
β β β <instancemethod write_end_of_file at 0x70442f389780>
|
| 427 |
+
β β <torch.PyTorchFileWriter object at 0x70409423aa70>
|
| 428 |
+
β <torch.serialization._open_zipfile_writer_file object at 0x70413470dba0>
|
| 429 |
+
|
| 430 |
+
RuntimeError: [enforce fail at inline_container.cc:626] . unexpected pos 201024832 vs 201024720
|