JessieLoki commited on
Commit
831e073
Β·
verified Β·
1 Parent(s): d7e8b3d

Upload folder using huggingface_hub

Browse files
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt CHANGED
@@ -1 +1 @@
1
- iter_000002300.pt
 
1
+ iter_000002400.pt
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002400.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ce70b842d2ab47c5a1901cb2a5dd73d5a66649bcb16bb737bd71893c1512e6f
3
+ size 161817668
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002400.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:47b953120827902385ba93044b20e186319e33c37bddfb9f2c680b3cb4988b72
3
+ size 204902400
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000002400.pt ADDED
File without changes
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002400.pt ADDED
File without changes
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/stdout.log CHANGED
@@ -331,3 +331,100 @@ clock 1455.000000 1500.000000 1410.000000
331
  [05-20 15:57:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002300.pt
332
  [05-20 15:57:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0008 seconds
333
  [05-20 15:57:13|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2300...
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
331
  [05-20 15:57:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002300.pt
332
  [05-20 15:57:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0008 seconds
333
  [05-20 15:57:13|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2300...
334
+ [05-20 16:30:57|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002400.pt
335
+ [05-20 16:30:57|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1518 seconds
336
+ [05-20 16:30:59|ERROR|cosmos_predict2/checkpointer.py:143:_save_worker_local] Checkpoint failed to save (local): [enforce fail at inline_container.cc:626] . unexpected pos 201024832 vs 201024720
337
+ Traceback (most recent call last):
338
+
339
+ File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 944, in save
340
+ _save(
341
+ β”” <function _save at 0x70440e394430>
342
+ File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 1216, in _save
343
+ zip_file.write_record(name, storage, num_bytes)
344
+ β”‚ β”‚ β”‚ β”‚ β”” 4194304
345
+ β”‚ β”‚ β”‚ β”” 97
346
+ β”‚ β”‚ β”‚ 253
347
+ β”‚ β”‚ β”‚ 17
348
+ β”‚ β”‚ β”‚ 52
349
+ β”‚ β”‚ β”‚ 168
350
+ β”‚ β”‚ β”‚ 176
351
+ β”‚ β”‚ β”‚ 13
352
+ β”‚ β”‚ β”‚ 50
353
+ β”‚ β”‚ β”‚ 61
354
+ β”‚ β”‚ β”‚ 29
355
+ β”‚ β”‚ β”‚ 9
356
+ β”‚ β”‚ β”‚ 51
357
+ β”‚ β”‚ β”‚ 255
358
+ β”‚ β”‚ β”‚ 75
359
+ β”‚ β”‚ β”‚ 198
360
+ β”‚ β”‚ β”‚ 49
361
+ β”‚ β”‚ β”‚ 167
362
+ β”‚ β”‚ β”‚ 101
363
+ β”‚ β”‚ β”‚ 203
364
+ β”‚ β”‚ β”‚ 51
365
+ β”‚ β”‚ β”‚ 70
366
+ β”‚ β”‚ β”‚ 36
367
+ β”‚ β”‚ β”‚ 99
368
+ β”‚ β”‚ β”‚ 51
369
+ β”‚ β”‚ β”‚ 24
370
+ β”‚ β”‚ β”‚ 115
371
+ β”‚ β”‚ β”‚ 15
372
+ β”‚ β”‚ β”‚ 48
373
+ β”‚ β”‚ β”‚ 161
374
+ β”‚ β”‚ β”‚ ...
375
+ β”‚ β”‚ β”” 'data/221'
376
+ β”‚ β”” <instancemethod write_record at 0x70442f389750>
377
+ β”” <torch.PyTorchFileWriter object at 0x70409423aa70>
378
+
379
+ RuntimeError: [enforce fail at inline_container.cc:815] . PytorchStreamWriter failed writing file data/221: file write failed
380
+
381
+
382
+ During handling of the above exception, another exception occurred:
383
+
384
+
385
+ Traceback (most recent call last):
386
+
387
+ File "/usr/lib/python3.10/threading.py", line 973, in _bootstrap
388
+ self._bootstrap_inner()
389
+ β”‚ β”” <function Thread._bootstrap_inner at 0x70448271d750>
390
+ β”” <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
391
+ File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
392
+ self.run()
393
+ β”‚ β”” <function Thread.run at 0x70448271d480>
394
+ β”” <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
395
+ File "/usr/lib/python3.10/threading.py", line 953, in run
396
+ self._target(*self._args, **self._kwargs)
397
+ β”‚ β”‚ β”‚ β”‚ β”‚ β”” {}
398
+ β”‚ β”‚ β”‚ β”‚ β”” <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
399
+ β”‚ β”‚ β”‚ β”” ({'state': {0: {'exp_avg': tensor([[[ 5.3950e-05, 3.5805e-04, -4.2178e-04, -1.2693e-03, 1.5098e-04,
400
+ β”‚ β”‚ β”‚ 1.2700e-03, ...
401
+ β”‚ β”‚ β”” <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
402
+ β”‚ β”” <bound method Checkpointer._save_worker_local of <cosmos_predict2.checkpointer.Checkpointer object at 0x704131e58eb0>>
403
+ β”” <Thread(Thread-100 (_save_worker_local), started 123413699950144)>
404
+
405
+ File "/home/shadeform/pdt-mimic/mimic-video/model/imaginaire/utils/misc.py", line 261, in wrapper
406
+ result = func(*args, **kwargs)
407
+ β”‚ β”‚ β”” {}
408
+ β”‚ β”” (<cosmos_predict2.checkpointer.Checkpointer object at 0x704131e58eb0>, {'state': {0: {'exp_avg': tensor([[[ 5.3950e-05, 3.58...
409
+ β”” <function Checkpointer._save_worker_local at 0x7041511623b0>
410
+
411
+ > File "/home/shadeform/pdt-mimic/mimic-video/model/cosmos_predict2/checkpointer.py", line 136, in _save_worker_local
412
+ torch.save(state_dict, checkpoint_path)
413
+ β”‚ β”‚ β”‚ β”” 'checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002400.pt'
414
+ β”‚ β”‚ β”” {'state': {0: {'exp_avg': tensor([[[ 5.3950e-05, 3.5805e-04, -4.2178e-04, -1.2693e-03, 1.5098e-04,
415
+ β”‚ β”‚ 1.2700e-03, -...
416
+ β”‚ β”” <function save at 0x70440e394310>
417
+ β”” <module 'torch' from '/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/__init__.py'>
418
+
419
+ File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 943, in save
420
+ with _open_zipfile_writer(f) as opened_zipfile:
421
+ β”‚ β”‚ β”” <torch.PyTorchFileWriter object at 0x70409423aa70>
422
+ β”‚ β”” 'checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002400.pt'
423
+ β”” <function _open_zipfile_writer at 0x70440e387c70>
424
+ File "/home/shadeform/pdt-mimic/mimic-video/model/.venv/lib/python3.10/site-packages/torch/serialization.py", line 784, in __exit__
425
+ self.file_like.write_end_of_file()
426
+ β”‚ β”‚ β”” <instancemethod write_end_of_file at 0x70442f389780>
427
+ β”‚ β”” <torch.PyTorchFileWriter object at 0x70409423aa70>
428
+ β”” <torch.serialization._open_zipfile_writer_file object at 0x70413470dba0>
429
+
430
+ RuntimeError: [enforce fail at inline_container.cc:626] . unexpected pos 201024832 vs 201024720