File size: 4,385 Bytes
78c54ec | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
Fetching ... files: 0it [00:00, ?it/s]
Fetching ... files: 7it [00:00, 11838.76it/s]
logs/b9ed9818-4ffe-4f5a-aff8-0df5b4a150d1.txt
Compiling model and warming up kernels (~7 minutes on first execution)
Sampling steps [0, 1, 4398, 4399, 4400, 4401, 8798, 8799, 8800, 8801, 13198, 13199, 13200, 13201] for warmup
W0615 01:45:46.376000 198186 torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
W0615 01:45:46.376000 198186 torch/distributed/elastic/multiprocessing/api.py:1010] Sending process 198206 closing signal SIGTERM
W0615 01:45:46.377000 198186 torch/distributed/elastic/multiprocessing/api.py:1010] Sending process 198206 closing signal SIGTERM
Traceback (most recent call last):
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
result = self._invoke_run(role)
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
time.sleep(monitor_interval)
~~~~~~~~~~^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
torch.distributed.elastic.multiprocessing.api.SignalException: Process 198186 got signal: 15
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/modded-nanogpt/.venv/bin/torchrun", line 6, in <module>
sys.exit(main())
~~~~^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
return f(*args, **kwargs)
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/run.py", line 991, in main
run(args)
~~~^^^^^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/run.py", line 982, in run
elastic_launch(
~~~~~~~~~~~~~~~
config=config,
~~~~~~~~~~~~~~
entrypoint=cmd,
~~~~~~~~~~~~~~~
)(*cmd_args)
~^^^^^^^^^^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
result = agent.run()
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
result = f(*args, **kwargs)
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/api.py", line 740, in run
self._shutdown(e.sigval)
~~~~~~~~~~~~~~^^^^^^^^^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 413, in _shutdown
self._pcontext.close(death_sig)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 659, in close
self._close(death_sig=death_sig, timeout=timeout)
~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 1022, in _close
handler.proc.wait(time_to_wait)
~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
File "/usr/lib/python3.14/subprocess.py", line 1279, in wait
return self._wait(timeout=timeout)
~~~~~~~~~~^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.14/subprocess.py", line 2078, in _wait
time.sleep(delay)
~~~~~~~~~~^^^^^^^
File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
torch.distributed.elastic.multiprocessing.api.SignalException: Process 198186 got signal: 1
|