Download kdflow/utils/distributed_util.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 1.08 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/utils/distributed_util.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/utils/distributed_util.py
-
curl -L -o distributed_util.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/utils/distributed_util.py
1.08 kB
| def torch_dist_barrier_and_cuda_sync(): | |
| """Synchronize distributed training and CUDA operations. | |
| This function ensures that: | |
| 1. All distributed processes reach this point (barrier) | |
| 2. All CUDA operations are completed (synchronize) | |
| """ | |
| import torch | |
| torch.distributed.barrier() | |
| torch.cuda.synchronize() | |
| def stateless_init_process_group(master_address, master_port, rank, world_size, device): | |
| """ | |
| vLLM provides `StatelessProcessGroup` to create a process group | |
| without considering the global process group in torch.distributed. | |
| It is recommended to create `StatelessProcessGroup`, and then initialize | |
| the data-plane communication (NCCL) between external (train processes) | |
| and vLLM workers. | |
| """ | |
| from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator | |
| from vllm.distributed.utils import StatelessProcessGroup | |
| pg = StatelessProcessGroup.create(host=master_address, port=master_port, rank=rank, world_size=world_size) | |
| pynccl = PyNcclCommunicator(pg, device=device) | |
| return pynccl | |