def torch_dist_barrier_and_cuda_sync(): """Synchronize distributed training and CUDA operations. This function ensures that: 1. All distributed processes reach this point (barrier) 2. All CUDA operations are completed (synchronize) """ import torch torch.distributed.barrier() torch.cuda.synchronize() def stateless_init_process_group(master_address, master_port, rank, world_size, device): """ vLLM provides `StatelessProcessGroup` to create a process group without considering the global process group in torch.distributed. It is recommended to create `StatelessProcessGroup`, and then initialize the data-plane communication (NCCL) between external (train processes) and vLLM workers. """ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator from vllm.distributed.utils import StatelessProcessGroup pg = StatelessProcessGroup.create(host=master_address, port=master_port, rank=rank, world_size=world_size) pynccl = PyNcclCommunicator(pg, device=device) return pynccl