ctopd / kdflow /utils /distributed_util.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
1.08 kB
def torch_dist_barrier_and_cuda_sync():
"""Synchronize distributed training and CUDA operations.
This function ensures that:
1. All distributed processes reach this point (barrier)
2. All CUDA operations are completed (synchronize)
"""
import torch
torch.distributed.barrier()
torch.cuda.synchronize()
def stateless_init_process_group(master_address, master_port, rank, world_size, device):
"""
vLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
and vLLM workers.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
pg = StatelessProcessGroup.create(host=master_address, port=master_port, rank=rank, world_size=world_size)
pynccl = PyNcclCommunicator(pg, device=device)
return pynccl