Buckets:
EmergentTwilight/MMCoT / projects /VLMEvalKit_Thinkmorph_llm /tests /dynamic_queue_distributed_smoke.py
| import time | |
| import torch.distributed as dist | |
| from vlmeval.utils.dynamic_queue import next_index | |
| def main() -> None: | |
| dist.init_process_group("gloo") | |
| rank = dist.get_rank() | |
| world = dist.get_world_size() | |
| store = dist.distributed_c10d._get_default_store() | |
| key = "dynamic-queue-smoke" | |
| if rank == 0: | |
| store.set(key, "0") | |
| dist.barrier() | |
| claims = [] | |
| while (index := next_index(store, key, total=40)) is not None: | |
| claims.append(index) | |
| time.sleep(0.001 * (rank + 1)) | |
| gathered = [None] * world | |
| dist.all_gather_object(gathered, claims) | |
| if rank == 0: | |
| flat = [index for worker in gathered for index in worker] | |
| assert sorted(flat) == list(range(40)) | |
| assert len(flat) == len(set(flat)) | |
| assert any(worker != list(range(i, 40, world)) for i, worker in enumerate(gathered)) | |
| print({"worker_claims": [len(worker) for worker in gathered]}) | |
| dist.destroy_process_group() | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 1.01 kB
- Xet hash:
- 9c279ff3a1e05d0cb98d9ddf99135a8f4258620c420cbbfc3bb45ff12a9955de
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.