luclima commited on
Commit
7e511a4
·
verified ·
1 Parent(s): fb1166a

Upload folder using huggingface_hub (part 41)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. western_ai/usa/pytorch/torch/distributed/CONTRIBUTING.md +87 -0
  2. western_ai/usa/pytorch/torch/distributed/_shard/sharding_plan/__init__.py +1 -0
  3. western_ai/usa/pytorch/torch/distributed/_shard/sharding_plan/api.py +86 -0
  4. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/__init__.py +10 -0
  5. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/_internals.py +244 -0
  6. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/api.py +264 -0
  7. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec.py +230 -0
  8. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/__init__.py +0 -0
  9. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/_common.py +350 -0
  10. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/embedding.py +294 -0
  11. western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/embedding_bag.py +477 -0
  12. western_ai/usa/pytorch/torch/distributed/_sharded_tensor/__init__.py +21 -0
  13. western_ai/usa/pytorch/torch/distributed/_sharding_spec/__init__.py +22 -0
  14. western_ai/usa/pytorch/torch/distributed/_state_dict_utils.py +829 -0
  15. western_ai/usa/pytorch/torch/distributed/_symmetric_memory/__init__.py +2387 -0
  16. western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_nccl.py +111 -0
  17. western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_nvshmem_triton.py +1220 -0
  18. western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_rocshmem_triton.py +459 -0
  19. western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_shmem_triton.py +37 -0
  20. western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_shmem_triton_utils.py +101 -0
  21. western_ai/usa/pytorch/torch/distributed/_tensor/__init__.py +45 -0
  22. western_ai/usa/pytorch/torch/distributed/_tensor/api.py +9 -0
  23. western_ai/usa/pytorch/torch/distributed/_tensor/placement_types.py +10 -0
  24. western_ai/usa/pytorch/torch/distributed/_token_switch.py +343 -0
  25. western_ai/usa/pytorch/torch/distributed/_tools/__init__.py +12 -0
  26. western_ai/usa/pytorch/torch/distributed/_tools/common_utils.py +43 -0
  27. western_ai/usa/pytorch/torch/distributed/_tools/fake_collectives.py +270 -0
  28. western_ai/usa/pytorch/torch/distributed/_tools/fsdp2_mem_tracker.py +595 -0
  29. western_ai/usa/pytorch/torch/distributed/_tools/ilp_utils.py +293 -0
  30. western_ai/usa/pytorch/torch/distributed/_tools/mem_tracker.py +956 -0
  31. western_ai/usa/pytorch/torch/distributed/_tools/memory_tracker.py +302 -0
  32. western_ai/usa/pytorch/torch/distributed/_tools/mod_tracker.py +259 -0
  33. western_ai/usa/pytorch/torch/distributed/_tools/runtime_estimator.py +401 -0
  34. western_ai/usa/pytorch/torch/distributed/_tools/sac_estimator.py +965 -0
  35. western_ai/usa/pytorch/torch/distributed/_tools/sac_ilp.py +297 -0
  36. western_ai/usa/pytorch/torch/distributed/algorithms/__init__.py +1 -0
  37. western_ai/usa/pytorch/torch/distributed/algorithms/_checkpoint/__init__.py +0 -0
  38. western_ai/usa/pytorch/torch/distributed/algorithms/_checkpoint/checkpoint_wrapper.py +313 -0
  39. western_ai/usa/pytorch/torch/distributed/algorithms/_comm_hooks/__init__.py +7 -0
  40. western_ai/usa/pytorch/torch/distributed/algorithms/_comm_hooks/default_hooks.py +191 -0
  41. western_ai/usa/pytorch/torch/distributed/algorithms/_optimizer_overlap/__init__.py +1 -0
  42. western_ai/usa/pytorch/torch/distributed/algorithms/_optimizer_overlap/optimizer_overlap.py +96 -0
  43. western_ai/usa/pytorch/torch/distributed/algorithms/_quantization/__init__.py +0 -0
  44. western_ai/usa/pytorch/torch/distributed/algorithms/_quantization/quantization.py +151 -0
  45. western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/__init__.py +141 -0
  46. western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py +464 -0
  47. western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/debugging_hooks.py +29 -0
  48. western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/default_hooks.py +211 -0
  49. western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/mixed_precision_hooks.py +86 -0
  50. western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/optimizer_overlap_hooks.py +163 -0
western_ai/usa/pytorch/torch/distributed/CONTRIBUTING.md ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Contributing to PyTorch Distributed
2
+
3
+ Please go through PyTorch's top level [Contributing Guide](../../CONTRIBUTING.md) before proceeding with this guide.
4
+
5
+ [PyTorch Distributed Overview](https://pytorch.org/tutorials//beginner/dist_overview.html) is a great starting point with a lot of tutorials, documentation and design docs covering PyTorch Distributed. We highly recommend going through some of that material before you start working on PyTorch Distributed.
6
+
7
+ In this document, we mostly focus on some of the code structure for PyTorch distributed and implementation details.
8
+
9
+ ### Onboarding Tasks
10
+
11
+ A list of onboarding tasks can be found [here](https://github.com/pytorch/pytorch/issues?q=is%3Aopen%20is%3Aissue%20label%3A%22pt_distributed_rampup%22).
12
+
13
+
14
+ ## Code Pointers
15
+
16
+ The relevant code for different modules is either inside the c++ C10D library or the torch python library.
17
+
18
+ #### Collectives and Communication Library (C10D)
19
+
20
+ This is the place to look if you are trying to find low-level communication APIs, process group creation, etc.
21
+
22
+ - API layer: [torch/distributed/distributed_c10d.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/distributed_c10d.py)
23
+ - Python Bindings: [torch/csrc/distributed/c10d/init.cpp](https://github.com/pytorch/pytorch/blob/main/torch/csrc/distributed/c10d/init.cpp)
24
+ - Implementations: [torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp](https://github.com/pytorch/pytorch/blob/main/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp)
25
+
26
+ #### DTensor
27
+
28
+ - API layer: ([torch/distributed/_tensor/api.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/_tensor/api.py))
29
+ - Implementation: see other files in the same folder
30
+
31
+ #### Distributed Data Parallel (DDP)
32
+
33
+ - API layer: [torch/nn/parallel/distributed.py](https://github.com/pytorch/pytorch/blob/main/torch/nn/parallel/distributed.py)
34
+ - Reducer (backend that schedules allreduces): [torch/csrc/distributed/c10d/reducer.cpp](https://github.com/pytorch/pytorch/blob/main/torch/csrc/distributed/c10d/reducer.cpp)
35
+ - Mixed Precision Hooks: [torch/distributed/algorithms/ddp_comm_hooks/default_hooks.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/algorithms/ddp_comm_hooks/default_hooks.py)
36
+ #### Fully Sharded Data Parallel (FSDP)
37
+
38
+ - FSDP: [torch/distributed/fsdp/api.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/fsdp/api.py)
39
+ - FSDP2: [torch/distributed/_composable/fsdp/fully_shard.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/_composable/fsdp/fully_shard.py)
40
+ - Implementations are contained in other files in the same folder as the API for each variant
41
+
42
+ #### Tensor Parallel (TP)
43
+
44
+ - API layer: [torch/distributed/tensor/parallel/api.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/tensor/parallel/api.py)
45
+ - Implementation: see other files in the same folder
46
+
47
+ #### Pipeline Parallel (PP)
48
+
49
+ - Pipeline Schedules: [torch/distributed/pipelining/schedules.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/pipelining/schedules.py)
50
+ - Pipeline Stage: [torch/distributed/pipelining/stage.py](https://github.com/pytorch/pytorch/blob/main/torch/distributed/pipelining/stage.py)
51
+
52
+
53
+ ## Adding Tests
54
+
55
+ You should write tests for your changes just like in other parts of PyTorch, but you may need to use some test infrastructure to run either multi-process tests on multiple GPUs, or use a FakeProcessGroup to mock out communications.
56
+
57
+ Most testing can be done from python, and you can find existing python tests [here](https://github.com/pytorch/pytorch/tree/main/test/distributed).
58
+
59
+ For an example of using the MultiProcessTestCase to run a test on multiple GPUs, see tests in [test_c10d_nccl.py](https://github.com/pytorch/pytorch/blob/main/test/distributed/test_c10d_nccl.py)
60
+
61
+ ## Testing Your Changes
62
+
63
+ All the unit tests can be found under the [test/distributed](../../test/distributed) directory and RPC tests in particular are under [test/distributed/rpc](../../test/distributed/rpc). A few examples on how to run unit tests:
64
+
65
+
66
+ ```
67
+ # Run the c10d unit tests.
68
+ python test/distributed/test_c10d_common.py
69
+ python test/distributed/test_c10d_gloo.py
70
+ python test/distributed/test_c10d_nccl.py
71
+
72
+ # Run the Store tests.
73
+ python test/distributed/test_store.py
74
+
75
+ # Run Process Group Wrapper tests.
76
+ python test/distributed/test_pg_wrapper.py
77
+
78
+ # Run distributed tests, including tests for Distributed Data Parallel.
79
+ python test/run_test.py --verbose -i distributed/test_distributed_spawn
80
+
81
+ # Run a single test in the test_distributed_spawn test suite.
82
+ touch /tmp/barrier && TEMP_DIR="/tmp" BACKEND="nccl" WORLD_SIZE="2" python test/distributed/test_distributed_spawn.py -v TestDistBackendWithSpawn.test_ddp_profiling_torch_profiler
83
+
84
+ # Run a specific test method. Uses pytest (pip install pytest).
85
+ # ProcessGroup gloo/nccl test
86
+ pytest -vs test/distributed/test_c10d_common.py -k test_multi_limit_single_dtype
87
+ ```
western_ai/usa/pytorch/torch/distributed/_shard/sharding_plan/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .api import ShardingPlan, ShardingPlanner
western_ai/usa/pytorch/torch/distributed/_shard/sharding_plan/api.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import abc
2
+ from dataclasses import dataclass
3
+
4
+ import torch.nn as nn
5
+ from torch.distributed._shard.sharder import Sharder
6
+ from torch.distributed._shard.sharding_spec import ShardingSpec
7
+
8
+
9
+ @dataclass
10
+ class ShardingPlan:
11
+ """
12
+ Representation of a sharding plan, describes how to shard a module
13
+ across hosts. `plan` is used to shard module parameters according to the spec provided,
14
+ `output_plan` and `return_local_tensor` are optional, they are used to specify the output
15
+ layout of a module with a spec, and when to convert back to data parallel fashion.
16
+
17
+ Args:
18
+ plan (Dict[str, Union[:class:`torch.distributed._shard.sharding_spec.ShardingSpec`,
19
+ :class:`torch.distributed._shard.sharder.Sharder`]):
20
+ a dict describes how to shard a module, there're currently two ways to shard a module:
21
+ 1. directly shard a module parameter by a `ShardingSpec`, keyed by the name of
22
+ a parameter to a `ShardingSpec`.
23
+ 2. shard a submodule by applying a `Sharder` on it, keyed by the name of a module
24
+ to a `Sharder` object.
25
+ output_plan (Dict[str, :class:`torch.distributed._shard.sharding_spec.ShardingSpec`), optional):
26
+ a dict specifies the layout of a module's output which produces a ShardedTensor,
27
+ keyed by the name of module to ShardingSpec("" in key means the root module).
28
+ Default: `None`
29
+ return_local_tensor (List[str], optional): a list of string, each element enables
30
+ a module's sharded output to be returned as a Tensor from its local shards to
31
+ ensure further processing in a data parallel fashion. ("" in list means the
32
+ root module).
33
+ Default: None
34
+ Example:
35
+ Suppose we want to shard a module with two linear layers and then run it with DDP, we also
36
+ want to convert the output of the second linear layer back to DDP, we can do it as follows:
37
+
38
+ >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
39
+ >>> class MyModule(nn.Module):
40
+ >>> def __init__(self) -> None:
41
+ >>> super().__init__()
42
+ >>> self.fc1 = nn.Linear()
43
+ >>> self.gelu = nn.GELU()
44
+ >>> self.fc2 = nn.Linear()
45
+ >>> self.relu = nn.Linear()
46
+ >>>
47
+ >>> def forward(self, input):
48
+ >>> return self.relu(self.fc2(self.gelu(self.fc1(input))))
49
+
50
+
51
+ >>> # xdoctest: +SKIP("Undefined spec1, spec2)
52
+ >>> sharding_plan = ShardingPlan(
53
+ >>> plan={
54
+ >>> "fc1.weight": spec1,
55
+ >>> "fc2.weight": spec2
56
+ >>> },
57
+ >>> output_plan={
58
+ >>> "fc2": output_spec
59
+ >>> },
60
+ >>> return_local_tensor=["fc2"]
61
+ >>> )
62
+ """
63
+
64
+ plan: dict[str, ShardingSpec | Sharder]
65
+ output_plan: dict[str, ShardingSpec] | None = None
66
+ return_local_tensor: list[str] | None = None
67
+
68
+
69
+ class ShardingPlanner(abc.ABC):
70
+ """
71
+ Default ShardingPlanner interface, can be extended and
72
+ implement advanced sharding strategies.
73
+ """
74
+
75
+ @abc.abstractmethod
76
+ def build_plan(self, module: nn.Module) -> ShardingPlan:
77
+ """
78
+ Given an nn.Module, define how to shard the module across
79
+ ranks, return a ShardingPlan
80
+ Args:
81
+ module (:class:`torch.nn.Module`):
82
+ The module to apply sharding to.
83
+ Returns:
84
+ A :class:`torch.distributed._shard.sharding_plan.ShardingPlan` object that
85
+ represents how to shard the module.
86
+ """
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/__init__.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch.distributed._shard.metadata import ShardMetadata
2
+
3
+ from .api import (
4
+ _infer_sharding_spec_from_shards_metadata,
5
+ DevicePlacementSpec,
6
+ EnumerableShardingSpec,
7
+ PlacementSpec,
8
+ ShardingSpec,
9
+ )
10
+ from .chunk_sharding_spec import ChunkShardingSpec as ChunkShardingSpec
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/_internals.py ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import math
3
+ import sys
4
+ from bisect import bisect_right, insort
5
+
6
+ from torch.distributed._shard.metadata import ShardMetadata
7
+
8
+
9
+ def _check_shard_metadata_pair_overlap(shard1: ShardMetadata, shard2: ShardMetadata):
10
+ """
11
+ Checks if two shards overlap.
12
+ """
13
+
14
+ # For each dim of each shard, check if one shard resides on the other
15
+ # end of second shard with respect to that dim. As an example for a 2D
16
+ # shard, we would check if one shard is above or on the left of the
17
+ # other shard.
18
+ ndims = len(shard1.shard_offsets)
19
+ for i in range(ndims):
20
+ if shard1.shard_offsets[i] >= shard2.shard_offsets[i] + shard2.shard_sizes[i]:
21
+ return False
22
+ if shard2.shard_offsets[i] >= shard1.shard_offsets[i] + shard1.shard_sizes[i]:
23
+ return False
24
+
25
+ return True
26
+
27
+
28
+ def _find_nd_overlapping_shards(
29
+ shards: list[ShardMetadata], sharded_dims: list[int]
30
+ ) -> tuple[int, int] | None:
31
+ """Find overlapping shards using sweep-line algorithm."""
32
+ if len(shards) <= 1:
33
+ return None
34
+
35
+ dims = len(sharded_dims)
36
+ if dims == 0:
37
+ return None
38
+
39
+ sweep_dim_idx = 0
40
+ if dims > 1:
41
+ max_size = 0
42
+ for i, dim in enumerate(sharded_dims):
43
+ dim_size = shards[0].shard_offsets[dim] + shards[0].shard_sizes[dim]
44
+ if dim_size > max_size:
45
+ max_size = dim_size
46
+ sweep_dim_idx = i
47
+ sweep_dim = sharded_dims[sweep_dim_idx]
48
+
49
+ sorted_indices = sorted(
50
+ range(len(shards)),
51
+ key=lambda idx: (
52
+ shards[idx].shard_offsets[sweep_dim],
53
+ *(shards[idx].shard_offsets[d] for d in sharded_dims if d != sweep_dim),
54
+ ),
55
+ )
56
+ active: list[tuple[int, int]] = []
57
+
58
+ for idx in sorted_indices:
59
+ current = shards[idx]
60
+ start = current.shard_offsets[sweep_dim]
61
+ end = start + current.shard_sizes[sweep_dim]
62
+
63
+ cutoff = bisect_right(active, (start, sys.maxsize))
64
+ if cutoff:
65
+ del active[:cutoff]
66
+
67
+ for _, other_idx in active:
68
+ other = shards[other_idx]
69
+
70
+ if _check_shard_metadata_pair_overlap(current, other):
71
+ return (other_idx, idx)
72
+ insort(active, (end, idx))
73
+ return None
74
+
75
+
76
+ def _find_1d_overlapping_shards(
77
+ shards: list[ShardMetadata], dim: int
78
+ ) -> tuple[int, int] | None:
79
+ # (begin, end, index_in_shards). Begin and end are inclusive.
80
+ intervals = [
81
+ (s.shard_offsets[dim], s.shard_offsets[dim] + s.shard_sizes[dim] - 1, i)
82
+ for i, s in enumerate(shards)
83
+ ]
84
+ intervals.sort()
85
+ for i in range(len(shards) - 1):
86
+ if intervals[i][1] >= intervals[i + 1][0]:
87
+ return (intervals[i][2], intervals[i + 1][2])
88
+ return None
89
+
90
+
91
+ def validate_non_overlapping_shards_metadata(shards: list[ShardMetadata]):
92
+ """
93
+ Ensures none of the shards overlap with each other.
94
+
95
+ Args:
96
+ shards(List[ShardMetadata]): List of :class:`ShardMetadata` objects representing
97
+ each shard.
98
+ Raises:
99
+ ``ValueError`` if there's overlap in any two shards.
100
+ """
101
+ if not shards or len(shards) == 1:
102
+ return
103
+
104
+ sharded_dims: list[int] = []
105
+ for dim in range(len(shards[0].shard_offsets)):
106
+ for i in range(1, len(shards)):
107
+ if (
108
+ shards[i].shard_offsets[dim] != shards[0].shard_offsets[dim]
109
+ or shards[i].shard_sizes[dim] != shards[0].shard_sizes[dim]
110
+ ):
111
+ sharded_dims.append(dim)
112
+ break
113
+
114
+ pair: tuple[int, int] | None = None
115
+ if len(sharded_dims) == 0:
116
+ # if shard is all zeros, we should consider as pass
117
+ all_zeros: bool = all(
118
+ # strictly limited all offsets to be 0 to pass
119
+ # could loosen it later on
120
+ shard.shard_offsets == [0] * len(shards[0].shard_offsets)
121
+ and math.prod(shard.shard_sizes) == 0 # one dimension is 0
122
+ for shard in shards
123
+ )
124
+ if all_zeros:
125
+ return
126
+ # All shards are the same, all dims are not partitioned. Choose any 2.
127
+ pair = (0, 1)
128
+ elif len(sharded_dims) == 1:
129
+ # Shards are partitioned over only one dimension. Overlap can be found
130
+ # using a O(nlogn) overlapping interval algorithm.
131
+ pair = _find_1d_overlapping_shards(shards, sharded_dims[0])
132
+ else:
133
+ # Shards are partitioned over more than one dimension.
134
+ # Use sweep-line algorithm for O(n log n) complexity.
135
+ pair = _find_nd_overlapping_shards(shards, sharded_dims)
136
+
137
+ if pair:
138
+ raise ValueError(f"Shards {shards[pair[0]]} and {shards[pair[1]]} overlap")
139
+
140
+
141
+ def check_tensor(shards_metadata, tensor_dims) -> None:
142
+ """
143
+ Checks if the shards_metadata is compatible with the provided tensor dims.
144
+
145
+ Args:
146
+ shards_metadata(List[ShardMetadata]): List of :class:`ShardMetadata`
147
+ objects representing each shard of the tensor.
148
+ tensor_dims(Sequence of int): Dimensions of tensor to verify
149
+ Raises:
150
+ ``ValueError`` if not compatible.
151
+ """
152
+
153
+ # If the tensor's volume matches the total volume of all shards and
154
+ # all shard boundaries are within tensor dims, we have a compatible
155
+ # sharding spec for this tensor. Note that we have already verified
156
+ # we don't have overlapping shards.
157
+ tensor_rank = len(tensor_dims)
158
+ shards_rank = len(shards_metadata[0].shard_offsets)
159
+ if tensor_rank != shards_rank:
160
+ raise ValueError(
161
+ f"Rank of tensor is {tensor_rank}, but shards rank is {shards_rank}"
162
+ )
163
+
164
+ total_shard_volume = 0
165
+ for shard in shards_metadata:
166
+ shard_volume = 1
167
+ for i, shard_length in enumerate(shard.shard_sizes):
168
+ shard_volume *= shard_length
169
+ if shard.shard_offsets[i] + shard.shard_sizes[i] > tensor_dims[i]:
170
+ raise ValueError(
171
+ f"Shard offset {shard.shard_offsets[i]} and length "
172
+ f"{shard.shard_sizes[i]} exceeds tensor dim: {tensor_dims[i]} for shard {shard}"
173
+ )
174
+ total_shard_volume += shard_volume
175
+
176
+ tensor_volume = 1
177
+ for size in tensor_dims:
178
+ tensor_volume *= size
179
+
180
+ if total_shard_volume != tensor_volume:
181
+ # TODO: Can we improve this error message to point out the gaps?
182
+ raise ValueError(
183
+ f"Total volume of shards: {total_shard_volume} "
184
+ f"does not match tensor volume: {tensor_volume}, in other words "
185
+ f"all the individual shards do not cover the entire tensor"
186
+ )
187
+
188
+
189
+ def get_split_size(dim_size, chunks):
190
+ """
191
+ Computes the split size inline with ``torch.chunk``
192
+
193
+ Args:
194
+ dim_size(int): Size of the dimension being chunked.
195
+ chunks(int): Number of chunks to create for ``dim_size``.
196
+
197
+ Returns:
198
+ An int indicating the split size to use.
199
+ """
200
+ return (dim_size + chunks - 1) // chunks
201
+
202
+
203
+ def get_chunked_dim_size(dim_size, split_size, idx):
204
+ """
205
+ Computes the dim size of the chunk for provided ``idx`` given ``dim_size``
206
+ and ``split_size``.
207
+
208
+ Args:
209
+ dim_size(int): Size of the dimension being chunked.
210
+ split_size(int): The chunk size for each chunk of ``dim_size``.
211
+ idx(int): The index of chunk whose dim size is being requested.
212
+
213
+ Returns:
214
+ An int indicating the dim size of the chunk.
215
+ """
216
+ return max(min(dim_size, split_size * (idx + 1)) - split_size * idx, 0)
217
+
218
+
219
+ def get_chunk_sharding_params(sharding_dim_size, world_size, spec, rank):
220
+ """
221
+ Generate the start pos and offset length for the current rank for
222
+ chunk sharding.
223
+
224
+ Args:
225
+ sharding_dim_size(int): The dimension length which we shard on.
226
+ world_size(int): number of ranks.
227
+ spec (:class:`torch.distributed._shard.sharding_spec.ChunkShardingSpec`):
228
+ sharding spec.
229
+ rank(int): # of cuda process.
230
+
231
+ Returns:
232
+ start_pos(int): start position of sharded tensor on the given rank.
233
+ chunk_size(int): chunk size of sharded tensor on the given rank.
234
+ """
235
+ split_size = get_split_size(sharding_dim_size, world_size)
236
+ current_offsets = 0
237
+ start_pos = current_offsets
238
+ for idx, placement in enumerate(spec.placements):
239
+ chunk_size = get_chunked_dim_size(sharding_dim_size, split_size, idx)
240
+ if rank == placement.rank():
241
+ start_pos = current_offsets
242
+ break
243
+ current_offsets += chunk_size
244
+ return start_pos, chunk_size # type: ignore[possibly-undefined]
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/api.py ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import functools
3
+ import operator
4
+ from abc import ABC, abstractmethod
5
+ from collections.abc import Callable
6
+ from dataclasses import dataclass
7
+ from typing import TYPE_CHECKING
8
+
9
+ import torch
10
+ import torch.distributed._shard.sharded_tensor.metadata as sharded_tensor_meta
11
+ from torch.distributed._shard.metadata import ShardMetadata
12
+ from torch.distributed._shard.op_registry_utils import _decorator_func
13
+
14
+ from ._internals import (
15
+ check_tensor,
16
+ get_chunked_dim_size,
17
+ get_split_size,
18
+ validate_non_overlapping_shards_metadata,
19
+ )
20
+
21
+
22
+ if TYPE_CHECKING:
23
+ # Only include ShardedTensor when do type checking, exclude it
24
+ # from run-time to resolve circular dependency.
25
+ from torch.distributed._shard.sharded_tensor import ShardedTensor
26
+
27
+
28
+ class PlacementSpec(ABC): # noqa: B024
29
+ """
30
+ Base class representing the placement of an entity. Subclasses of this
31
+ class can be used to specify customized placements which might not be
32
+ covered by existing APIs.
33
+ """
34
+
35
+
36
+ @dataclass
37
+ class DevicePlacementSpec(PlacementSpec):
38
+ """
39
+ Associates placement of an entity with a single device.
40
+
41
+ Args:
42
+ device(:class:`torch.distributed._remote_device`): The device to place the entity on.
43
+ """
44
+
45
+ device: torch.distributed._remote_device
46
+
47
+ def __post_init__(self):
48
+ if not isinstance(self.device, torch.distributed._remote_device):
49
+ self.device = torch.distributed._remote_device(self.device)
50
+
51
+
52
+ class ShardingSpec(ABC):
53
+ """
54
+ Base class representing sharding specifications.
55
+ """
56
+
57
+ @abstractmethod
58
+ def build_metadata(
59
+ self,
60
+ tensor_sizes: torch.Size,
61
+ tensor_properties: sharded_tensor_meta.TensorProperties,
62
+ ) -> sharded_tensor_meta.ShardedTensorMetadata:
63
+ """
64
+ Given a global tensor size, define how to shard a tensor like this shape
65
+ across ranks, return ShardedTensorMetadata
66
+ Args:
67
+ tensor_sizes (:class:`torch.Size`):
68
+ The tensor shape to shard on, a `torch.Size` object that represents the
69
+ tensor shape to be sharded according to the ShardingSpec.
70
+ tensor_properties(:class:`torch.distributed._shard.sharded_tensor.TensorProperties):
71
+ Tensor properties used to create a ShardedTensor.
72
+ Returns:
73
+ A :class:`ShardedTensorMetadata` object that encodes the information about
74
+ the layout of the ShardedTensor and its properties.
75
+ """
76
+
77
+ @abstractmethod
78
+ def shard(
79
+ self, tensor: torch.Tensor, src_rank: int = 0, process_group=None
80
+ ) -> "ShardedTensor":
81
+ """
82
+ Given a global tensor on src_rank, shard this tensor
83
+ across ranks within the process group, return a ShardedTensor.
84
+ Args:
85
+ tensor (:class:`torch.Tensor`): Tensor needs to be sharded.
86
+ Keyword args:
87
+ src_rank (int, optional): The source rank which is used as the ground truth of
88
+ the data for the parameter that would be sharded and scattered
89
+ across the rest of the ranks.
90
+ Default: 0.
91
+ process_group (ProcessGroup, optional): The process group to work on. If None,
92
+ the default process group will be used.
93
+ Returns:
94
+ A :class:`ShardedTensor` sharded from the given tensor.
95
+ """
96
+
97
+
98
+ # Ops customized for a particular ShardingSpec.
99
+ _CUSTOM_SHARDING_SPEC_OPS: dict[str, dict[Callable, Callable]] = {}
100
+
101
+
102
+ def _has_custom_op(sharding_spec, op):
103
+ """
104
+ Returns whether or not the ShardingSpec has a custom op implementation.
105
+ """
106
+ class_name = type(sharding_spec).__qualname__
107
+ return (
108
+ class_name in _CUSTOM_SHARDING_SPEC_OPS
109
+ and op in _CUSTOM_SHARDING_SPEC_OPS[class_name]
110
+ )
111
+
112
+
113
+ def _dispatch_custom_op(
114
+ sharding_spec, op: Callable, types, args, kwargs, process_group
115
+ ):
116
+ """
117
+ Calls the custom op for this ShardingSpec if it exists.
118
+ """
119
+ class_name = type(sharding_spec).__qualname__
120
+ if not _has_custom_op(sharding_spec, op):
121
+ raise RuntimeError(f"Custom op: {op} not registered for {class_name}")
122
+ func = _CUSTOM_SHARDING_SPEC_OPS[class_name][op]
123
+ return func(types, args, kwargs, process_group)
124
+
125
+
126
+ def custom_sharding_spec_op(sharding_spec_class, func):
127
+ """
128
+ Decorator to allow custom registration of ops.
129
+ Args:
130
+ sharding_spec_class(type): The ShardingSpec for which we need to add this custom op.
131
+ func(Callable): The op to override (ex: torch.bmm)
132
+ """
133
+ class_name = sharding_spec_class.__qualname__
134
+ if class_name not in _CUSTOM_SHARDING_SPEC_OPS:
135
+ _CUSTOM_SHARDING_SPEC_OPS[class_name] = {}
136
+ return functools.partial(
137
+ _decorator_func, op=func, op_table=_CUSTOM_SHARDING_SPEC_OPS[class_name]
138
+ )
139
+
140
+
141
+ @dataclass
142
+ class EnumerableShardingSpec(ShardingSpec):
143
+ """
144
+ This is a type of PlacementSpec that allows users to specify a generic
145
+ sharding scheme by enumerating exactly how each shard is laid out.
146
+
147
+ Args:
148
+ shards(List[ShardMetadata]): List of :class:`ShardMetadata` objects representing
149
+ each shard. Note that none of the shards should overlap.
150
+ """
151
+
152
+ shards: list[ShardMetadata]
153
+
154
+ def __post_init__(self):
155
+ if len(self.shards) == 0:
156
+ raise ValueError(f"Empty shard list provided: {self.shards}")
157
+
158
+ # Validate each shard has same rank.
159
+ rank = -1
160
+ for shard in self.shards:
161
+ if rank != -1 and rank != len(shard.shard_offsets):
162
+ raise ValueError(
163
+ f"Found inconsistent ranks for shards: {rank} and {len(shard.shard_offsets)}"
164
+ )
165
+ rank = len(shard.shard_offsets)
166
+
167
+ validate_non_overlapping_shards_metadata(self.shards)
168
+
169
+ def build_metadata(
170
+ self,
171
+ tensor_sizes: torch.Size,
172
+ tensor_properties: sharded_tensor_meta.TensorProperties,
173
+ ) -> sharded_tensor_meta.ShardedTensorMetadata:
174
+ # check if shards form a valid tensor
175
+ check_tensor(self.shards, tensor_sizes)
176
+ return sharded_tensor_meta.ShardedTensorMetadata(
177
+ self.shards, tensor_sizes, tensor_properties
178
+ )
179
+
180
+ def shard(
181
+ self, tensor: torch.Tensor, src_rank: int = 0, process_group=None
182
+ ) -> "ShardedTensor":
183
+ # TODO: figure out a generic and efficient way to scatter the shards for EnumerableShardingSpec
184
+ raise NotImplementedError("EnumerableShardingSpec.shard not implemented yet!")
185
+
186
+
187
+ def _infer_sharding_spec_from_shards_metadata(shards_metadata):
188
+ """
189
+ Infer the sharding spec from the metadata of each shard of a ShardedTensor.
190
+ If the tensor is sharded only on one dimension, we can then verify whether it's
191
+ a ChunkShardingSpec or not. The way to verify it is to first get the total length
192
+ and perform a chunk sharding with the given placements to see if we can have the
193
+ same chunk size as the given shards_metadata. If not, we assume it's enum sharded.
194
+
195
+ Args:
196
+ shards_metadata (List[ShardMetadata]): List of Metadata of local shards.
197
+
198
+ Returns:
199
+ A :class:`torch.distributed._shard.sharding_spec.ShardingSpec` object of sharding
200
+ spec for one sharded tensor.
201
+ """
202
+ placements = []
203
+ chunk_sharding_dim = None
204
+ chunk_offset_list = []
205
+ shard_size_list = []
206
+ shard_offset_list = []
207
+ # collect local shard metadata from the global sharded_tensor_metadata
208
+ for shard_metadata in shards_metadata: # type: ignore[attr-defined]
209
+ placements.append(shard_metadata.placement)
210
+ local_offsets = shard_metadata.shard_offsets
211
+ chunk_offset_list.append(sum(local_offsets))
212
+ shard_size_list.append(shard_metadata.shard_sizes)
213
+ shard_offset_list.append(shard_metadata.shard_offsets)
214
+ shard_dims = [idx for idx, e in enumerate(local_offsets) if e != 0]
215
+ # If the offset is [0, 0, ..., 0] (all zeros),
216
+ # we cannot decide whether how the tensor is sharded.
217
+ if len(shard_dims) == 0:
218
+ continue
219
+ # If the offset is [0, N, .,0, M, 0, .., 0],
220
+ # we are sure it's sharded by more than one dimension.
221
+ if len(shard_dims) != 1:
222
+ chunk_sharding_dim = None
223
+ break
224
+ # If the offset is [0, 0, .,0, M, 0, .., 0], aka, it's sharded by just
225
+ # one dimension, we need to make sure all ranks share the same dimension.
226
+ if not chunk_sharding_dim:
227
+ chunk_sharding_dim = shard_dims[0]
228
+ elif chunk_sharding_dim != shard_dims[0]:
229
+ chunk_sharding_dim = None
230
+ break
231
+
232
+ if chunk_sharding_dim is not None:
233
+ # Ensure we infer the correct placement order from offsets
234
+ placements = [
235
+ x
236
+ for _, x in sorted(
237
+ zip(chunk_offset_list, placements), key=operator.itemgetter(0)
238
+ )
239
+ ]
240
+
241
+ from .chunk_sharding_spec import ChunkShardingSpec
242
+
243
+ chunk_spec = ChunkShardingSpec(
244
+ dim=chunk_sharding_dim,
245
+ placements=placements,
246
+ )
247
+
248
+ shard_sizes = sorted([x[chunk_sharding_dim] for x in shard_size_list])
249
+ shard_total_length = sum(shard_sizes)
250
+ shard_offsets = sorted([x[chunk_sharding_dim] for x in shard_offset_list])
251
+
252
+ chunks = len(placements)
253
+ split_size = get_split_size(shard_total_length, chunks)
254
+ chunk_shard_sizes = sorted(
255
+ [
256
+ get_chunked_dim_size(shard_total_length, split_size, idx)
257
+ for idx in range(chunks)
258
+ ]
259
+ )
260
+ # Should match ChunkShardingSpec offsets calculation
261
+ chunk_shard_offsets = [split_size * idx for idx in range(chunks)]
262
+ if shard_sizes == chunk_shard_sizes and shard_offsets == chunk_shard_offsets:
263
+ return chunk_spec
264
+ return EnumerableShardingSpec(shards_metadata)
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec.py ADDED
@@ -0,0 +1,230 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from dataclasses import dataclass
3
+ from typing import cast, TYPE_CHECKING
4
+
5
+ import torch
6
+ import torch.distributed as dist
7
+ import torch.distributed._shard.sharded_tensor.metadata as sharded_tensor_meta
8
+ import torch.distributed.distributed_c10d as distributed_c10d
9
+ from torch.distributed._shard._utils import narrow_tensor
10
+ from torch.distributed._shard.metadata import ShardMetadata
11
+ from torch.distributed._shard.sharded_tensor.shard import Shard
12
+ from torch.distributed._shard.sharded_tensor.utils import (
13
+ _parse_and_validate_remote_device,
14
+ )
15
+
16
+ from ._internals import get_chunked_dim_size, get_split_size
17
+ from .api import ShardingSpec
18
+
19
+
20
+ if TYPE_CHECKING:
21
+ # Only include ShardedTensor when do type checking, exclude it
22
+ # from run-time to resolve circular dependency.
23
+ from torch.distributed._shard.sharded_tensor import ShardedTensor
24
+
25
+
26
+ @dataclass
27
+ class ChunkShardingSpec(ShardingSpec):
28
+ """
29
+ This is a type of PlacementSpec that defines the placement as being sharded
30
+ across multiple devices. In particular, it represents sharding a Tensor
31
+ along a single dimension into equal chunks (similar to :meth:`torch.chunk`).
32
+
33
+ The semantics of how a tensor is partitioned is inline with
34
+ :meth:`torch.chunk`, where ``dim`` in torch.chunk corresponds to the
35
+ specified ``dim`` and ``chunks`` in torch.chunk is the number of elements
36
+ in the placement specified.
37
+
38
+ Args:
39
+ dim (int):
40
+ The dimension to shard on, an integer representing the dimension.
41
+ placement(List[Union[_remote_device, str]]):
42
+ Specifies the placement of each shard of the Tensor. The size of
43
+ the list represents the number of shards to be created. This could
44
+ be a list of
45
+ :class:`torch.distributed._remote_device`'s. This list
46
+ could also contain a string which represents remote
47
+ device as accepted by
48
+ :class:`torch.distributed._remote_device`
49
+ """
50
+
51
+ ShardingDim = int | str
52
+
53
+ dim: ShardingDim
54
+ placements: list[torch.distributed._remote_device | str]
55
+
56
+ def __post_init__(self):
57
+ self._verify_dim(self.dim)
58
+ for i, remote_device in enumerate(self.placements):
59
+ if not isinstance(remote_device, torch.distributed._remote_device):
60
+ self.placements[i] = torch.distributed._remote_device(remote_device)
61
+
62
+ @staticmethod
63
+ def _verify_dim(dim):
64
+ # Validate the sharding spec.
65
+ # TODO: support named dimension
66
+ if isinstance(dim, str):
67
+ raise NotImplementedError(
68
+ "ChunkShardingSpec does not support named dimension yet!"
69
+ )
70
+
71
+ if not isinstance(dim, int):
72
+ raise ValueError(f"Sharding dim needs to be an integer, found: {dim}")
73
+
74
+ def build_metadata(
75
+ self,
76
+ tensor_sizes: torch.Size,
77
+ tensor_properties: sharded_tensor_meta.TensorProperties,
78
+ ) -> sharded_tensor_meta.ShardedTensorMetadata:
79
+ tensor_num_dim = len(tensor_sizes)
80
+
81
+ self._verify_dim(self.dim)
82
+ if self.dim >= tensor_num_dim or self.dim < -tensor_num_dim: # type: ignore[operator]
83
+ raise ValueError(f"Invalid sharding dim: {self.dim}")
84
+
85
+ shards_metadata = []
86
+ sharding_dim_size = tensor_sizes[self.dim] # type: ignore[index]
87
+ chunks = len(self.placements)
88
+ split_size = get_split_size(sharding_dim_size, chunks)
89
+ for idx, placement in enumerate(self.placements):
90
+ # generate ShardMetadata for each placement device
91
+ chunked_dim_size = get_chunked_dim_size(sharding_dim_size, split_size, idx)
92
+ shard_size = list(tensor_sizes)
93
+ current_offsets = [0] * tensor_num_dim
94
+ current_offsets[self.dim] = split_size * idx # type: ignore[index]
95
+ shard_size[self.dim] = chunked_dim_size # type: ignore[index]
96
+
97
+ shard_metadata = ShardMetadata(
98
+ shard_offsets=current_offsets,
99
+ shard_sizes=shard_size,
100
+ placement=placement,
101
+ )
102
+ shards_metadata.append(shard_metadata)
103
+
104
+ return sharded_tensor_meta.ShardedTensorMetadata(
105
+ shards_metadata, tensor_sizes, tensor_properties
106
+ )
107
+
108
+ def shard(
109
+ self, tensor: torch.Tensor, src_rank: int = 0, process_group=None
110
+ ) -> "ShardedTensor":
111
+ """
112
+ Args:
113
+ src_rank: group rank relative to ``process_group``
114
+
115
+ N.B. If ``process_group`` is None, ``src_rank`` is a global rank.
116
+ """
117
+ # relative imports to avoid circular dependency
118
+ from torch.distributed._shard.sharded_tensor import ShardedTensor
119
+
120
+ tensor_properties = sharded_tensor_meta.TensorProperties(
121
+ dtype=tensor.dtype,
122
+ layout=tensor.layout,
123
+ requires_grad=tensor.requires_grad,
124
+ memory_format=torch.contiguous_format,
125
+ pin_memory=tensor.is_pinned(),
126
+ )
127
+ current_rank = dist.get_rank(process_group)
128
+ current_global_rank = dist.get_rank()
129
+ tensor_meta = self.build_metadata(tensor.size(), tensor_properties)
130
+ local_shards = []
131
+ local_tensor = None
132
+ local_metadata = None
133
+
134
+ tensors_to_scatter = cast(
135
+ list[torch.Tensor | None],
136
+ [None] * dist.get_world_size(process_group),
137
+ )
138
+
139
+ sharding_dim_size = tensor.size()[self.dim] # type: ignore[index]
140
+ chunks = len(self.placements)
141
+ split_size = get_split_size(sharding_dim_size, chunks)
142
+ scatter_shape = list(tensor.size())
143
+ scatter_shape[self.dim] = split_size # type: ignore[index]
144
+
145
+ for shard_meta in tensor_meta.shards_metadata:
146
+ remote_global_rank, device = _parse_and_validate_remote_device(
147
+ process_group, shard_meta.placement
148
+ )
149
+ if current_rank == src_rank:
150
+ # Reshape to get shard for this rank and we don't want autograd
151
+ # recording here for the narrow op and 'local_shard' should be a
152
+ # leaf variable in the autograd graph.
153
+ narrowed_tensor = narrow_tensor(tensor, shard_meta)
154
+ if shard_meta.shard_sizes[self.dim] < split_size: # type: ignore[index]
155
+ # for the last shard that might be smaller to other shards
156
+ # resize the narrowed tensor to the same size and use it for
157
+ # the scatter collective as dist.scatter requires same size
158
+ # inputs on every rank
159
+ tensor_to_scatter = (
160
+ narrowed_tensor.detach().clone().resize_(scatter_shape)
161
+ )
162
+ else:
163
+ tensor_to_scatter = narrowed_tensor.detach().clone(
164
+ memory_format=torch.contiguous_format
165
+ )
166
+
167
+ tensors_to_scatter[
168
+ # pyrefly: ignore [bad-argument-type]
169
+ dist.get_group_rank(process_group, remote_global_rank)
170
+ ] = tensor_to_scatter
171
+
172
+ if current_global_rank == remote_global_rank:
173
+ local_tensor = torch.empty(
174
+ scatter_shape,
175
+ dtype=tensor.dtype,
176
+ layout=tensor.layout,
177
+ device=device,
178
+ )
179
+ local_metadata = shard_meta
180
+
181
+ # each rank should have local_tensor and local_metadata initialized if we build
182
+ # the metadata list in a correct way.
183
+ if local_tensor is None:
184
+ raise AssertionError
185
+ if local_metadata is None:
186
+ raise AssertionError
187
+
188
+ # Scatter the shards to all ranks in the pg
189
+ # scatter takes the global rank as ``src``
190
+ src_for_scatter = src_rank
191
+ if (
192
+ process_group is not None
193
+ and process_group is not distributed_c10d._get_default_group()
194
+ ):
195
+ src_for_scatter = distributed_c10d.get_global_rank(
196
+ process_group, src_for_scatter
197
+ )
198
+
199
+ tensors_to_scatter_: list[torch.Tensor] | None = None
200
+ if current_rank == src_rank:
201
+ tensors_to_scatter_ = []
202
+ for t in tensors_to_scatter:
203
+ if not isinstance(t, torch.Tensor):
204
+ raise AssertionError
205
+ tensors_to_scatter_.append(t)
206
+
207
+ dist.scatter(
208
+ local_tensor,
209
+ scatter_list=tensors_to_scatter_,
210
+ src=src_for_scatter,
211
+ group=process_group,
212
+ )
213
+
214
+ if list(local_tensor.size()) != local_metadata.shard_sizes:
215
+ # detach again after receiving to ensure local shards remain a leaf node
216
+ local_tensor = local_tensor.resize_(local_metadata.shard_sizes).detach()
217
+
218
+ # Sync requires_grad to local_shard.
219
+ local_tensor.requires_grad = tensor.requires_grad
220
+
221
+ local_shards.append(Shard(tensor=local_tensor, metadata=local_metadata))
222
+
223
+ st = ShardedTensor._init_from_local_shards_and_global_metadata(
224
+ local_shards, tensor_meta, process_group=process_group
225
+ )
226
+
227
+ # Manually set sharding_spec
228
+ st._sharding_spec = self
229
+
230
+ return st
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/__init__.py ADDED
File without changes
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/_common.py ADDED
@@ -0,0 +1,350 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+
3
+ import torch
4
+ import torch.distributed as dist
5
+ from torch.distributed._shard.sharded_tensor import ShardedTensor
6
+ from torch.distributed._shard.sharded_tensor._ops._common import _sharded_op_common
7
+ from torch.distributed._shard.sharding_spec import ChunkShardingSpec
8
+ from torch.distributed._shard.sharding_spec._internals import (
9
+ get_chunk_sharding_params,
10
+ get_chunked_dim_size,
11
+ get_split_size,
12
+ )
13
+ from torch.distributed._shard.sharding_spec.api import custom_sharding_spec_op
14
+ from torch.distributed.nn.functional import (
15
+ _all_gather_base,
16
+ all_reduce,
17
+ all_to_all_single,
18
+ )
19
+
20
+
21
+ def _chunk_sharding_spec_check(spec, op):
22
+ """
23
+ For the given op implementation check if the sharding spec is ChunkShardingSpec.
24
+ """
25
+ if not isinstance(spec, ChunkShardingSpec):
26
+ raise NotImplementedError(
27
+ f"Only ChunkShardingSpec supported for '{op.__name__}'."
28
+ )
29
+
30
+
31
+ def _register_sharded_op_on_local_tensor(
32
+ op, early_stop_func=None, extra_check=None, customized_func=None
33
+ ):
34
+ """
35
+ Handles ``__torch_function__`` dispatch for ops which are performed on
36
+ the single local tensor of the sharded tensor such as op like
37
+ ``torch.nn.functional.softmax`` or ``torch.Tensor.view``.
38
+
39
+ For more complicated ops, a customized func can be used to generate
40
+ the new local tensor, sharding spec and sharded tensor size.
41
+
42
+ Args:
43
+ op: The op to be registered and applied to all shards of the st.
44
+ early_stop_func (Callable, optional): the func for early stop.
45
+ Default: if ``None``, no early stop.
46
+ extra_check (Callable, optional): the func for extra condition check.
47
+ Default: if ``None``, no extra check.
48
+ customized_func (Callable, optional): the func for customized logic
49
+ to generate the new local tensor, sharding spec and sharded tensor size.
50
+ Default: if ``None``, we simply lower to the real op call with
51
+ the single local tensor of the st.
52
+
53
+ Return:
54
+ func (Callable): registered implementation for sharded op for
55
+ ``__torch_function__`` dispatch.
56
+ """
57
+
58
+ @custom_sharding_spec_op(ChunkShardingSpec, op)
59
+ @_sharded_op_common(op, early_stop_func, extra_check)
60
+ def sharded_tensor_op_on_local_tensor(types, args=(), kwargs=None, pg=None):
61
+ # pyrefly: ignore [bad-index]
62
+ st = args[0]
63
+ sharding_spec = st.sharding_spec()
64
+ if len(st.local_shards()) != 1:
65
+ raise TypeError(
66
+ f"torch function '{op.__name__}', with args: {args} and "
67
+ f"kwargs: {kwargs} only supported for single local tensor!"
68
+ )
69
+ st_size = st.size()
70
+ if customized_func:
71
+ local_tensor, sharding_spec, st_size = customized_func(args, kwargs, pg)
72
+ else:
73
+ args = (st.local_tensor(), *args[1:])
74
+ local_tensor = op(*args, **kwargs)
75
+ return ShardedTensor._init_from_local_tensor(
76
+ local_tensor.contiguous(),
77
+ sharding_spec,
78
+ st_size, # type: ignore[arg-type]
79
+ process_group=pg,
80
+ init_rrefs=st._init_rrefs,
81
+ )
82
+
83
+
84
+ def _handle_col_wise_sharding_base(
85
+ op_func,
86
+ col_dim,
87
+ input,
88
+ world_size,
89
+ weight,
90
+ local_shard,
91
+ pg,
92
+ gathered_inputs,
93
+ mode=None,
94
+ gathered_per_sample_weights=None,
95
+ gathered_offsets=None,
96
+ padding_idx=None,
97
+ ):
98
+ """
99
+ For col-wise sharding of weight, lots of logic are common.
100
+ So we extract the common logic and put in this function:
101
+ Step 1. To get input from each rank and
102
+ Step 2. To perform the op on the concatenated tensor.
103
+ Step 3. To distribute results to each rank with col rearrangement.
104
+ Step 4. To concatenate all results from all ranks.
105
+
106
+ Args:
107
+ op_func: operator which is applied to the input tensor.
108
+ col_dim: dim of result tensor after the operation.
109
+ input: tensor to be applied op on.
110
+ world_size: number of ranks.
111
+ weight: sharded weight tensor.
112
+ local_shard: col-wise sharded weight tensor.
113
+ pg: process group.
114
+ gathered_inputs: list of inputs from all ranks. If specified, we
115
+ don't need to communicate with each rank any more.
116
+ mode: aggregation mode of EmbeddingBag.
117
+ gathered_per_sample_weights: per_sample_weights across all ranks.
118
+ gathered_offsets: offsets across all ranks.
119
+ padding_idx: If specified, the entries at padding_idx do
120
+ not contribute to the gradient; therefore, the embedding
121
+ vector at padding_idx is not updated during training,
122
+ i.e. it remains as a fixed "pad".
123
+ Note that the embedding vector at padding_idx is
124
+ excluded from the reduction.
125
+
126
+ Return: final result of input being applied with the op.
127
+ """
128
+ # run the operator's function for all the inputs.
129
+ results = []
130
+ for i, inp in enumerate(gathered_inputs):
131
+ if op_func is torch.nn.functional.embedding_bag:
132
+ result = op_func(
133
+ inp,
134
+ local_shard,
135
+ offsets=gathered_offsets[i] if gathered_offsets is not None else None,
136
+ # pyrefly: ignore [bad-argument-type]
137
+ mode=mode,
138
+ per_sample_weights=gathered_per_sample_weights[i]
139
+ if gathered_per_sample_weights is not None
140
+ else None,
141
+ padding_idx=padding_idx,
142
+ )
143
+ elif op_func is torch.nn.functional.embedding:
144
+ result = op_func(
145
+ inp,
146
+ local_shard,
147
+ padding_idx=padding_idx,
148
+ )
149
+ else:
150
+ result = op_func(inp, local_shard)
151
+ results.append(torch.transpose(result, 0, col_dim))
152
+
153
+ # Distribute results to each rank with col rearrangement.
154
+ output = _result_distribute_with_col_rearrange(
155
+ results, input, world_size, weight, pg
156
+ )
157
+
158
+ # transpose the output and return result.
159
+ return torch.transpose(output, 0, col_dim)
160
+
161
+
162
+ def _result_distribute_with_col_rearrange(results, input, world_size, weight, pg):
163
+ """
164
+ For col-wise sharding of weight, we need to distribute
165
+ results to each rank. We do them in this function.
166
+ Note that, if the index in the Sharding Spec is not equal to
167
+ the rank number, we need to do the rearrangement based on the
168
+ order given by the Sharding Spec (placement).
169
+
170
+ Args:
171
+ results: results from ops applied to inputs from all ranks.
172
+ We need to distribute them back to their original ranks.
173
+ input: tensor to be applied op to.
174
+ world_size: number of ranks.
175
+ weight: sharded weight tensor.
176
+ pg: process group.
177
+
178
+ Return: column rearranged result.
179
+ """
180
+ # Process results and outputs for all2all.
181
+ sharding_dim = weight._sharding_spec.dim
182
+ sharding_dim_size = weight.size(sharding_dim)
183
+ dims = list(results[0].size())
184
+ dims[0] = sharding_dim_size
185
+ combined_results = torch.cat(results)
186
+ output = torch.empty(
187
+ *dims, device=combined_results.device, dtype=combined_results.dtype
188
+ )
189
+
190
+ # Compute output splits
191
+ split_size = get_split_size(sharding_dim_size, world_size)
192
+ output_split_sizes = [0] * world_size
193
+ for idx, placement in enumerate(weight._sharding_spec.placements):
194
+ output_split_sizes[placement.rank()] = get_chunked_dim_size(
195
+ sharding_dim_size, split_size, idx
196
+ )
197
+
198
+ # distribute the outputs using all2all.
199
+ output = all_to_all_single(
200
+ output, combined_results, output_split_sizes=output_split_sizes, group=pg
201
+ )
202
+
203
+ # Check if we need to rearrange columns appropriately for output.
204
+ rearrange_columns = any(
205
+ idx != placement.rank()
206
+ for idx, placement in enumerate(weight._sharding_spec.placements)
207
+ )
208
+ if not rearrange_columns:
209
+ return output
210
+
211
+ indices = []
212
+ for placement in weight._sharding_spec.placements:
213
+ dim_size = output_split_sizes[placement.rank()]
214
+ start = sum(
215
+ split_size if i < placement.rank() else 0
216
+ for i, split_size in enumerate(output_split_sizes)
217
+ )
218
+ indices += list(range(start, start + dim_size))
219
+
220
+ return output.index_select(0, torch.tensor(indices, device=output.device))
221
+
222
+
223
+ def _handle_max_norm_col_wise(
224
+ max_norm,
225
+ norm_type,
226
+ local_shard,
227
+ input,
228
+ world_size,
229
+ gathered_inputs,
230
+ pg,
231
+ ):
232
+ """
233
+ For col-wise sharding of weight, we need to aggregate the
234
+ norm across all ranks before we can perform the proper re-norm.
235
+ Note that, the max_norm logic is only applied to the embedding
236
+ indices that are looked up and not the whole shard.
237
+
238
+ Args:
239
+ max_norm: If given, each embedding vector with norm larger
240
+ than max_norm is renormalized to have norm max_norm.
241
+ Note: this will modify weight in-place.
242
+ norm_type: The p in the p-norm to compute for the max_norm option.
243
+ local_shard: col-wise shared local weight used for lookup.
244
+ input: tensor to be applied op to.
245
+ world_size: number of ranks.
246
+ gathered_inputs: list of inputs from all ranks.
247
+ pg: process group.
248
+
249
+ Return:
250
+ local_shard_norm_renormed: local_shard re-normed to max_norm if the norm is larger
251
+ than it.
252
+
253
+ """
254
+ norm_type = norm_type if norm_type is not None else 2.0
255
+ unique_inp = torch.unique(torch.cat(gathered_inputs))
256
+ local_shard_sum = torch.sum(
257
+ torch.pow(torch.abs(local_shard), norm_type), dim=1, dtype=local_shard.dtype
258
+ )
259
+ # For col-wise sharding, we need to first aggregate the powered sum
260
+ # from each rank first and then calculate the norm.
261
+ local_shard_sum = all_reduce(local_shard_sum, group=pg)
262
+ local_shard_norm = torch.pow(local_shard_sum, 1.0 / norm_type)
263
+ max_norm_tensor = torch.full(
264
+ (local_shard.size(0),),
265
+ float("inf"),
266
+ dtype=local_shard.dtype,
267
+ device=input.device,
268
+ )
269
+ max_norm_tensor[unique_inp] = max_norm
270
+ local_shard_t = local_shard.t().contiguous()
271
+ normalized_tensor = torch.where(
272
+ local_shard_norm > max_norm_tensor, max_norm_tensor, local_shard_norm
273
+ )
274
+ # Make sure divisor is not zero.
275
+ local_shard_norm[local_shard_norm == 0.0] = 1.0
276
+ local_shard_norm_renormed = (
277
+ torch.div(torch.mul(local_shard_t, normalized_tensor), local_shard_norm)
278
+ .t()
279
+ .contiguous()
280
+ )
281
+ return local_shard_norm_renormed
282
+
283
+
284
+ def _all_gather_base_input(input, pg):
285
+ """
286
+ Use _all_gather_base to get a concatenated input from each rank.
287
+
288
+ Args:
289
+ input: tensor to be applied op on.
290
+ pg: process group.
291
+
292
+ Returns:
293
+ gathered_inputs: input gathered from each rank and concat by dim 0.
294
+ """
295
+ # allgather the inputs first.
296
+ gather_inp_size = list(input.size())
297
+ gather_inp_size[0] = input.size(0) * dist.get_world_size(pg)
298
+ gather_inp = torch.empty(gather_inp_size, device=input.device, dtype=input.dtype)
299
+ return _all_gather_base(gather_inp, input, group=pg)
300
+
301
+
302
+ def _handle_row_wise_mask(gather_inp, padding_idx, weight, world_size, rank):
303
+ """
304
+ Mask the input for embedding look-up for IDs which are not stored
305
+ on the current rank. This function also adjust the ``padding_idx``
306
+ so that it is only used on the rank where the corresponding row is
307
+ stored.
308
+
309
+ Note that, with ``max_norm`` flag on, only weights of rows being
310
+ looked up will be re-normed. So we need an extra row for masked ID
311
+ so that it does not affect the final result and ``max_norm``.
312
+
313
+ Args:
314
+ gather_inp: tensor to be applied op on gathered from all ranks.
315
+ padding_idx: If specified, the entries at padding_idx do
316
+ not contribute to the gradient; therefore, the embedding
317
+ vector at padding_idx is not updated during training,
318
+ i.e. it remains as a fixed "pad".
319
+ Note that the embedding vector at padding_idx is
320
+ excluded from the reduction.
321
+ weight: weight tensor of Embedding look-up table.
322
+ world_size: number of ranks.
323
+ rank: # of cuda process.
324
+
325
+ Returns:
326
+ lookup_input: Tensor of masked input.
327
+ padding_idx: adjusted padding_idx.
328
+ padding_row: The extra row we used during lookup so that
329
+ looking up does not affect ``max_norm``.
330
+ """
331
+ (start_pos, chunk_size) = get_chunk_sharding_params(
332
+ weight.size(0), world_size, weight._sharding_spec, rank
333
+ )
334
+ mask = (gather_inp < start_pos) | (gather_inp >= start_pos + chunk_size)
335
+ lookup_input = gather_inp.clone() - start_pos
336
+ lookup_input[mask] = chunk_size
337
+ if (
338
+ padding_idx is not None
339
+ and padding_idx >= start_pos
340
+ and padding_idx < (start_pos + chunk_size)
341
+ ):
342
+ padding_idx = padding_idx - start_pos
343
+ else:
344
+ padding_idx = None
345
+
346
+ # When max_norm is set, it will only re-norm the row being looked up.
347
+ padding_row = torch.zeros(
348
+ 1, weight.size(1), device=gather_inp.device, dtype=weight.dtype
349
+ )
350
+ return lookup_input, padding_idx, padding_row
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/embedding.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+
3
+ import torch
4
+ import torch.distributed as dist
5
+ from torch.distributed._shard.sharded_tensor import ShardedTensor
6
+ from torch.distributed._shard.sharding_spec import ChunkShardingSpec
7
+ from torch.distributed._shard.sharding_spec.api import custom_sharding_spec_op
8
+ from torch.distributed.nn.functional import all_gather, reduce_scatter
9
+
10
+ from ._common import (
11
+ _all_gather_base_input,
12
+ _handle_col_wise_sharding_base,
13
+ _handle_max_norm_col_wise,
14
+ _handle_row_wise_mask,
15
+ )
16
+
17
+
18
+ @custom_sharding_spec_op(ChunkShardingSpec, torch.nn.functional.embedding)
19
+ def sharded_embedding(types, args, kwargs, pg):
20
+ """
21
+ Handles ``__torch_function__`` dispatch for ``torch.nn.functional.embedding``.
22
+ This method computes a sharded embedding lookup and has the following limitations:
23
+
24
+ 1. Supports only sharding of ``weight``.
25
+ 2. Supports only ``ChunkShardingSpec``.
26
+ 3. Supports only a single local shard per rank.
27
+ 4. Supports all specs except for scale_grad_by_freq, sparse, etc.
28
+
29
+ Based on the dimension that the weight is sharded on, there are two
30
+ algorithms:
31
+
32
+ ROWWISE SHARDING
33
+ ================
34
+ For row-wise sharding the weight is sharded on dimension 0.
35
+
36
+ The overall algorithm can be best explained with an example. Let's assume
37
+ the dims for input are (4 x 6) and W are (10 x 17) and W is sharded across
38
+ 4 GPUs creating 3 shard of (3 x 17) and 1 shard of (1 x 17).
39
+ The algorithm is as follows:
40
+
41
+ 1. First the input is all gathered to all ranks, since this is SPMD and
42
+ input is actually sharded across all ranks. The inputs then become a
43
+ 4 (4 x 6) tensor on each rank. For example if the given input is
44
+ tensor([[6, 5, 2, 9, 6, 3],
45
+ [3, 1, 2, 4, 7, 6],
46
+ [4, 0, 4, 9, 8, 9],
47
+ [8, 6, 6, 4, 6, 1]])
48
+ on rank 0.
49
+ Then on every rank, we will have this tensor.
50
+ If input itself is already replicated, no all-gather will be done.
51
+ 2. Next, we mask the ID which are not stored on that rank.
52
+ For example on rank 0, we store ID [0, 1, 2]. We only keep the ID
53
+ inside the set of numbers. The rest of them will be masked to an extra row.
54
+ The masked matrix will be used for embedding look up and is like:
55
+ tensor([[4, 4, 2, 4, 4, 4],
56
+ [4, 1, 2, 4, 4, 4],
57
+ [4, 0, 4, 4, 4, 4],
58
+ [4, 4, 4, 4, 4, 1]])
59
+ The reason of having an extra row (aka, number 4 in the example) is
60
+ because when max_norm is specified only weight which has looked will
61
+ be re-normed so mask IDs whose embeddings are not stored in current
62
+ rank will to an extra row will ensure max_norm still works as expected.
63
+ 3. If max_norm is specified, the extra row guarantees that the mask ID will
64
+ not affect the behavior of weight re-norm.
65
+
66
+ COLWISE SHARDING
67
+ ================
68
+ For col-wise sharding the weight is sharded on dimension 1.
69
+
70
+ The overall algorithm can be best explained with an example. Let's assume
71
+ the dims for input are (4 x 6) and W are (16 x 17) and W is sharded across
72
+ 4 GPUs creating 3 shards of (16 x 5) and 1 shard of (16 x 2).
73
+ The algorithm is as follows:
74
+
75
+ 1. First the input is broadcasted to all ranks, since this is SPMD we
76
+ actually do an all_gather for all the inputs resulting in 4 (4 x 6)
77
+ inputs on each rank.
78
+ 2. Next we perform local embedding lookup operation by apply each
79
+ input (4 x 6) with the local shard (16 x 5) ((16 x 2) for the last).
80
+ This results in 4 (5 x 6 x 4) ((2 x 6 x 4) for the last) matrices
81
+ on each rank. We transpose dim 0 and dim 2.
82
+ 3. Next, we concat these 4 matrices and perform an all2all to share the
83
+ appropriate (5 x 6 x 4) or (2 x 6 x 4) matrices to each rank.
84
+ 4. Now, each rank receives a (17 x 6 x 4) matrix which is basically the
85
+ size of the result we need.
86
+ 5. If placements are not in order any appropriate rearrangement of columns
87
+ are done for the (17 x 6 x 4) matrix and finally we transpose the
88
+ dim 0 and dim 2 again.
89
+ 6. If max_norm is specified, we manually sum up the norm and renorm. Because
90
+ the renorm must be in place, we need to override the local_shard to mimic
91
+ this behavior.
92
+ """
93
+ # Validate input params
94
+ _validate_embedding_param(args, kwargs)
95
+
96
+ input = args[0]
97
+ weight = args[1]
98
+ max_norm = kwargs.get("max_norm")
99
+ norm_type = kwargs.get("norm_type")
100
+ padding_idx = kwargs.get("padding_idx")
101
+
102
+ local_shard = weight.local_tensor().contiguous()
103
+ sharding_dim = weight._sharding_spec.dim
104
+ world_size = dist.get_world_size(pg)
105
+ rank = dist.get_rank(pg)
106
+
107
+ if sharding_dim == 1:
108
+ output, local_shard = _handle_col_wise_sharding(
109
+ input, world_size, weight, local_shard, max_norm, norm_type, padding_idx, pg
110
+ )
111
+ weight.local_shards()[0].tensor = local_shard
112
+ return output
113
+ elif sharding_dim == 0:
114
+ return _handle_row_wise_sharding(
115
+ input,
116
+ world_size,
117
+ weight,
118
+ local_shard,
119
+ max_norm,
120
+ norm_type,
121
+ padding_idx,
122
+ rank,
123
+ pg,
124
+ )
125
+ else:
126
+ raise RuntimeError(
127
+ f"nn.Embedding weight sharded on dim {sharding_dim} not supported!"
128
+ )
129
+
130
+
131
+ def _validate_embedding_param(args, kwargs):
132
+ """
133
+ Validate input params of sharded embedding op.
134
+
135
+ Args:
136
+ input: list of ID used for lookup.
137
+ weight: sharded weight tensor.
138
+ kwargs: same as normal Embedding.
139
+
140
+ Return: None.
141
+ """
142
+
143
+ input = args[0]
144
+ weight = args[1]
145
+ max_norm = kwargs.get("max_norm")
146
+ scale_grad_by_freq = kwargs.get("scale_grad_by_freq")
147
+ sparse = kwargs.get("sparse")
148
+
149
+ # Validate types
150
+ if not isinstance(input, torch.Tensor):
151
+ raise TypeError("input need to be torch.Tensor")
152
+ if not isinstance(weight, ShardedTensor):
153
+ raise TypeError("weight needs to be ShardedTensor")
154
+ weight_size = weight.size()
155
+ if len(weight_size) != 2:
156
+ raise ValueError("Weight needs to have exactly 2 dims")
157
+ if int(torch.min(input).item()) < 0:
158
+ raise ValueError(
159
+ "Index out of range in Input %d %d",
160
+ int(torch.min(input).item()),
161
+ weight_size[1],
162
+ )
163
+ if int(torch.max(input).item()) >= weight_size[0]:
164
+ raise ValueError(
165
+ "Index out of range in Input %d %d",
166
+ int(torch.max(input).item()),
167
+ weight_size[1],
168
+ )
169
+ if scale_grad_by_freq:
170
+ raise RuntimeError(
171
+ 'nn.Embedding weight sharded with flag on "scale_grad_by_freq" not supported!'
172
+ )
173
+ if sparse:
174
+ raise RuntimeError(
175
+ 'nn.Embedding weight sharded with flag on "sparse" not supported!'
176
+ )
177
+ if max_norm and max_norm <= 0.0:
178
+ raise ValueError('"max_norm" must be larger than zero!')
179
+
180
+ if not isinstance(weight._sharding_spec, ChunkShardingSpec):
181
+ raise ValueError("Only ChunkShardingSpec supported for ShardedTensor ops!")
182
+ if len(weight.local_shards()) != 1:
183
+ raise ValueError("Only one local shard supported!")
184
+
185
+
186
+ def _handle_col_wise_sharding(
187
+ input, world_size, weight, local_shard, max_norm, norm_type, padding_idx, pg
188
+ ):
189
+ """
190
+ Entry-point function to handle the logic of col-wise sharding of weight
191
+ for embedding. (Detailed explanations of the logic can be found in
192
+ the comment for sharded_embedding.)
193
+
194
+ Args:
195
+ input: list of ID used for lookup and aggregation.
196
+ world_size: number of ranks.
197
+ weight: sharded weight tensor.
198
+ local_shard: col-wise shared local weight used for lookup.
199
+ max_norm: If given, each embedding vector with norm larger
200
+ than max_norm is renormalized to have norm max_norm.
201
+ Note: this will modify weight in-place.
202
+ norm_type: The p in the p-norm to compute for the max_norm option.
203
+ padding_idx: If specified, the entries at padding_idx do
204
+ not contribute to the gradient; therefore, the embedding
205
+ vector at padding_idx is not updated during training,
206
+ i.e. it remains as a fixed "pad".
207
+ pg: process group.
208
+
209
+ Returns: final result of lookup.
210
+ """
211
+ # allgather the inputs first for non Replicated Tensor.
212
+ gathered_inputs = all_gather(input, group=pg)
213
+
214
+ if max_norm is not None:
215
+ # max_norm changes the weight in-place
216
+ local_shard = _handle_max_norm_col_wise(
217
+ max_norm, norm_type, local_shard, input, world_size, gathered_inputs, pg
218
+ )
219
+
220
+ output = _handle_col_wise_sharding_base(
221
+ torch.nn.functional.embedding,
222
+ len(input.size()),
223
+ input,
224
+ world_size,
225
+ weight,
226
+ local_shard,
227
+ pg,
228
+ gathered_inputs,
229
+ padding_idx=padding_idx,
230
+ )
231
+ return (output, local_shard)
232
+
233
+
234
+ def _handle_row_wise_sharding(
235
+ input, world_size, weight, local_shard, max_norm, norm_type, padding_idx, rank, pg
236
+ ):
237
+ """
238
+ Entry-point function to handle the logic of row-wise sharding of weight
239
+ for embedding. (Detailed explanations of the logic can be found in
240
+ the comment for sharded_embedding.)
241
+
242
+ Args:
243
+ input: list of ID used for lookup and aggregation.
244
+ world_size: number of ranks.
245
+ weight: sharded weight tensor.
246
+ local_shard: row-wise shared local weight used for lookup.
247
+ max_norm: If given, each embedding vector with norm larger
248
+ than max_norm is renormalized to have norm max_norm.
249
+ Note: this will modify weight in-place.
250
+ norm_type: The p in the p-norm to compute for the max_norm option.
251
+ padding_idx: If specified, the entries at padding_idx do
252
+ not contribute to the gradient; therefore, the embedding
253
+ vector at padding_idx is not updated during training,
254
+ i.e. it remains as a fixed "pad".
255
+ rank: # of cuda process.
256
+ pg: process group.
257
+
258
+ Returns: final result of lookup.
259
+ """
260
+ # allgather the inputs first for non Replicated Tensor.
261
+ gather_inp = _all_gather_base_input(input, pg)
262
+
263
+ # Mask the input according to sharding spec.
264
+ lookup_input, padding_idx, padding_row = _handle_row_wise_mask(
265
+ gather_inp, padding_idx, weight, world_size, rank
266
+ )
267
+
268
+ # When input is a large tensor, the value of weight is changed.
269
+ # This is a walk-around for now. GH issue: #81717
270
+ if max_norm is not None:
271
+ torch.nn.functional.embedding(
272
+ torch.unique(lookup_input)[:-1],
273
+ local_shard,
274
+ padding_idx=padding_idx,
275
+ max_norm=max_norm,
276
+ norm_type=norm_type,
277
+ )
278
+ max_norm = None
279
+
280
+ local_input_embeddings = torch.nn.functional.embedding(
281
+ lookup_input,
282
+ torch.cat([local_shard, padding_row]),
283
+ padding_idx=padding_idx,
284
+ max_norm=max_norm,
285
+ norm_type=norm_type,
286
+ )
287
+
288
+ # TODO: Make the result a PartialTensor.
289
+ local_shards = local_input_embeddings.chunk(pg.size())
290
+ return reduce_scatter(
291
+ torch.empty_like(local_shards[0]),
292
+ list(local_shards),
293
+ group=pg,
294
+ )
western_ai/usa/pytorch/torch/distributed/_shard/sharding_spec/chunk_sharding_spec_ops/embedding_bag.py ADDED
@@ -0,0 +1,477 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+
3
+ from typing import cast
4
+
5
+ import torch
6
+ import torch.distributed as dist
7
+ from torch._C._distributed_c10d import ReduceOp
8
+ from torch.distributed._shard.sharded_tensor import ShardedTensor
9
+ from torch.distributed._shard.sharding_spec import ChunkShardingSpec
10
+ from torch.distributed._shard.sharding_spec.api import custom_sharding_spec_op
11
+ from torch.distributed.nn.functional import all_gather, reduce_scatter
12
+
13
+ from ._common import (
14
+ _all_gather_base_input,
15
+ _handle_col_wise_sharding_base,
16
+ _handle_max_norm_col_wise,
17
+ _handle_row_wise_mask,
18
+ )
19
+
20
+
21
+ @custom_sharding_spec_op(ChunkShardingSpec, torch.nn.functional.embedding_bag)
22
+ def sharded_embedding_bag(types, args, kwargs, pg):
23
+ """
24
+ Handles ``__torch_function__`` dispatch for ``torch.nn.functional.embedding_bag``.
25
+ This method computes a sharded embedding bag aggregation and has the following limitations:
26
+
27
+ 1. Supports only sharding of ``weight``.
28
+ 2. Supports only ``ChunkShardingSpec``.
29
+ 3. Supports only a single local shard per rank.
30
+ 4. Supports all specs except for scale_grad_by_freq, sparse, etc.
31
+
32
+ Based on the dimension that the weight is sharded on, there are two
33
+ algorithms:
34
+
35
+ ROWWISE SHARDING
36
+ ================
37
+ For row-wise sharding the weight is sharded on dimension 0.
38
+
39
+ The overall algorithm can be best explained with an example. Let's assume
40
+ the dims for input are (4 x 6) and W are (16 x 17) and W is sharded across
41
+ 4 GPUs creating 4 shard of (4 x 17).
42
+ The algorithm is as follows:
43
+
44
+ 1. First the input is all gathered to all ranks, since this is SPMD and
45
+ input is actually sharded across all ranks. The inputs then become a
46
+ 4 (4 x 6) tensor on each rank. For example if the given input is
47
+ tensor([[6, 5, 2, 9, 6, 3],
48
+ [3, 1, 2, 4, 7, 6],
49
+ [4, 0, 4, 9, 8, 9],
50
+ [8, 6, 6, 4, 6, 1]])
51
+ on rank 0.
52
+ Then on every rank, we will have this tensor.
53
+ If input itself is already replicated, no all-gather will be done.
54
+ 2. Next, we mask the ID which are not stored on that rank.
55
+ For example on rank 0, we store ID [0, 1, 2]. We only keep the ID
56
+ inside the set of numbers. The rest of them will be masked to an extra row.
57
+ The masked matrix will be used for embedding look up and is like:
58
+ tensor([[4, 4, 2, 4, 4, 4],
59
+ [4, 1, 2, 4, 4, 4],
60
+ [4, 0, 4, 4, 4, 4],
61
+ [4, 4, 4, 4, 4, 1]])
62
+ 3. If ``max_norm`` is specified, the extra row guarantees that the mask ID will
63
+ not affect the behavior of weight re-norm.
64
+ 4. The example above only happens in one rank and each rank does a very similar thing.
65
+ For "Mean" mode we need to divide by either column size (2D) or the interval length
66
+ defined by the offset (excluding the row specified in ``padding_idx``).
67
+ We also need to mask the nonexistent row to neg Inf so that negative value does not
68
+ gets wiped out in the "Max" mode.
69
+
70
+ COLWISE SHARDING
71
+ ================
72
+ For col-wise sharding the weight is sharded on dimension 1.
73
+
74
+ The overall algorithm can be best explained with an example. Let's assume
75
+ the dims for input are (4 x 6) and W are (16 x 17) and W is sharded across
76
+ 4 GPUs creating 3 shards of (16 x 5) and 1 shard of (16 x 2).
77
+ The algorithm is as follows:
78
+
79
+ 1. First the input is broadcasted to all ranks, since this is SPMD we
80
+ actually do an all_gather for all the inputs resulting in 4 (4 x 6)
81
+ inputs on each rank.
82
+ 2. Next we perform local embedding bag operation under the given mode by
83
+ apply each input (4 x 6) with the local shard (16 x 5) ((16 x 2) for the last).
84
+ This results in 4 (5 x 4) ((2 x 4) for the last) matrices on each rank.
85
+ We transpose the aggregation result.
86
+ 3. Next, we concatenate these 4 matrices and perform an all2all to share the
87
+ appropriate (5 x 4) or (2 x 4) matrices to each rank.
88
+ 4. Now, each rank receives a (17 x 4) matrix which is basically the
89
+ size of the result we need.
90
+ 5. If placements are not in order any appropriate rearrangement of columns
91
+ are done for the (17 x 4) matrix and finally we transpose the output again.
92
+ 6. If max_norm is specified, we manually sum up the norm and renorm. Because
93
+ the renorm must be in place, we need to override the local_shard to mimic
94
+ this behavior.
95
+ """
96
+ # Validate input params
97
+ _validate_embedding_bag_param(args, kwargs)
98
+
99
+ input = args[0]
100
+ weight = args[1]
101
+ offsets = kwargs.get("offsets")
102
+ per_sample_weights = kwargs.get("per_sample_weights")
103
+ mode = kwargs.get("mode")
104
+ max_norm = kwargs.get("max_norm")
105
+ norm_type = kwargs.get("norm_type")
106
+ include_last_offset = kwargs.get("include_last_offset")
107
+ padding_idx = kwargs.get("padding_idx")
108
+
109
+ local_shard = weight.local_tensor().contiguous()
110
+ sharding_dim = weight._sharding_spec.dim
111
+ world_size = dist.get_world_size(pg)
112
+ rank = dist.get_rank(pg)
113
+ if include_last_offset:
114
+ offsets = offsets[:-1]
115
+
116
+ if sharding_dim == 1:
117
+ output, local_shard = _handle_col_wise_sharding(
118
+ input,
119
+ world_size,
120
+ weight,
121
+ local_shard,
122
+ offsets,
123
+ per_sample_weights,
124
+ mode,
125
+ max_norm,
126
+ norm_type,
127
+ padding_idx,
128
+ pg,
129
+ )
130
+ weight.local_shards()[0].tensor = local_shard
131
+ return output
132
+ elif sharding_dim == 0:
133
+ return _handle_row_wise_sharding(
134
+ input,
135
+ world_size,
136
+ weight,
137
+ local_shard,
138
+ offsets,
139
+ per_sample_weights,
140
+ mode,
141
+ max_norm,
142
+ norm_type,
143
+ padding_idx,
144
+ rank,
145
+ pg,
146
+ )
147
+ else:
148
+ raise RuntimeError(
149
+ f"nn.EmbeddingBag weight sharded on dim {sharding_dim} not supported!"
150
+ )
151
+
152
+
153
+ def _validate_embedding_bag_param(args, kwargs):
154
+ """
155
+ Validate input params of sharded embeddingBag op.
156
+
157
+ Args:
158
+ input: list of ID used for lookup and aggregation.
159
+ weight: sharded weight tensor.
160
+ kwargs: same as normal EmbeddingBag.
161
+
162
+ Return: None.
163
+ """
164
+
165
+ input = args[0]
166
+ weight = args[1]
167
+ offsets = kwargs.get("offsets")
168
+ per_sample_weights = kwargs.get("per_sample_weights")
169
+ mode = kwargs.get("mode")
170
+ max_norm = kwargs.get("max_norm")
171
+ scale_grad_by_freq = kwargs.get("scale_grad_by_freq")
172
+ sparse = kwargs.get("sparse")
173
+ include_last_offset = kwargs.get("include_last_offset")
174
+
175
+ # Validate types
176
+ if not isinstance(input, torch.Tensor):
177
+ raise TypeError("input need to be torch.Tensor")
178
+ if offsets is not None and not isinstance(offsets, torch.Tensor):
179
+ raise TypeError("offsets need to be torch.Tensor")
180
+ if per_sample_weights is not None and not isinstance(
181
+ per_sample_weights, torch.Tensor
182
+ ):
183
+ raise TypeError("per_sample_weights need to be torch.Tensor")
184
+ if not isinstance(weight, ShardedTensor):
185
+ raise TypeError("weight needs to be ShardedTensor")
186
+ if len(input.size()) > 2:
187
+ raise ValueError("Input more than 2 dims not supported")
188
+ weight_size = weight.size()
189
+ if len(weight_size) != 2:
190
+ raise ValueError("Weight needs to have exactly 2 dims")
191
+ if int(torch.min(input).item()) < 0:
192
+ raise ValueError(
193
+ "Index out of range in Input %d %d",
194
+ int(torch.min(input).item()),
195
+ weight_size[1],
196
+ )
197
+ if int(torch.max(input).item()) >= weight_size[0]:
198
+ raise ValueError(
199
+ "Index out of range in Input %d %d",
200
+ int(torch.max(input).item()),
201
+ weight_size[1],
202
+ )
203
+ if offsets is not None and len(input.size()) != 1:
204
+ raise ValueError("Input dimension needs to be exactly 1 dim")
205
+ if len(input.size()) == 1 and offsets is None:
206
+ raise ValueError("offsets is required for 1D input")
207
+ if per_sample_weights is not None and per_sample_weights.size() != input.size():
208
+ raise ValueError(
209
+ f"per_sample_weights size {per_sample_weights.size()} not equal to input size {input.size()}"
210
+ )
211
+ if mode is None:
212
+ mode = "mean"
213
+ if mode not in ["sum", "mean", "max"]:
214
+ raise ValueError(f"mode '{mode}' is not supported")
215
+ if scale_grad_by_freq:
216
+ raise RuntimeError(
217
+ 'nn.Embedding weight sharded with flag on "scale_grad_by_freq" not supported!'
218
+ )
219
+ if sparse:
220
+ raise RuntimeError(
221
+ 'nn.Embedding weight sharded with flag on "sparse" not supported!'
222
+ )
223
+ if include_last_offset and offsets is None:
224
+ raise ValueError('offsets is required for flag "include_last_offset"!')
225
+ if include_last_offset and cast(list[int], offsets)[-1] != input.size(0):
226
+ raise ValueError(
227
+ 'offsets need to have the input size in the end when the flag "include_last_offset" is on!'
228
+ )
229
+
230
+ if max_norm and max_norm <= 0.0:
231
+ raise ValueError('"max_norm" must be larger than zero!')
232
+
233
+ if not isinstance(weight._sharding_spec, ChunkShardingSpec):
234
+ raise ValueError("Only ChunkShardingSpec supported for ShardedTensor ops!")
235
+ if len(weight.local_shards()) != 1:
236
+ raise ValueError("Only one local shard supported!")
237
+
238
+
239
+ def _handle_col_wise_sharding(
240
+ input,
241
+ world_size,
242
+ weight,
243
+ local_shard,
244
+ offsets,
245
+ per_sample_weights,
246
+ mode,
247
+ max_norm,
248
+ norm_type,
249
+ padding_idx,
250
+ pg,
251
+ ):
252
+ """
253
+ Entry-point function to handle the logic of col-wise sharding of weight
254
+ for embeddingBag. (Detailed explanations of the logic can be found in
255
+ the comment for sharded_embedding_bag.)
256
+
257
+ Args:
258
+ input: list of ID used for lookup and aggregation.
259
+ world_size: number of ranks.
260
+ weight: sharded weight tensor.
261
+ local_shard: col-wise shared local weight used for lookup.
262
+ offsets: list of start positions of each bag for 1D input.
263
+ per_sample_weights: weights for weighted sum mode.
264
+ mode: aggregation method of each bag.
265
+ max_norm: If given, each embedding vector with norm larger
266
+ than max_norm is renormalized to have norm max_norm.
267
+ Note: this will modify weight in-place.
268
+ norm_type: The p in the p-norm to compute for the max_norm option.
269
+ padding_idx: If specified, the entries at padding_idx do
270
+ not contribute to the gradient; therefore, the embedding
271
+ vector at padding_idx is not updated during training,
272
+ i.e. it remains as a fixed "pad".
273
+ Note that the embedding vector at padding_idx is
274
+ excluded from the reduction.
275
+ pg: process group.
276
+
277
+ Return:
278
+ output: final result of lookup and aggregation.
279
+ local_shard: col-wise shared local weight used for lookup.
280
+ If max_norm, this will be the renormed weight.
281
+ """
282
+ # allgather the special input of embedding bag first.
283
+ (
284
+ gathered_inputs,
285
+ gathered_per_sample_weights,
286
+ gathered_offsets,
287
+ ) = _all_gather_embedding_bag_input(input, per_sample_weights, offsets, pg)
288
+
289
+ if max_norm is not None:
290
+ # max_norm changes the weight in-place
291
+ local_shard = _handle_max_norm_col_wise(
292
+ max_norm, norm_type, local_shard, input, world_size, gathered_inputs, pg
293
+ )
294
+
295
+ output = _handle_col_wise_sharding_base(
296
+ torch.nn.functional.embedding_bag,
297
+ 1,
298
+ input,
299
+ world_size,
300
+ weight,
301
+ local_shard,
302
+ pg,
303
+ gathered_inputs,
304
+ mode=mode,
305
+ gathered_per_sample_weights=gathered_per_sample_weights,
306
+ gathered_offsets=gathered_offsets,
307
+ padding_idx=padding_idx,
308
+ )
309
+ return (output, local_shard)
310
+
311
+
312
+ def _handle_row_wise_sharding(
313
+ input,
314
+ world_size,
315
+ weight,
316
+ local_shard,
317
+ offsets,
318
+ per_sample_weights,
319
+ mode,
320
+ max_norm,
321
+ norm_type,
322
+ padding_idx,
323
+ rank,
324
+ pg,
325
+ ):
326
+ """
327
+ Entry-point function to handle the logic of row-wise sharding of weight
328
+ for embeddingBag. (Detailed explanations of the logic can be found in
329
+ the comment for sharded_embedding_bag.)
330
+
331
+ Args:
332
+ input: list of ID used for lookup and aggregation.
333
+ world_size: number of ranks.
334
+ weight: sharded weight tensor.
335
+ local_shard: row-wise shared local weight used for lookup.
336
+ offsets: list of start positions of each bag for 1D input.
337
+ per_sample_weights: weights for weighted sum mode.
338
+ mode: aggregation method of each bag.
339
+ max_norm: If given, each embedding vector with norm larger
340
+ than max_norm is renormalized to have norm max_norm.
341
+ Note: this will modify weight in-place.
342
+ norm_type: The p in the p-norm to compute for the max_norm option.
343
+ padding_idx: If specified, the entries at padding_idx do
344
+ not contribute to the gradient; therefore, the embedding
345
+ vector at padding_idx is not updated during training,
346
+ i.e. it remains as a fixed "pad".
347
+ Note that the embedding vector at padding_idx is
348
+ excluded from the reduction.
349
+ rank: # of cuda process.
350
+ pg: process group.
351
+
352
+ Returns:
353
+ gathered_output: final result of lookup and aggregation.
354
+ """
355
+ if input.dim() > 1 and per_sample_weights is None:
356
+ # allgather the inputs first for non Replicated Tensor.
357
+ gather_inp = _all_gather_base_input(input, pg)
358
+ else:
359
+ (
360
+ gathered_inputs,
361
+ gathered_per_sample_weights,
362
+ gathered_offsets,
363
+ ) = _all_gather_embedding_bag_input(input, per_sample_weights, offsets, pg)
364
+ cat_dim = 0 if input.dim() != 1 else -1
365
+ gather_inp = torch.cat(gathered_inputs, dim=cat_dim)
366
+ if per_sample_weights is not None:
367
+ per_sample_weights = torch.cat(gathered_per_sample_weights, dim=cat_dim)
368
+ offset_add = 0 if input.dim() > 1 else input.size(0)
369
+ if offsets is not None:
370
+ offsets_list = torch.cat(
371
+ [gathered_offsets[i] + (offset_add * i) for i in range(pg.size())],
372
+ dim=cat_dim,
373
+ )
374
+
375
+ # Mask the input according to sharding spec.
376
+ lookup_input, padding_local, padding_row = _handle_row_wise_mask(
377
+ gather_inp, padding_idx, weight, world_size, rank
378
+ )
379
+ if mode == "max":
380
+ padding_row[:] = -float("Inf")
381
+
382
+ # When input is a large tensor, the value of weight is changed.
383
+ # This is a walk-around for now. GH issue: #81717.
384
+ if max_norm is not None:
385
+ torch.nn.functional.embedding_bag(
386
+ torch.unique(lookup_input)[:-1],
387
+ local_shard,
388
+ offsets=torch.tensor([0], device=local_shard.device, dtype=torch.long),
389
+ mode=mode,
390
+ per_sample_weights=None,
391
+ max_norm=max_norm,
392
+ norm_type=norm_type,
393
+ padding_idx=padding_local,
394
+ )
395
+ max_norm = None
396
+ result = torch.nn.functional.embedding_bag(
397
+ lookup_input,
398
+ torch.cat([local_shard, padding_row]),
399
+ offsets=offsets_list if offsets is not None else offsets, # type: ignore[possibly-undefined]
400
+ mode=mode if mode != "mean" else "sum",
401
+ per_sample_weights=per_sample_weights,
402
+ max_norm=max_norm,
403
+ norm_type=norm_type,
404
+ padding_idx=padding_local,
405
+ )
406
+
407
+ op = ReduceOp.SUM if mode != "max" else ReduceOp.MAX
408
+ # TODO: Make the result a PartialTensor and move the logic below there.
409
+ local_shards = result.chunk(pg.size())
410
+ result = reduce_scatter(
411
+ torch.empty_like(local_shards[0]),
412
+ list(local_shards),
413
+ op=op,
414
+ group=pg,
415
+ )
416
+
417
+ # For Mean, we cannot do the division until very end because the sum of means
418
+ # not equal to the mean of sum. (Divisor is different)
419
+ if mode == "mean":
420
+ if input.dim() > 1:
421
+ padding_idx = padding_idx if padding_idx is not None else -1
422
+ split_sizes = torch.sum(
423
+ torch.ne(input, padding_idx), dim=-1, dtype=local_shard.dtype
424
+ )
425
+ else:
426
+ split_sizes = torch.cat(
427
+ (
428
+ offsets[1 : offsets.size(0)] - offsets[0:-1],
429
+ (input.size(0) - offsets[-1]).unsqueeze(0),
430
+ ),
431
+ dim=-1,
432
+ )
433
+ return torch.div(result, split_sizes.unsqueeze(1))
434
+
435
+ # Return the appropriate local result.
436
+ return result
437
+
438
+
439
+ def _all_gather_embedding_bag_input(input, per_sample_weights, offsets, pg):
440
+ """
441
+ In case we need to gather input and all other parameters of embeddingBag
442
+ ops, we need to stack all input together to perform ``all_gather``
443
+ collective communication just once.
444
+
445
+ Note that since offsets does not share the same size as input and
446
+ is always smaller than input, we resize it during the communication.
447
+
448
+ Args:
449
+ input: tensor to be applied op on.
450
+ per_sample_weights: weights for weighted sum mode.
451
+ offsets: when input is 1D. offsets determines the starting
452
+ index position of each bag (sequence) in input.
453
+ pg: process group.
454
+
455
+ Returns:
456
+ gathered_inputs: list of input tensor gathered from each rank.
457
+ gathered_per_sample_weights: list of per_sample_weights from each rank.
458
+ gathered_offsets: list of offsets from each rank.
459
+ """
460
+ input_to_gather = [input]
461
+ if per_sample_weights is not None:
462
+ input_to_gather.append(per_sample_weights)
463
+ if offsets is not None:
464
+ input_to_gather.append(offsets.clone().resize_(input.size()))
465
+ gathered_inputs = all_gather(torch.stack(input_to_gather), group=pg)
466
+
467
+ gathered_per_sample_weights = None
468
+ if per_sample_weights is not None:
469
+ gathered_per_sample_weights = [t[1] for t in gathered_inputs]
470
+ gathered_offsets = None
471
+ if offsets is not None:
472
+ idx = 2 if per_sample_weights is not None else 1
473
+ gathered_offsets = [
474
+ t[idx].resize_(offsets.size()).to(offsets.dtype) for t in gathered_inputs
475
+ ]
476
+ gathered_inputs = [t[0].to(input.dtype) for t in gathered_inputs]
477
+ return gathered_inputs, gathered_per_sample_weights, gathered_offsets
western_ai/usa/pytorch/torch/distributed/_sharded_tensor/__init__.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Keep old package for BC purposes, this file should be removed once
2
+ # everything moves to the `torch.distributed._shard` package.
3
+ import sys
4
+ import warnings
5
+
6
+ import torch
7
+ from torch.distributed._shard.sharded_tensor import * # noqa: F403
8
+
9
+
10
+ with warnings.catch_warnings():
11
+ warnings.simplefilter("always")
12
+ warnings.warn(
13
+ "`torch.distributed._sharded_tensor` will be deprecated, "
14
+ "use `torch.distributed._shard.sharded_tensor` instead",
15
+ DeprecationWarning,
16
+ stacklevel=2,
17
+ )
18
+
19
+ sys.modules["torch.distributed._sharded_tensor"] = (
20
+ torch.distributed._shard.sharded_tensor
21
+ )
western_ai/usa/pytorch/torch/distributed/_sharding_spec/__init__.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Keep old package for BC purposes, this file should be removed once
2
+ # everything moves to the `torch.distributed._shard` package.
3
+ import sys
4
+ import warnings
5
+
6
+ import torch
7
+ from torch.distributed._shard.sharding_spec import * # noqa: F403
8
+
9
+
10
+ with warnings.catch_warnings():
11
+ warnings.simplefilter("always")
12
+ warnings.warn(
13
+ "`torch.distributed._sharding_spec` will be deprecated, "
14
+ "use `torch.distributed._shard.sharding_spec` instead",
15
+ DeprecationWarning,
16
+ stacklevel=2,
17
+ )
18
+
19
+ import torch.distributed._shard.sharding_spec as _sharding_spec
20
+
21
+
22
+ sys.modules["torch.distributed._sharding_spec"] = _sharding_spec
western_ai/usa/pytorch/torch/distributed/_state_dict_utils.py ADDED
@@ -0,0 +1,829 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import copy
3
+ import io
4
+ import math
5
+ import weakref
6
+ from collections.abc import Callable, Mapping, MutableMapping
7
+ from typing import Any, cast, NamedTuple, TYPE_CHECKING
8
+
9
+ import torch
10
+ import torch.cuda._pin_memory_utils as pin_memory_utils
11
+ import torch.distributed as dist
12
+ import torch.nn.functional as F
13
+ from torch.distributed._functional_collectives import AsyncCollectiveTensor
14
+
15
+
16
+ if dist.is_available() or TYPE_CHECKING:
17
+ from torch.distributed import distributed_c10d
18
+ from torch.distributed._shard.sharded_tensor import ShardedTensor
19
+ from torch.distributed.tensor import distribute_tensor, DTensor, Replicate
20
+ from torch.distributed.tensor._utils import compute_local_shape_and_global_offset
21
+
22
+
23
+ def _identity_func(
24
+ obj: torch.Tensor,
25
+ pg: dist.ProcessGroup | None,
26
+ device: torch.device | None,
27
+ companion_obj: Any,
28
+ ) -> torch.Tensor:
29
+ return obj
30
+
31
+
32
+ def _all_gather_sharded_tensor(
33
+ sharded_tensor: "ShardedTensor",
34
+ pg: dist.ProcessGroup | None = None,
35
+ device: torch.device | None = None,
36
+ ) -> torch.Tensor:
37
+ if pg is None:
38
+ pg = distributed_c10d._get_default_group()
39
+ world_size = dist.get_world_size(pg)
40
+ shards = sharded_tensor.local_shards()
41
+ dim_0_size = sharded_tensor.size()[0] # type: ignore[index]
42
+ tensor_numel = sharded_tensor.size().numel() # type: ignore[union-attr]
43
+ chunk_size = math.ceil(dim_0_size / world_size) * tensor_numel // dim_0_size
44
+ pg_device = (
45
+ distributed_c10d._get_pg_default_device(pg) if device is None else device
46
+ )
47
+ if shards:
48
+ local_tensor = shards[0].tensor.flatten()
49
+ if local_tensor.device.type != pg_device.type:
50
+ local_tensor = local_tensor.to(pg_device)
51
+ num_padding = chunk_size - local_tensor.numel()
52
+ if num_padding > 0:
53
+ local_tensor = F.pad(local_tensor, [0, num_padding])
54
+ else:
55
+ local_tensor = torch.zeros(
56
+ chunk_size, dtype=sharded_tensor.dtype, device=pg_device
57
+ )
58
+
59
+ tensor = torch.empty(
60
+ chunk_size * world_size,
61
+ dtype=local_tensor.dtype,
62
+ device=pg_device,
63
+ )
64
+ dist.all_gather_single(tensor, local_tensor, group=pg)
65
+
66
+ tensor = tensor.narrow(0, 0, tensor_numel).reshape(sharded_tensor.size())
67
+ return tensor
68
+
69
+
70
+ class CompanionMismatch(Exception):
71
+ pass
72
+
73
+
74
+ def _iterate_state_dict(
75
+ iter_object: Any,
76
+ sharded_tensor_func: Callable,
77
+ dtensor_func: Callable,
78
+ tensor_func: Callable,
79
+ *,
80
+ pg: dist.ProcessGroup | None = None,
81
+ device: torch.device | None = None,
82
+ cpu_offload: bool = False,
83
+ companion_obj: Any = None,
84
+ ranks_only: tuple[int, ...] = (),
85
+ type_check: bool = True,
86
+ non_blocking: bool = True,
87
+ ) -> dict[str, Any]:
88
+ """Iterate through the state dict, applying the given functions to each tensor type.
89
+
90
+ Args:
91
+ iter_object (Any): the target state_dict.
92
+ sharded_tensor_func (Callable): the function to apply to ShardedTensor
93
+ dtensor_func (Callable): the function to apply to DTensor
94
+ tensor_func (Callable): the function to apply to Tensor
95
+ pg (Optional[dist.ProcessGroup]): process group passed to tensor functions
96
+ device (Optional[torch.device]): device passed to tensor functions
97
+ cpu_offload (bool): whether to offload the tensors to CPU memory. This option is ignored
98
+ if a companion_obj is supplied.
99
+ companion_obj (Any): A companion object to the state dict. If this object
100
+ is supplied, we attempt to copy the tensor to the companion object.
101
+ ranks_only (Tuple[int, ...]): if this tuple is empty, all ranks will
102
+ have the same state_dicts. Otherwise only ranks that in ``ranks_only``
103
+ have the same state_dicts. Other ranks will get empty state_dicts.
104
+ type_check (bool): check if the instance data type is a supported type
105
+ that can be saved by DCP. The current supported data types are
106
+ torch.Tensor, DTensor, int, float, str, list, dict, None.
107
+ non_blocking (bool): whether to use non-blocking copy when copying to the companion object.
108
+ """
109
+ # TODO: should we use pytree?
110
+ cpu_device = torch.device("cpu")
111
+ if isinstance(iter_object, ShardedTensor):
112
+ ret = sharded_tensor_func(iter_object, pg, device, companion_obj)
113
+ elif isinstance(iter_object, DTensor):
114
+ ret = dtensor_func(iter_object, pg, device, companion_obj)
115
+ elif isinstance(iter_object, torch.Tensor):
116
+ ret = tensor_func(iter_object, pg, device, companion_obj)
117
+ elif (
118
+ isinstance(iter_object, (int, float, str, bytes, io.BytesIO))
119
+ or iter_object is None
120
+ ):
121
+ ret = iter_object
122
+ elif isinstance(iter_object, dict):
123
+ if companion_obj is not None and (
124
+ not isinstance(companion_obj, dict)
125
+ or set(companion_obj.keys()) != set(iter_object.keys())
126
+ ):
127
+ msg = (
128
+ ""
129
+ if isinstance(companion_obj, dict)
130
+ else f"{set(companion_obj.keys())=} {set(iter_object.keys())=}"
131
+ )
132
+ raise CompanionMismatch(msg)
133
+
134
+ ret = {
135
+ key: _iterate_state_dict(
136
+ value,
137
+ sharded_tensor_func,
138
+ dtensor_func,
139
+ tensor_func,
140
+ pg=pg,
141
+ device=device,
142
+ cpu_offload=cpu_offload,
143
+ companion_obj=companion_obj[key] if companion_obj is not None else None,
144
+ ranks_only=ranks_only,
145
+ type_check=type_check,
146
+ non_blocking=non_blocking,
147
+ )
148
+ for key, value in iter_object.items()
149
+ }
150
+ elif isinstance(iter_object, (list, tuple)):
151
+ if companion_obj is not None and (
152
+ not isinstance(companion_obj, (list, tuple))
153
+ or len(companion_obj) != len(iter_object)
154
+ ):
155
+ raise CompanionMismatch
156
+
157
+ ret = [
158
+ _iterate_state_dict(
159
+ v,
160
+ sharded_tensor_func,
161
+ dtensor_func,
162
+ tensor_func,
163
+ pg=pg,
164
+ device=device,
165
+ cpu_offload=cpu_offload,
166
+ companion_obj=companion_obj[idx] if companion_obj is not None else None,
167
+ ranks_only=ranks_only,
168
+ type_check=type_check,
169
+ non_blocking=non_blocking,
170
+ )
171
+ for idx, v in enumerate(iter_object)
172
+ ]
173
+ if isinstance(iter_object, tuple):
174
+ ret = tuple(ret)
175
+ elif not type_check:
176
+ ret = copy.deepcopy(iter_object)
177
+ else:
178
+ raise ValueError(f"Unexpected value type {type(iter_object)}")
179
+
180
+ if not ranks_only or dist.get_rank(pg) in ranks_only:
181
+ if isinstance(ret, torch.Tensor):
182
+ if cpu_offload and companion_obj is None:
183
+ ret = ret.to(cpu_device)
184
+
185
+ if companion_obj is not None:
186
+ if isinstance(companion_obj, DTensor):
187
+ if not isinstance(ret, DTensor):
188
+ raise AssertionError(
189
+ "ret must be a DTensor when companion_obj is a DTensor"
190
+ )
191
+ companion_obj._local_tensor.copy_(
192
+ ret._local_tensor, non_blocking=non_blocking
193
+ )
194
+ elif isinstance(companion_obj, ShardedTensor):
195
+ if not isinstance(ret, ShardedTensor):
196
+ raise AssertionError(
197
+ "ret must be a ShardedTensor when companion_obj is a ShardedTensor"
198
+ )
199
+ for idx, shard in enumerate(companion_obj.local_shards()):
200
+ shard.tensor.copy_(
201
+ ret.local_shards()[idx].tensor, non_blocking=non_blocking
202
+ )
203
+ else:
204
+ companion_obj.copy_(ret, non_blocking=non_blocking)
205
+ ret = companion_obj
206
+ else:
207
+ ret = {} if isinstance(ret, dict) else None
208
+
209
+ # pyrefly: ignore [bad-return]
210
+ return ret
211
+
212
+
213
+ def _gather_state_dict(
214
+ state_dict: dict[str, Any],
215
+ *,
216
+ pg: dist.ProcessGroup | None = None,
217
+ device: torch.device | None = None,
218
+ cpu_offload: bool = False,
219
+ ranks_only: tuple[int, ...] = (),
220
+ type_check: bool = True,
221
+ ) -> dict[str, Any]:
222
+ """
223
+ Given a state_dict, this API gathers all the ShardedTensors or DTensors in
224
+ the state_dict.
225
+
226
+
227
+ Args:
228
+ state_dict (Dict[str, Any]): the target sharded state_dict.
229
+ pg (Optional[dist.ProcessGroup]): the process group that is used to
230
+ gather ShardedTensor. Note that gathering a DTensor will use
231
+ the DeviceMesh. So this argument will be ignored when gathering a
232
+ DTensor.
233
+ device: (Optional[torch.device]): the device that is used to
234
+ perform allgather for ShardedTensor. Note that gathering a DTensor
235
+ will use the DeviceMesh. So this argument will be ignored when
236
+ gathering a DTensor.
237
+ cpu_offload (bool): whether to offload the tensors to CPU memory. The
238
+ default value is False.
239
+ ranks_only: (Tuple[int, ...]): if this tuple is empty, all ranks will
240
+ have the same state_dicts. Otherwise only ranks that in ``ranks_only``
241
+ have the same state_dicts. Other ranks will get empty state_dicts.
242
+ type_check: (bool): check if the instance data type is a supported type
243
+ that can be saved by DCP. The current supported data types are
244
+ torch.Tensor, DTensor, int, float, str, list, dict, None.
245
+
246
+ Returns:
247
+ The gathered state dictionary.
248
+ """
249
+
250
+ def sharded_tensor_func(value, pg, device, companion_obj):
251
+ # ShardedTensor does not seem to record the original device type.
252
+ # So if the tensor is moved to CPU, we won't know the original type.
253
+ # As a result, we have to rely on the user to tell us the correct one.
254
+ cpu_device = torch.device("cpu")
255
+ output_tensor = _all_gather_sharded_tensor(value, pg, device)
256
+ local_shard_device = (
257
+ value.local_shards()[0].tensor.device
258
+ if value.local_shards()
259
+ else cpu_device
260
+ )
261
+ if output_tensor.device != local_shard_device:
262
+ value = output_tensor.to(local_shard_device)
263
+ else:
264
+ value = output_tensor
265
+ return value
266
+
267
+ def dtensor_func(value, pg, device, companion_obj):
268
+ if value.device != value.device_mesh.device_type:
269
+ value = value.to(value.device_mesh.device_type)
270
+ # FSDP all_gather: [Shard(0)] -> [Replicate()]
271
+ # HSDP all_gather: [Replicate(), Shard(0)] -> [Replicate(), Replicate()]
272
+ # 2D FSDP + TP all_gather:
273
+ # - [Shard(0), Shard(n)] -> [Replicate(), Replicate()]
274
+ # - [Shard(0), Replicate()] -> [Replicate(), Replicate()]
275
+ placements = [Replicate() for _ in value.placements]
276
+ value = value.redistribute(
277
+ device_mesh=value.device_mesh,
278
+ placements=placements,
279
+ )
280
+ # Call `wait()` to force the tensor to be synchronous with respect
281
+ # to the main stream.
282
+ # See the discussion in https://github.com/pytorch/pytorch/pull/117799.
283
+ value = value.to_local()
284
+ if isinstance(value, AsyncCollectiveTensor):
285
+ value = value.wait()
286
+ return value
287
+
288
+ return _iterate_state_dict(
289
+ state_dict,
290
+ sharded_tensor_func,
291
+ dtensor_func,
292
+ _identity_func,
293
+ pg=pg,
294
+ device=device,
295
+ cpu_offload=cpu_offload,
296
+ ranks_only=ranks_only,
297
+ type_check=type_check,
298
+ )
299
+
300
+
301
+ def _offload_state_dict_to_cpu(
302
+ state_dict: dict[str, Any],
303
+ *,
304
+ ranks_only: tuple[int, ...] = (),
305
+ type_check: bool = True,
306
+ ) -> dict[str, Any]:
307
+ """
308
+ Given a state_dict, this API offload all the tensors to CPU memory.
309
+
310
+ Args:
311
+ state_dict (Dict[str, Any]): the target state_dict.
312
+ pg (Optional[dist.ProcessGroup]): the process group that is used to
313
+ gather ShardedTensor. Note that gathering a DTensor will use
314
+ the DeviceMesh. So this argument will be ignored when gathering a
315
+ DTensor.
316
+ ranks_only: (Tuple[int, ...]): if this tuple is empty, all ranks will
317
+ have the same state_dicts. Otherwise only ranks that in ``ranks_only``
318
+ have the same state_dicts. Other ranks will get empty state_dicts.
319
+ type_check: (bool): check if the instance data type is a supported type
320
+ that can be saved by DCP. The current supported data types are
321
+ torch.Tensor, DTensor, int, float, str, list, dict, None.
322
+
323
+ Returns:
324
+ The gathered state dictionary.
325
+ """
326
+
327
+ ret = _iterate_state_dict(
328
+ state_dict,
329
+ _identity_func,
330
+ _identity_func,
331
+ _identity_func,
332
+ pg=None,
333
+ device=None,
334
+ cpu_offload=True,
335
+ ranks_only=ranks_only,
336
+ type_check=type_check,
337
+ )
338
+ return ret
339
+
340
+
341
+ @torch.no_grad()
342
+ def _copy_state_dict(
343
+ state_dict: dict[str, Any],
344
+ copy_state_dict: dict[str, Any],
345
+ non_blocking: bool = False,
346
+ type_check: bool = True,
347
+ ) -> dict[str, Any]:
348
+ """
349
+ Copies all tensors in a given state dict into a different state_dict with the
350
+ same structure. Additionally, a copied state dict with the same value references
351
+ is returned. Editing the keys on this state dict will not affect the
352
+ passed in copy_state_dict (but the value references are the same).
353
+
354
+ .. warning::
355
+ It is expected by this function that state_dict and copy_state_dict share
356
+ the same structure and data types.
357
+
358
+ .. warning::
359
+ The current supported data types are
360
+ torch.Tensor, DTensor, int, float, str, list, dict, None.
361
+
362
+ Args:
363
+ state_dict (Dict[str, Any]): the target state_dict.
364
+ copy_state_dict (Dict[str, Any]):
365
+ The state dict we are copying into. This state_dict must have exactly
366
+ the same structure as the source `state_dict`.
367
+ non_blocking: (bool): Whether copy ops should be performed asynchronously
368
+ type_check (bool): check if the instance data type is a supported type
369
+ that can be saved by DCP. The current supported data types are
370
+ torch.Tensor, DTensor, int, float, str, list, dict, None.
371
+
372
+ Returns:
373
+ State Dict copy
374
+ """
375
+
376
+ return _iterate_state_dict(
377
+ state_dict,
378
+ _identity_func,
379
+ _identity_func,
380
+ _identity_func,
381
+ pg=None,
382
+ device=None,
383
+ cpu_offload=False,
384
+ ranks_only=(),
385
+ companion_obj=copy_state_dict,
386
+ type_check=type_check,
387
+ non_blocking=non_blocking,
388
+ )
389
+
390
+
391
+ @torch.no_grad()
392
+ def _create_cpu_state_dict(
393
+ state_dict: dict[str, Any], pin_memory: bool = False, share_memory: bool = False
394
+ ) -> dict[str, Any]:
395
+ """
396
+ Given a state_dict, create another state_dict with the same structure and elements.
397
+ However, all tensors in the returned state_dict are new tensors on CPU. These
398
+ tensors can be placed on pin_memory or share_memory based on the provided arguments.
399
+
400
+ .. warning::
401
+ Setting both `pin_memory` and `share_memory` to True significantly increases the
402
+ latency of this method because of the nuances which require us to register memory
403
+ as pinned directly as opposed to relying on the pin_memory cache allocator. This
404
+ option should only be used for long lived tensors which are required to be shared.
405
+ This is not the case as long as at least one of `pin_memory` or `share_memory` is
406
+ set to False.
407
+
408
+ """
409
+
410
+ def tensor_func(
411
+ obj: torch.Tensor,
412
+ pg: dist.ProcessGroup | None,
413
+ device: torch.device | None,
414
+ _: Any,
415
+ ) -> torch.Tensor:
416
+ if len(obj.size()) == 0:
417
+ return torch.tensor(0, dtype=obj.dtype)
418
+
419
+ # sometimes, a tensor might have non-zero size and 0 numel. In this case, pinning memory will fail
420
+ # so we take a best guess at how to replicate the tensor below to maintain symmetry in the returned
421
+ # state dict.
422
+ if obj.numel() == 0 or obj.data_ptr() == 0:
423
+ t = torch.zeros_like(obj, device="cpu")
424
+ if share_memory:
425
+ t = t.share_memory_()
426
+ return t
427
+
428
+ if share_memory:
429
+ t = torch.empty(*tuple(obj.size()), dtype=obj.dtype)
430
+ t = t.share_memory_()
431
+ if pin_memory:
432
+ pin_memory_utils.pin_memory(t.data_ptr(), t.numel() * t.element_size())
433
+ weakref.finalize(t, pin_memory_utils.unpin_memory, t.data_ptr())
434
+
435
+ return t
436
+ elif pin_memory:
437
+ return torch.empty(*tuple(obj.size()), dtype=obj.dtype).pin_memory()
438
+ else:
439
+ return torch.empty(*tuple(obj.size()), dtype=obj.dtype)
440
+
441
+ def dtensor_func(
442
+ obj: DTensor,
443
+ pg: dist.ProcessGroup | None,
444
+ device: torch.device | None,
445
+ _: Any,
446
+ ) -> DTensor:
447
+ if len(obj.size()) == 0:
448
+ return obj
449
+
450
+ if obj.device != torch.device("cpu"):
451
+ ret = cast(DTensor, obj.to(device="cpu"))
452
+ else:
453
+ ret = copy.deepcopy(obj)
454
+ ret._local_tensor = tensor_func(ret._local_tensor, pg, device, None)
455
+ return ret
456
+
457
+ def sharded_tensor_func(
458
+ obj: ShardedTensor,
459
+ pg: dist.ProcessGroup | None,
460
+ device: torch.device | None,
461
+ _: Any,
462
+ ) -> ShardedTensor:
463
+ if not obj.local_shards():
464
+ return obj
465
+
466
+ if obj.device != torch.device("cpu"):
467
+ ret = obj.to(device="cpu")
468
+ else:
469
+ ret = copy.deepcopy(obj)
470
+
471
+ for shards in ret.local_shards():
472
+ shards.tensor = tensor_func(shards.tensor, pg, device, None)
473
+
474
+ return ret
475
+
476
+ ret = _iterate_state_dict(
477
+ state_dict,
478
+ sharded_tensor_func,
479
+ dtensor_func,
480
+ tensor_func,
481
+ pg=None,
482
+ device=None,
483
+ cpu_offload=False,
484
+ ranks_only=(),
485
+ type_check=False,
486
+ )
487
+ return ret
488
+
489
+
490
+ def _check_state_dict_similarity(
491
+ state_dict: dict[str, Any],
492
+ compared_state_dict: dict[str, Any],
493
+ ) -> bool:
494
+ """
495
+ Given two state_dicts, check if the structures are the same. And
496
+ if a [key, tensor] pair exist in one state_dict there must be
497
+ a corresponding pair, [key, other_tensor], in the other state_dict,
498
+ where tensor and other_tensor have the same size and dtype.
499
+
500
+ Return the check result.
501
+ """
502
+
503
+ def tensor_func(
504
+ obj: torch.Tensor,
505
+ pg: dist.ProcessGroup | None,
506
+ device: torch.device | None,
507
+ companion_obj: Any,
508
+ ) -> torch.Tensor:
509
+ if companion_obj.dtype != obj.dtype or companion_obj.size() != obj.size():
510
+ raise CompanionMismatch
511
+ return obj
512
+
513
+ try:
514
+ _iterate_state_dict(
515
+ state_dict,
516
+ _identity_func,
517
+ _identity_func,
518
+ tensor_func,
519
+ pg=None,
520
+ device=None,
521
+ cpu_offload=False,
522
+ ranks_only=(),
523
+ companion_obj=compared_state_dict,
524
+ type_check=False,
525
+ )
526
+ except CompanionMismatch:
527
+ return False
528
+
529
+ return True
530
+
531
+
532
+ class _TensorInfo(NamedTuple):
533
+ size: torch.Size
534
+ dtype: torch.dtype
535
+
536
+
537
+ def _broadcast_tensors(
538
+ full_state_dict: dict[str, Any],
539
+ local_state_dict: dict[str, Any],
540
+ keys: list[str],
541
+ device: torch.device,
542
+ pg: dist.ProcessGroup | None = None,
543
+ ) -> None:
544
+ if pg is None:
545
+ pg = dist.distributed_c10d._get_default_group()
546
+ pg_device = (
547
+ device
548
+ if device.type in {pg_device.type for pg_device in pg._device_types}
549
+ else pg._device_types[0]
550
+ )
551
+
552
+ tensors: list[torch.Tensor] = []
553
+ for key in keys:
554
+ if dist.get_rank() == 0:
555
+ full_state = full_state_dict[key]
556
+ if not isinstance(full_state, torch.Tensor):
557
+ raise AssertionError("full_state must be a torch.Tensor")
558
+ full_tensor = full_state.detach().to(pg_device)
559
+ else:
560
+ tensor_info = full_state_dict[key]
561
+ full_tensor = torch.empty(
562
+ size=tensor_info.size,
563
+ device=pg_device,
564
+ dtype=tensor_info.dtype,
565
+ )
566
+
567
+ tensors.append(full_tensor)
568
+
569
+ if (local_state := local_state_dict.get(key)) is None:
570
+ continue
571
+
572
+ local_state_dict[key] = (
573
+ (local_state, full_tensor)
574
+ if isinstance(local_state, DTensor)
575
+ else full_tensor
576
+ )
577
+
578
+ if len(tensors) > 1:
579
+ dist._broadcast_coalesced(pg, tensors, 500, 0)
580
+ else:
581
+ dist.broadcast(tensors[0], src=0, group=pg)
582
+
583
+ if pg_device != device:
584
+ for key, full_tensor in zip(keys, tensors):
585
+ if (local_state := local_state_dict.get(key)) is not None:
586
+ local_state_dict[key] = (
587
+ (local_state[0], full_tensor.to(device))
588
+ if (
589
+ isinstance(local_state, tuple)
590
+ and isinstance(local_state[0], DTensor)
591
+ )
592
+ else full_tensor.to(device)
593
+ )
594
+
595
+ _distribute_tensors(local_state_dict, keys, device, pg)
596
+
597
+
598
+ def _distribute_tensors(
599
+ local_state_dict: dict[str, Any],
600
+ keys: list[str],
601
+ device: torch.device,
602
+ pg: dist.ProcessGroup | None = None,
603
+ ) -> None:
604
+ if pg is None:
605
+ pg = dist.distributed_c10d._get_default_group()
606
+ for key in keys:
607
+ _local_state = local_state_dict.get(key)
608
+ if _local_state is None or torch.is_tensor(_local_state):
609
+ continue
610
+
611
+ local_state = _local_state[0]
612
+ full_tensor = _local_state[1]
613
+
614
+ shape, offset = compute_local_shape_and_global_offset(
615
+ full_tensor.shape, local_state.device_mesh, local_state.placements
616
+ )
617
+ slices = [
618
+ slice(cur_offset, cur_offset + cur_shape)
619
+ for cur_shape, cur_offset in zip(shape, offset)
620
+ ]
621
+ if local_state.is_meta:
622
+ # Use .clone() here rather than view to clone and return only the sliced portion, minimizing memory access and cost.
623
+ local_tensor = full_tensor[tuple(slices)].detach().clone()
624
+ # TODO: currently, we cannot handle strided sharding if the dp dimension is not even. For example,
625
+ # one of the case that is not yet supported is when placements = (Shard(0), _StridedShard(0, sf=2)).
626
+ ret = DTensor.from_local(
627
+ local_tensor,
628
+ local_state.device_mesh,
629
+ local_state.placements,
630
+ shape=local_state.shape,
631
+ stride=local_state.stride(),
632
+ )
633
+ else:
634
+ ret = local_state
635
+ # Copy full_tensor[slices] into local_state.to_local() to reduce memory footprint.
636
+ ret.to_local().copy_(full_tensor[tuple(slices)])
637
+ local_state_dict[key] = ret
638
+
639
+
640
+ def _broadcast_state_dict(
641
+ full_state_dict: dict[str, Any],
642
+ local_state_dict: dict[str, Any],
643
+ device: torch.device,
644
+ pg: dist.ProcessGroup | None = None,
645
+ strict: bool = False,
646
+ cpu_offload: bool = False,
647
+ ) -> None:
648
+ # Broadcast from rank0's `full_state_dict` to all ranks' `local_state_dict`.
649
+ # If strict is True, any keys in `local_state_dict` but not in `full_state_dict`
650
+ # will be removed from `local_state_dict`.
651
+ ret = {}
652
+ if dist.get_rank() == 0:
653
+ for key, value in full_state_dict.items():
654
+ if not torch.is_tensor(value):
655
+ ret[key] = value
656
+ elif value.dim() == 0:
657
+ ret[key] = value.cpu()
658
+ else:
659
+ ret[key] = _TensorInfo(value.size(), value.dtype)
660
+
661
+ broadcast_list = [ret]
662
+ dist.broadcast_object_list(broadcast_list, src=0, group=pg)
663
+ ret = broadcast_list[0]
664
+ # Gather values
665
+ keys = []
666
+ local_state_dict_keys = set(local_state_dict.keys())
667
+ global_keys = set()
668
+ for key, value in ret.items():
669
+ global_keys.add(key)
670
+ if not isinstance(value, _TensorInfo):
671
+ if key in local_state_dict:
672
+ local_state_dict[key] = value
673
+ continue
674
+
675
+ if dist.get_rank() == 0:
676
+ ret[key] = full_state_dict[key]
677
+
678
+ keys.append(key)
679
+ # Broadcast every tensor to avoid OOM for now.
680
+ if len(keys) >= 1:
681
+ _broadcast_tensors(ret, local_state_dict, keys, device, pg)
682
+ if cpu_offload:
683
+ for key in keys:
684
+ local_state_dict[key] = local_state_dict[key].cpu()
685
+ keys.clear()
686
+
687
+ if strict:
688
+ if missing_keys := (local_state_dict_keys - global_keys):
689
+ for key in missing_keys:
690
+ local_state_dict.pop(key)
691
+
692
+ if keys:
693
+ _broadcast_tensors(ret, local_state_dict, keys, device, pg)
694
+ if cpu_offload:
695
+ for key in keys:
696
+ local_state_dict[key] = local_state_dict[key].cpu()
697
+
698
+
699
+ def _distribute_state_dict(
700
+ full_state_dict: dict[str, Any],
701
+ local_state_dict: dict[str, Any],
702
+ device: torch.device,
703
+ pg: dist.ProcessGroup | None = None,
704
+ ) -> None:
705
+ # Full_state_dict = True, broadcast_from_rank0 = False here. Each rank has
706
+ # full_state_dict. Skip the broadcast in ``_broadcast_state_dict`` and
707
+ # distribute tensors in each rank
708
+ for key, value in full_state_dict.items():
709
+ if key not in full_state_dict:
710
+ continue
711
+ if not torch.is_tensor(value):
712
+ local_state_dict[key] = value
713
+ elif value.dim() == 0:
714
+ local_state_dict[key] = value.cpu()
715
+ else:
716
+ if not isinstance(value, torch.Tensor):
717
+ raise AssertionError("value must be a torch.Tensor")
718
+ local_state = local_state_dict.get(key)
719
+ if local_state is None:
720
+ continue
721
+ elif isinstance(local_state, DTensor):
722
+ local_state_dict[key] = distribute_tensor(
723
+ value.detach().to(device),
724
+ local_state.device_mesh,
725
+ local_state.placements,
726
+ )
727
+ else:
728
+ local_state_dict[key] = value.detach().to(device)
729
+
730
+
731
+ # These APIs are from torch.distributed.checkpoint.
732
+ # TODO: We should consolidate the code here as some not all modules can depend on
733
+ # DCP.
734
+ PATH_ITEM = str | int
735
+ OBJ_PATH = tuple[PATH_ITEM, ...]
736
+ FLATTEN_MAPPING = dict[str, OBJ_PATH]
737
+ STATE_DICT_TYPE = dict[str, Any]
738
+ CONTAINER_TYPE = MutableMapping[PATH_ITEM, Any]
739
+
740
+
741
+ def _traverse_state_dict(
742
+ state_dict: STATE_DICT_TYPE,
743
+ visitor: Callable[[OBJ_PATH, Any], None],
744
+ ) -> None:
745
+ """
746
+ Invoke ``visitor`` for each value recursively in ``state_dict``.
747
+ Mapping, list, and tuple will be flattened and other value types are treated
748
+ as the terminal values and will invoke ``visitor``.
749
+ """
750
+
751
+ def _traverse_obj(path: OBJ_PATH, value: Any) -> None:
752
+ if isinstance(value, Mapping):
753
+ for k, v in value.items():
754
+ _traverse_obj(path + (str(k),), v)
755
+ elif isinstance(value, (list, tuple)):
756
+ for i, v in enumerate(value):
757
+ _traverse_obj(path + (i,), v)
758
+ else:
759
+ visitor(path, value)
760
+
761
+ for key, value in state_dict.items():
762
+ _traverse_obj((str(key),), value)
763
+
764
+
765
+ def _flatten_state_dict(
766
+ state_dict: STATE_DICT_TYPE,
767
+ ) -> tuple[STATE_DICT_TYPE, FLATTEN_MAPPING]:
768
+ """
769
+ Flatten ``state_dict`` made of nested dicts and lists into a top level dictionary.
770
+
771
+ Use ``unflatten_state_dict`` to revert this process.
772
+ Returns:
773
+ A tuple with the flatten state_dict and a mapping from original to new state_dict.
774
+ N.B. The new keys are derived from the object paths, joined by dot.
775
+ For example: ``{ 'a': {'b':...}}`` results in the key `a.b`.
776
+ """
777
+ flattened: STATE_DICT_TYPE = {}
778
+ mappings: FLATTEN_MAPPING = {}
779
+
780
+ def flat_copy(path: OBJ_PATH, value: Any) -> None:
781
+ new_fqn = ".".join(map(str, path))
782
+ if new_fqn in flattened:
783
+ raise ValueError(f"duplicated flatten key {new_fqn}")
784
+ flattened[new_fqn] = value
785
+ mappings[new_fqn] = path
786
+
787
+ _traverse_state_dict(state_dict, flat_copy)
788
+ return flattened, mappings
789
+
790
+
791
+ def _set_element(root_dict: STATE_DICT_TYPE, path: OBJ_PATH, value: Any) -> None:
792
+ """Set ``value`` in ``root_dict`` along the ``path`` object path."""
793
+ cur_container = cast(CONTAINER_TYPE, root_dict)
794
+
795
+ def extend_list(lst: list[Any], idx: int) -> None:
796
+ while len(lst) <= idx:
797
+ lst.append(None)
798
+
799
+ for i in range(1, len(path)):
800
+ prev_key = path[i - 1]
801
+ key = path[i]
802
+ def_val: CONTAINER_TYPE | list[Any] = {} if type(key) is str else []
803
+
804
+ if isinstance(cur_container, Mapping):
805
+ cur_container = cast(
806
+ CONTAINER_TYPE, cur_container.setdefault(prev_key, def_val)
807
+ )
808
+ else:
809
+ # pyrefly: ignore [bad-argument-type]
810
+ extend_list(cur_container, prev_key)
811
+ if cur_container[prev_key] is None:
812
+ cur_container[prev_key] = def_val
813
+ cur_container = cur_container[prev_key]
814
+
815
+ key = path[-1]
816
+ if type(key) is int:
817
+ extend_list(cast(list[Any], cur_container), key)
818
+
819
+ cur_container[key] = value
820
+
821
+
822
+ def _unflatten_state_dict(
823
+ state_dict: STATE_DICT_TYPE, mapping: FLATTEN_MAPPING
824
+ ) -> STATE_DICT_TYPE:
825
+ """Restore the original nested state_dict according to ``mapping`` and the flattened ``state_dict``."""
826
+ nested: STATE_DICT_TYPE = {}
827
+ for key, value in state_dict.items():
828
+ _set_element(nested, mapping[key], value)
829
+ return nested
western_ai/usa/pytorch/torch/distributed/_symmetric_memory/__init__.py ADDED
@@ -0,0 +1,2387 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import math
4
+ import os
5
+ import socket
6
+ import uuid
7
+ from collections.abc import Callable, Generator
8
+ from contextlib import contextmanager
9
+ from datetime import timedelta
10
+ from enum import Enum
11
+ from functools import partial
12
+ from typing import Any, Literal
13
+ from typing_extensions import deprecated
14
+
15
+ import torch
16
+ import torch.distributed._functional_collectives as funcol
17
+ import torch.distributed.distributed_c10d as c10d
18
+ from torch._C._autograd import DeviceType
19
+ from torch._C._distributed_c10d import _SymmetricMemory, Work as _Work
20
+
21
+
22
+ _group_name_to_store: dict[str, c10d.Store] = {}
23
+
24
+
25
+ @deprecated(
26
+ "`enable_symm_mem_for_group` is deprecated. There is no need to call this function anymore.",
27
+ category=FutureWarning,
28
+ )
29
+ def enable_symm_mem_for_group(group_name: c10d.GroupName) -> None:
30
+ """
31
+ Enables symmetric memory for a process group.
32
+
33
+ Args:
34
+ group_name (str): the name of the process group.
35
+ """
36
+ if group_name in _group_name_to_store:
37
+ return
38
+
39
+ group = c10d._resolve_process_group(group_name)
40
+ global_ranks = sorted(c10d._world.pg_group_ranks[group].keys())
41
+ # Different subgroups with the same name should use different stores
42
+ global_ranks_str = "_".join(map(str, global_ranks))
43
+ store = c10d.PrefixStore(
44
+ f"symmetric_memory-{global_ranks_str}",
45
+ c10d._get_process_group_store(group),
46
+ )
47
+ _group_name_to_store[group_name] = store
48
+ _SymmetricMemory.set_group_info(
49
+ group_name,
50
+ group.rank(),
51
+ group.size(),
52
+ store,
53
+ )
54
+
55
+
56
+ _is_test_mode: bool = False
57
+ _mocked_group_names: set[str] | None = None
58
+
59
+
60
+ @contextmanager
61
+ def _test_mode(group_names: set[str] | None = None) -> Generator[None, None, None]:
62
+ """
63
+ Forces ``is_symm_mem_enabled_for_group()`` to return ``True`` and the ops
64
+ defined in the ``symm_mem`` namespace to use fallback implementations.
65
+
66
+ The context manager is not thread safe.
67
+ """
68
+ global _is_test_mode
69
+ global _mocked_group_names
70
+ prev = _is_test_mode
71
+ prev_group_names = _mocked_group_names
72
+ try:
73
+ _is_test_mode = True
74
+ _mocked_group_names = group_names
75
+ yield
76
+ finally:
77
+ _is_test_mode = prev
78
+ _mocked_group_names = prev_group_names
79
+
80
+
81
+ def is_symm_mem_enabled_for_group(group_name: c10d.GroupName) -> bool:
82
+ """
83
+ Check if symmetric memory is enabled for a process group.
84
+
85
+ Args:
86
+ group_name (str): the name of the process group.
87
+ """
88
+ if _is_test_mode:
89
+ return _mocked_group_names is None or group_name in _mocked_group_names
90
+ return group_name in _group_name_to_store
91
+
92
+
93
+ _group_name_to_workspace_tensor: dict[str, torch.Tensor | None] = {}
94
+
95
+
96
+ def get_symm_mem_workspace(
97
+ group_name: c10d.GroupName, min_size: int
98
+ ) -> _SymmetricMemory:
99
+ """
100
+ Get the symmetric memory workspace associated with the process group. If
101
+ ``min_size`` is greater than the workspace associated with ``group_name``,
102
+ the workspace will be re-allocated and re-rendezvous'd.
103
+
104
+ Args:
105
+ group_name (str): the name of the process group.
106
+ min_size (int): the size requirement for the workspace in bytes.
107
+
108
+ Returns:
109
+ _SymmetricMemory: the symmetric memory workspace associated with the
110
+ group.
111
+ """
112
+ tensor = _group_name_to_workspace_tensor.get(group_name)
113
+ size = tensor.numel() * tensor.element_size() if tensor is not None else 0
114
+ if tensor is None or size < min_size:
115
+ if (
116
+ torch.accelerator.is_available()
117
+ and torch.accelerator.current_stream().is_capturing()
118
+ ):
119
+ curr_size = 0 if tensor is None else tensor.numel() * tensor.element_size()
120
+ raise RuntimeError(
121
+ f"get_symm_mem_workspace(): the requested size ({min_size} bytes) "
122
+ "is greater than the size of the currently allocated workspace "
123
+ f"({curr_size} bytes). It's currently not possible to expand the "
124
+ "workspace size during graph capture. Please invoke "
125
+ f'`get_symm_mem_workspace(group_name="{group_name}", '
126
+ f'min_size="{min_size}")` before initiating the graph capture '
127
+ "and try again."
128
+ )
129
+ tensor = _SymmetricMemory.empty_strided_p2p(
130
+ (max(size, min_size),),
131
+ [1],
132
+ torch.uint8,
133
+ torch.device(torch.accelerator.current_device_index()),
134
+ group_name,
135
+ )
136
+ _group_name_to_workspace_tensor[group_name] = tensor
137
+ return _SymmetricMemory.rendezvous(tensor)
138
+
139
+
140
+ _backend_streams: dict[int, torch.Stream] = {}
141
+
142
+
143
+ def _get_backend_stream(priority: int = 0) -> torch.Stream:
144
+ if priority not in _backend_streams:
145
+ _backend_streams[priority] = torch.Stream(priority=priority)
146
+ return _backend_streams[priority]
147
+
148
+
149
+ def _pipelined_multi_all_gather_and_consume(
150
+ shard: list[torch.Tensor],
151
+ shard_consumer: Callable[[list[torch.Tensor], int], None],
152
+ ag_out: list[torch.Tensor],
153
+ group_name: c10d.GroupName,
154
+ ag_out_needed: bool = True,
155
+ ) -> None:
156
+ """
157
+ Perform the following logic with micro-pipelined computation and
158
+ communication:
159
+
160
+ gathered = [
161
+ all_gather_single(x, gather_dim=0, group=group)
162
+ for x in shard
163
+ ]
164
+
165
+ shards = [[] for _ in range(group_size)]
166
+ for x in ag_out:
167
+ for i, y in enumerate(x.chunk(group_size)):
168
+ shards[i].append(y)
169
+
170
+ for src_rank, shard in enumerate(shards):
171
+ shard_consumer(shard, src_rank)
172
+ """
173
+ p2p_workspace_size_req = 0
174
+ for x in shard:
175
+ p2p_workspace_size_req += x.numel() * x.element_size()
176
+ symm_mem = get_symm_mem_workspace(group_name, min_size=p2p_workspace_size_req)
177
+ group_size = symm_mem.world_size
178
+ rank = symm_mem.rank
179
+
180
+ symm_mem.barrier(channel=0)
181
+ backend_stream = _get_backend_stream()
182
+ backend_stream.wait_stream(torch.accelerator.current_stream())
183
+
184
+ for x, y in zip(shard, ag_out):
185
+ if not x.is_contiguous():
186
+ raise AssertionError(
187
+ "_pipelined_all_gather_and_consume: all tensors "
188
+ "in `shard` must be contiguous"
189
+ )
190
+ if not y.is_contiguous():
191
+ raise AssertionError(
192
+ "_pipelined_all_gather_and_consume: all tensors "
193
+ "in `ag_out` must be contiguous"
194
+ )
195
+ if x.shape[0] * group_size != y.shape[0]:
196
+ raise AssertionError
197
+ if x.shape[1:] != y.shape[1:]:
198
+ raise AssertionError
199
+
200
+ def copy_shard(dst: list[torch.Tensor], src: list[torch.Tensor]) -> None:
201
+ for d, s in zip(dst, src):
202
+ d.copy_(s)
203
+
204
+ def get_p2p_bufs(remote_rank: int) -> list[torch.Tensor]:
205
+ offset_bytes = 0
206
+ bufs = []
207
+ for x in shard:
208
+ buf = symm_mem.get_buffer(
209
+ remote_rank,
210
+ x.shape,
211
+ x.dtype,
212
+ storage_offset=offset_bytes // x.element_size(),
213
+ )
214
+ bufs.append(buf)
215
+ offset_bytes += buf.numel() * buf.element_size()
216
+ return bufs
217
+
218
+ local_p2p_bufs = get_p2p_bufs(rank)
219
+
220
+ # shards[i] => shard from rank i
221
+ shards: list[list[torch.Tensor]] = [[] for _ in range(group_size)]
222
+ for x in ag_out:
223
+ for i, y in enumerate(x.chunk(group_size)):
224
+ shards[i].append(y)
225
+
226
+ # Parallelization strategy: after each rank copies its shard into its local
227
+ # p2p buffer, every rank issues independent p2p copy -> shard_consumer
228
+ # sequences to two streams. In addition to computation/communication
229
+ # overlapping, the strategy allows for computation/computation overlapping,
230
+ # greatly reducing quantization inefficiency.
231
+ #
232
+ # Notation:
233
+ # - "mv" for the copy to local buffer
234
+ # - "cp" for p2p copies
235
+ # - "b" for barriers
236
+ #
237
+ # Constraints:
238
+ # - The GPU scheduler may or may not overlap "mv" with the first shard_consumer.
239
+ # - "cp" from different streams cannot overlap.
240
+ #
241
+ # Ideal scenario 0 - "mv" overlaps with the first shard_consumer:
242
+ #
243
+ # stream 0: [ shard_consumer ][ cp ][ shard_consumer ]
244
+ # stream 1: [ mv ][b][ cp ][ shard_consumer ]
245
+ #
246
+ # Ideal scenario 1 - "mv" is scheduled before the first shard_consumer:
247
+ #
248
+ # stream 0: [ shard_consumer ][ cp ][ shard_consumer ]
249
+ # stream 1: [ mv ][b][ cp ][ shard_consumer ]
250
+ #
251
+ # Suboptimal scenario 0 - "mv" is scheduled after the first shard_consumer:
252
+ #
253
+ # stream 0: [ shard_consumer ] [ cp ][ shard_consumer ]
254
+ # stream 1: [ mv ][b][ cp ][ shard_consumer ]
255
+ #
256
+ # Suboptimal scenario 0 - "b" is scheduled after the first shard_consumer:
257
+ #
258
+ # stream 0: [ shard_consumer ] [ cp ][ shard_consumer ]
259
+ # stream 1: [ mv ] [b][ cp ][ shard_consumer ]
260
+ #
261
+ # We haven't yet figured out a way to ensure "mv" and "b" are either
262
+ # overlapped with or scheduled before the first shard_consumer. Thus, to
263
+ # prevent suboptimal scenarios, we are giving up the chance to overlap "mv"
264
+ # and "b" with the first shard_consumer for now.
265
+ copy_shard(dst=local_p2p_bufs, src=shard)
266
+ symm_mem.barrier(channel=1)
267
+ backend_stream.wait_stream(torch.accelerator.current_stream())
268
+
269
+ # At this point, all ranks have copied their local shard to
270
+ # their local p2p buffer. Each rank can now copy and consume
271
+ # remote shards.
272
+ shard_consumer(shard, rank)
273
+
274
+ for step in range(1, group_size):
275
+ if step % 2 == 0:
276
+ stream = torch.accelerator.current_stream()
277
+ else:
278
+ stream = backend_stream
279
+ remote_rank = (step + rank) % group_size
280
+ remote_p2p_bufs = get_p2p_bufs(remote_rank)
281
+ with stream:
282
+ copy_shard(dst=shards[remote_rank], src=remote_p2p_bufs)
283
+ shard_consumer(shards[remote_rank], remote_rank)
284
+
285
+ if ag_out_needed:
286
+ # Copy from input to the all-gather output. Opportunistically overlap
287
+ # it with the last shard_consumer.
288
+ if group_size % 2 == 0:
289
+ stream = torch.accelerator.current_stream()
290
+ else:
291
+ stream = backend_stream
292
+ with stream:
293
+ copy_shard(dst=shards[rank], src=shard)
294
+
295
+ torch.accelerator.current_stream().wait_stream(backend_stream)
296
+ symm_mem.barrier(channel=0)
297
+
298
+
299
+ def _pipelined_all_gather_and_consume(
300
+ shard: torch.Tensor,
301
+ shard_consumer: Callable[[torch.Tensor, int], None],
302
+ ag_out: torch.Tensor,
303
+ group_name: c10d.GroupName,
304
+ ag_out_needed: bool = True,
305
+ ) -> None:
306
+ """
307
+ Perform the following logic with micro-pipelined computation and
308
+ communication:
309
+
310
+ ag_out = all_gather_single(shard, gather_dim=0, group=group)
311
+ shards = ag_out.chunk(group.size())
312
+ for src_rank, shard in enumerate(shards):
313
+ shard_consumer(shard, src_rank)
314
+ """
315
+
316
+ def adapter(shard: list[torch.Tensor], rank: int) -> None:
317
+ shard_consumer(shard[0], rank)
318
+
319
+ _pipelined_multi_all_gather_and_consume(
320
+ [shard],
321
+ adapter,
322
+ [ag_out],
323
+ group_name,
324
+ ag_out_needed,
325
+ )
326
+
327
+
328
+ def _pipelined_produce_and_all2all(
329
+ chunk_producer: Callable[[int, torch.Tensor], None],
330
+ output: torch.Tensor,
331
+ group_name: c10d.GroupName,
332
+ out_chunk_dim: int = 0,
333
+ ) -> None:
334
+ """
335
+ Perform the following logic with micro-pipelined computation and
336
+ communication:
337
+
338
+ chunks = [
339
+ chunk_producer(dst_rank, chunks[dst_rank])
340
+ for dst_rank in range(group_size):
341
+ ]
342
+ dist.all_to_all_single(output=output, input=torch.cat(chunks))
343
+ """
344
+ out_chunks = output.chunk(
345
+ c10d._get_group_size_by_name(group_name), dim=out_chunk_dim
346
+ )
347
+ p2p_workspace_size_req = out_chunks[0].numel() * out_chunks[0].element_size() * 2
348
+ symm_mem = get_symm_mem_workspace(group_name, min_size=p2p_workspace_size_req)
349
+ group_size = symm_mem.world_size
350
+ rank = symm_mem.rank
351
+
352
+ symm_mem.barrier(channel=0)
353
+ backend_stream = _get_backend_stream()
354
+ backend_stream.wait_stream(torch.accelerator.current_stream())
355
+
356
+ def get_p2p_buf(rank: int, idx: int) -> torch.Tensor:
357
+ if idx not in (0, 1):
358
+ raise AssertionError
359
+ offset = 0 if idx == 0 else out_chunks[0].numel()
360
+ return symm_mem.get_buffer(
361
+ rank, out_chunks[0].shape, out_chunks[0].dtype, offset
362
+ )
363
+
364
+ # Prepare two local p2p buffers, so that a remote rank can pull the result
365
+ # of step [i] in one p2p buffer while the local rank can compute the
366
+ # result of step [i+1] and write it directly the other p2p buffer.
367
+ local_p2p_buf_0 = get_p2p_buf(rank, 0)
368
+ local_p2p_buf_1 = get_p2p_buf(rank, 1)
369
+
370
+ for step in range(1, group_size):
371
+ remote_rank = (rank - step) % group_size
372
+ if step % 2 == 0:
373
+ stream = torch.accelerator.current_stream()
374
+ p2p_buf = local_p2p_buf_1
375
+ remote_p2p_buf = get_p2p_buf(remote_rank, 1)
376
+ else:
377
+ stream = backend_stream
378
+ p2p_buf = local_p2p_buf_0
379
+ remote_p2p_buf = get_p2p_buf(remote_rank, 0)
380
+ with stream:
381
+ # Parallelization strategy: every rank issues independent compute
382
+ # -> barrier -> p2p copy sequences on two streams. In addition to
383
+ # computation/communication overlapping, the strategy allows for
384
+ # computation/computation overlapping, greatly reducing
385
+ # quantization inefficiency.
386
+ #
387
+ # Ideally, stream activities would look like this ("b" for
388
+ # barriers, "cp" for p2p copies):
389
+ #
390
+ # [rank 0]
391
+ # stream 0: [ chunk_producer ][b][ cp ][ chunk_producer ][b][ cp ]
392
+ # stream 1: [ chunk_producer ][b][ cp ][ chunk_producer ][b][ cp ]
393
+ #
394
+ # [rank 1]
395
+ # stream 0: [ chunk_producer ][b][ cp ][ chunk_producer ][b][ cp ]
396
+ # stream 1: [ chunk_producer ][b][ cp ][ chunk_producer ][b][ cp ]
397
+ #
398
+ # Note that the barriers synchronize streams with the same ID
399
+ # across ranks. They don't synchronize streams on the same rank.
400
+ #
401
+ # Since the work on both streams is independent, there's no
402
+ # guarantee that the chunk_producer from stream 0 or stream 1 will
403
+ # be scheduled first. If there is a scheduling mismatch across
404
+ # ranks, the barrier forces all ranks to wait for the slowest.
405
+ #
406
+ # When scheduling mismatches occur among ranks, the stream
407
+ # activities might look like this (note that p2p copies from
408
+ # different streams cannot overlap with each other):
409
+ #
410
+ # [rank 0]
411
+ # stream 0: [ chunk_producer ][b ][ cp ][ chunk_producer ][b ][ cp ]
412
+ # stream 1: [ chunk_producer ][b] [ cp ][ chunk_producer ][b] [ cp ]
413
+ #
414
+ # [rank 1]
415
+ # stream 0: [ chunk_producer ][b] [ cp ][ chunk_producer ][b] [ cp ]
416
+ # stream 1: [ chunk_producer ][b ][ cp ][ chunk_producer ][b ][ cp ]
417
+ #
418
+ # To prevent this, we need to ensure that the chunk_producer on
419
+ # stream 1 gets scheduled first on every rank. Without access to
420
+ # the underlying kernels, CUDA offers no API to control the
421
+ # scheduling order of two independent, overlapping kernels. Our
422
+ # solution is to issue a small sleep kernel in stream 0. The sleep
423
+ # duration is insignificant, but having an extra task in stream 0
424
+ # will almost guarantee that the chunk_producer on stream 1 gets
425
+ # scheduled first. Once the first chunk_producer is scheduled in
426
+ # the correct order, there's very little room for the scheduling
427
+ # order of subsequent kernels to be inconsistent across ranks.
428
+ if step == 2 and torch.cuda.is_available():
429
+ torch.cuda._sleep(100)
430
+ chunk_producer((rank + step) % group_size, p2p_buf)
431
+ symm_mem.barrier(channel=step % 2)
432
+ out_chunks[remote_rank].copy_(remote_p2p_buf)
433
+ # The local P2P buffer can only be overwritten by the next
434
+ # chunk_producer after all peers have finished reading from it.
435
+ symm_mem.barrier(channel=step % 2)
436
+
437
+ # If the sleep wasn't issued in the above loop, do it now.
438
+ if group_size == 2 and torch.cuda.is_available():
439
+ torch.cuda._sleep(100)
440
+
441
+ chunk_producer(rank, out_chunks[rank])
442
+ torch.accelerator.current_stream().wait_stream(backend_stream)
443
+ symm_mem.barrier(channel=0)
444
+
445
+
446
+ lib = torch.library.Library("symm_mem", "DEF")
447
+ lib.define(
448
+ "fused_all_gather_matmul("
449
+ "Tensor A, Tensor[] Bs, int gather_dim, str group_name, *, bool return_A = True) -> (Tensor?, Tensor[])",
450
+ tags=[torch._C.Tag.needs_fixed_stride_order],
451
+ )
452
+ lib.define(
453
+ "fused_all_gather_scaled_matmul("
454
+ "Tensor A, Tensor[] Bs, Tensor A_scale, Tensor[] B_scales, "
455
+ "int gather_dim, str group_name, "
456
+ "Tensor?[] biases, "
457
+ "Tensor?[] result_scales, "
458
+ "ScalarType?[] out_dtypes, "
459
+ "bool[] use_fast_accum) -> (Tensor, Tensor[])",
460
+ tags=[torch._C.Tag.needs_fixed_stride_order],
461
+ )
462
+ lib.define(
463
+ "fused_matmul_reduce_scatter(Tensor A, Tensor B, str reduce_op, int scatter_dim, str group_name) -> Tensor",
464
+ tags=[torch._C.Tag.needs_fixed_stride_order],
465
+ )
466
+ lib.define(
467
+ "fused_scaled_matmul_reduce_scatter("
468
+ "Tensor A, Tensor B, Tensor A_scale, Tensor B_scale, "
469
+ "str reduce_op, int orig_scatter_dim, int scatter_dim_after_maybe_reshape, str group_name, SymInt[]? output_shape, "
470
+ "Tensor? bias = None, "
471
+ "Tensor? result_scale = None, "
472
+ "ScalarType? out_dtype = None, "
473
+ "bool use_fast_accum = False) -> Tensor",
474
+ tags=[torch._C.Tag.needs_fixed_stride_order],
475
+ )
476
+ lib.define("_low_contention_all_gather(Tensor tensor, str group_name) -> Tensor")
477
+ lib.define(
478
+ "_low_contention_reduce_scatter(Tensor tensor, str reduce_op, str group_name) -> Tensor"
479
+ )
480
+
481
+ lib.define("get_remote_tensors(Tensor x, str group_name) -> Tensor[]")
482
+ """
483
+ Given a local tensor and a group name, return a tuple of tensors that are
484
+ symmetric on other devices. The returned tensors are ordered by rank IDs. The
485
+ length of the tuple equals to the size of the group.
486
+
487
+ Note: this API works only when `world_within_direct_access()` returns True, i.e.
488
+ only when the group is within NVLink domain or similar. It does not work across
489
+ network interfaces.
490
+ """
491
+
492
+
493
+ @torch.library.impl(lib, "get_remote_tensors", "CUDA")
494
+ @torch.library.impl(lib, "get_remote_tensors", "XPU")
495
+ def _get_remote_tensors_default(
496
+ local: torch.Tensor, group_name: c10d.GroupName
497
+ ) -> tuple[torch.Tensor, ...]:
498
+ hdl = rendezvous(local, group_name)
499
+ if hdl is None:
500
+ raise ValueError("Tensor is not allocated from Symmetric Memory")
501
+
502
+ return tuple(
503
+ hdl.get_remote_tensor(peer, local.size(), local.dtype)
504
+ for peer in range(hdl.world_size)
505
+ )
506
+
507
+
508
+ @torch.library.impl(lib, "get_remote_tensors", "Meta")
509
+ def _get_remote_tensors_meta(
510
+ local: torch.Tensor, group_name: c10d.GroupName
511
+ ) -> tuple[torch.Tensor, ...]:
512
+ group = c10d._resolve_process_group(group_name)
513
+ return tuple(torch.empty_like(local) for _ in range(group.size()))
514
+
515
+
516
+ class _ScaleMode(Enum):
517
+ UNSCALED = "unscaled"
518
+ TENSOR_WISE = "tensor-wise"
519
+ ROW_WISE_SHARDED = "row-wise-sharded"
520
+ ROW_WISE_REPLICATED = "row-wise-replicated"
521
+
522
+
523
+ def _check_and_verify_fp8_all_gather_scale_mode(
524
+ shard: torch.Tensor, scale: torch.Tensor | None, gather_dim: int, group_size: int
525
+ ) -> _ScaleMode:
526
+ full_shape = list(shard.shape)
527
+ full_shape[gather_dim] *= group_size
528
+
529
+ if scale is None:
530
+ return _ScaleMode.UNSCALED
531
+ elif scale.shape[:-1] == shard.shape[:-1] and scale.shape[-1] == 1:
532
+ # Row-wise scaling
533
+ #
534
+ # NOTE: when the last dim of both A_shard and A_scale is one, we can't
535
+ # tell if A_scale is replicated tensor-wise scale or sharded row-wise
536
+ # scale. Treating it as row-wise scaling for safety.
537
+ return _ScaleMode.ROW_WISE_SHARDED
538
+ elif scale.numel() == 1:
539
+ return _ScaleMode.TENSOR_WISE
540
+ elif list(scale.shape[:-1]) == full_shape[:-1]:
541
+ return _ScaleMode.ROW_WISE_REPLICATED
542
+ else:
543
+ raise ValueError(
544
+ "Invalid scale shape for fp8 all-gather "
545
+ f"(shard shape: {shard.shape}, scale shape: {scale.shape})"
546
+ )
547
+
548
+
549
+ def _fused_all_gather_matmul_impl(
550
+ mm_out_op: torch._ops.OpOverload,
551
+ A_shard: torch.Tensor,
552
+ Bs: list[torch.Tensor],
553
+ A_scale: torch.Tensor | None,
554
+ kwargs_list: list[dict[str, Any]],
555
+ out_dtypes: list[torch.dtype | None],
556
+ gather_dim: int,
557
+ group_name: c10d.GroupName,
558
+ return_A: bool,
559
+ ) -> tuple[torch.Tensor | None, list[torch.Tensor]]:
560
+ if A_shard.dim() < 2:
561
+ raise ValueError("A_shard must be a matrix")
562
+ for B in Bs:
563
+ if B.dim() != 2:
564
+ raise ValueError("B must be a matrix")
565
+ if len(out_dtypes) != len(Bs):
566
+ raise ValueError("len(out_types) must be the same as len(Bs)")
567
+ if len(kwargs_list) != len(Bs):
568
+ raise ValueError("len(kwargs_list) must be the same as len(Bs)")
569
+ if gather_dim < 0 or gather_dim >= A_shard.dim():
570
+ raise ValueError("Invalid gather_dim")
571
+
572
+ group = c10d._resolve_process_group(group_name)
573
+
574
+ if gather_dim == A_shard.ndim - 1 or gather_dim == -1:
575
+ return _fused_all_gather_matmul_last_gather_dim_impl(
576
+ mm_out_op,
577
+ A_shard,
578
+ Bs,
579
+ A_scale,
580
+ kwargs_list,
581
+ out_dtypes,
582
+ gather_dim,
583
+ group_name,
584
+ return_A,
585
+ )
586
+
587
+ # Move the gather_dim to the front and flatten the tensor into a 2D matrix.
588
+ # The flattened tensor doesn't need to be contiguous (for computation
589
+ # efficiency), as _pipelined_all_gather_and_consume guarantees that shards
590
+ # passed to shard_consumer are contiguous.
591
+ A_shard_flat = A_shard.movedim(gather_dim, 0)
592
+ leading_dims = [group.size()] + list(A_shard_flat.shape[:-1])
593
+ A_shard_flat = A_shard_flat.flatten(0, -2)
594
+
595
+ # Helper function for reverting the above transformation
596
+ def unflatten(t: torch.Tensor) -> torch.Tensor:
597
+ return t.view(*leading_dims, -1).flatten(0, 1).movedim(0, gather_dim)
598
+
599
+ A_flat = A_shard_flat.new_empty(
600
+ A_shard_flat.shape[0] * group.size(),
601
+ A_shard_flat.shape[1],
602
+ )
603
+
604
+ outputs = [
605
+ A_flat.new_empty(A_flat.shape[0], B.shape[1], dtype=out_dtype or B.dtype)
606
+ for B, out_dtype in zip(Bs, out_dtypes)
607
+ ]
608
+ output_shards = [output.chunk(group.size()) for output in outputs]
609
+
610
+ scale_mode = _check_and_verify_fp8_all_gather_scale_mode(
611
+ shard=A_shard, scale=A_scale, gather_dim=gather_dim, group_size=group.size()
612
+ )
613
+
614
+ # Computing block-wise matmul along the first dim of A
615
+ if scale_mode == _ScaleMode.ROW_WISE_SHARDED:
616
+ if A_scale is None:
617
+ raise AssertionError
618
+ A_scale_shard = A_scale.movedim(gather_dim, 0).flatten(0, -2)
619
+ A_scale_flat = A_scale_shard.new_empty(
620
+ A_scale_shard.shape[0] * group.size(),
621
+ A_scale_shard.shape[1],
622
+ )
623
+
624
+ def row_wise_sharded_consumer(shard: list[torch.Tensor], rank: int) -> None:
625
+ for idx, (B, kwargs) in enumerate(zip(Bs, kwargs_list)):
626
+ mm_out_op(
627
+ shard[0],
628
+ B,
629
+ scale_a=shard[1],
630
+ **kwargs,
631
+ out=output_shards[idx][rank],
632
+ )
633
+
634
+ _pipelined_multi_all_gather_and_consume(
635
+ [A_shard_flat, A_scale_shard],
636
+ row_wise_sharded_consumer,
637
+ [A_flat, A_scale_flat],
638
+ group_name,
639
+ return_A,
640
+ )
641
+ elif scale_mode == _ScaleMode.ROW_WISE_REPLICATED:
642
+ if A_scale is None:
643
+ raise AssertionError
644
+ A_scale_shards = (
645
+ A_scale.movedim(gather_dim, 0).flatten(0, -2).chunk(group.size())
646
+ )
647
+
648
+ def row_wise_replicated_consumer(shard: torch.Tensor, rank: int) -> None:
649
+ for idx, (B, kwargs) in enumerate(zip(Bs, kwargs_list)):
650
+ mm_out_op(
651
+ shard,
652
+ B,
653
+ scale_a=A_scale_shards[rank],
654
+ **kwargs,
655
+ out=output_shards[idx][rank],
656
+ )
657
+
658
+ _pipelined_all_gather_and_consume(
659
+ A_shard_flat,
660
+ row_wise_replicated_consumer,
661
+ A_flat,
662
+ group_name,
663
+ return_A,
664
+ )
665
+ else:
666
+ if scale_mode == _ScaleMode.TENSOR_WISE:
667
+ if A_scale is None:
668
+ raise AssertionError
669
+ for kwargs in kwargs_list:
670
+ kwargs["scale_a"] = A_scale
671
+ else:
672
+ if scale_mode != _ScaleMode.UNSCALED:
673
+ raise AssertionError
674
+
675
+ def default_consumer(shard: torch.Tensor, rank: int) -> None:
676
+ for idx, (B, kwargs) in enumerate(zip(Bs, kwargs_list)):
677
+ mm_out_op(shard, B, **kwargs, out=output_shards[idx][rank])
678
+
679
+ _pipelined_all_gather_and_consume(
680
+ A_shard_flat,
681
+ default_consumer,
682
+ A_flat,
683
+ group_name,
684
+ return_A,
685
+ )
686
+
687
+ A = unflatten(A_flat) if return_A else None
688
+ return A, [unflatten(output) for output in outputs]
689
+
690
+
691
+ def _pipelined_all_gather_and_consume_last_dim(
692
+ shard: torch.Tensor,
693
+ shard_consumer: Callable[[torch.Tensor, int], None],
694
+ ag_out: torch.Tensor,
695
+ group_name: c10d.GroupName,
696
+ ag_out_needed: bool = True,
697
+ ) -> None:
698
+ p2p_workspace_size_req = 0
699
+ p2p_workspace_size_req = shard.numel() * shard.element_size()
700
+ symm_mem = get_symm_mem_workspace(group_name, min_size=p2p_workspace_size_req)
701
+ group_size = symm_mem.world_size
702
+ rank = symm_mem.rank
703
+
704
+ symm_mem.barrier(channel=0)
705
+ backend_stream = _get_backend_stream()
706
+ backend_stream.wait_stream(torch.accelerator.current_stream())
707
+
708
+ def copy_shard(dst: torch.Tensor, src: torch.Tensor) -> None:
709
+ dst.copy_(src)
710
+
711
+ def get_p2p_buf(remote_rank: int) -> torch.Tensor:
712
+ buf = symm_mem.get_buffer(
713
+ remote_rank,
714
+ shard.shape,
715
+ shard.dtype,
716
+ )
717
+ return buf
718
+
719
+ local_p2p_buf = get_p2p_buf(rank)
720
+
721
+ shards = ag_out.chunk(group_size)
722
+
723
+ copy_shard(dst=local_p2p_buf, src=shard)
724
+ symm_mem.barrier(channel=1)
725
+ backend_stream.wait_stream(torch.accelerator.current_stream())
726
+
727
+ # At this point, all ranks have copied their local shard to
728
+ # their local p2p buffer. Each rank can now copy and consume
729
+ # remote shards.
730
+ shard_consumer(shard, rank)
731
+
732
+ for step in range(1, group_size):
733
+ if step % 2 == 0:
734
+ stream = torch.accelerator.current_stream()
735
+ else:
736
+ stream = backend_stream
737
+ remote_rank = (step + rank) % group_size
738
+ remote_p2p_buf = get_p2p_buf(remote_rank)
739
+ with stream:
740
+ copy_shard(dst=shards[remote_rank], src=remote_p2p_buf)
741
+ shard_consumer(shards[remote_rank], remote_rank)
742
+
743
+ if ag_out_needed:
744
+ # Copy from input to the all-gather output. Opportunistically overlap
745
+ # it with the last shard_consumer.
746
+ if group_size % 2 == 0:
747
+ stream = torch.accelerator.current_stream()
748
+ else:
749
+ stream = backend_stream
750
+ with stream:
751
+ copy_shard(dst=shards[rank], src=shard)
752
+
753
+ torch.accelerator.current_stream().wait_stream(backend_stream)
754
+ symm_mem.barrier(channel=0)
755
+
756
+
757
+ def _fused_all_gather_matmul_last_gather_dim_impl(
758
+ mm_out_op: torch._ops.OpOverload,
759
+ A_shard: torch.Tensor,
760
+ Bs: list[torch.Tensor],
761
+ A_scale: torch.Tensor | None,
762
+ kwargs_list: list[dict[str, Any]],
763
+ out_dtypes: list[torch.dtype | None],
764
+ gather_dim: int,
765
+ group_name: c10d.GroupName,
766
+ return_A: bool,
767
+ ) -> tuple[torch.Tensor | None, list[torch.Tensor]]:
768
+ group = c10d._resolve_process_group(group_name)
769
+ group_size = group.size()
770
+
771
+ B_shards = [B.chunk(group.size()) for B in Bs]
772
+
773
+ leading_dims = list(A_shard.shape[:-1])
774
+ A_shard_flat = A_shard.flatten(0, -2)
775
+
776
+ def unflatten(t: torch.Tensor) -> torch.Tensor:
777
+ return t.view(*leading_dims, -1)
778
+
779
+ A_flat_out = A_shard_flat.new_empty(
780
+ A_shard_flat.shape[0] * group.size(),
781
+ A_shard_flat.shape[1],
782
+ )
783
+
784
+ outputs = [
785
+ torch.empty(
786
+ (A_shard_flat.shape[0], B.shape[1]),
787
+ dtype=out_dtype or B.dtype,
788
+ device=A_shard.device,
789
+ )
790
+ for B, out_dtype in zip(Bs, out_dtypes)
791
+ ]
792
+
793
+ first = True
794
+ events = [torch.Event() for _ in outputs]
795
+
796
+ def default_consumer(shard: torch.Tensor, rank: int) -> None:
797
+ nonlocal first
798
+ for out, event, B_shard, kwargs in zip(outputs, events, B_shards, kwargs_list):
799
+ event.wait()
800
+ if first:
801
+ torch.ops.aten.mm.out(shard, B_shard[rank], **kwargs, out=out)
802
+ else:
803
+ out.addmm_(shard, B_shard[rank])
804
+ event.record()
805
+
806
+ first = False
807
+
808
+ _pipelined_all_gather_and_consume_last_dim(
809
+ A_shard_flat,
810
+ default_consumer,
811
+ A_flat_out,
812
+ group_name,
813
+ return_A,
814
+ )
815
+ ret_A = None
816
+ if return_A:
817
+ # This path is inefficient and will be filtered out at passes stage
818
+ # Added only for completeness.
819
+ A_split_cat_out_flat = torch.cat(A_flat_out.chunk(group_size), dim=-1)
820
+ ret_A = unflatten(A_split_cat_out_flat)
821
+
822
+ return ret_A, [unflatten(output) for output in outputs]
823
+
824
+
825
+ @torch.library.impl(lib, "fused_all_gather_matmul", "Meta")
826
+ def _fused_all_gather_matmul_fallback(
827
+ A_shard: torch.Tensor,
828
+ Bs: list[torch.Tensor],
829
+ gather_dim: int,
830
+ group_name: c10d.GroupName,
831
+ *,
832
+ return_A: bool = True,
833
+ ) -> tuple[torch.Tensor | None, list[torch.Tensor]]:
834
+ group_size = c10d._get_group_size_by_name(group_name)
835
+ A = torch.ops._c10d_functional.all_gather_into_tensor(
836
+ A_shard.contiguous(), group_size, group_name
837
+ )
838
+ A = torch.ops._c10d_functional.wait_tensor(A)
839
+ if gather_dim == A.ndim - 1 or gather_dim == -1:
840
+ A_splits = A.chunk(group_size)
841
+ A_mm = torch.cat(A_splits, dim=-1)
842
+ res = [torch.matmul(A_mm, B) for B in Bs]
843
+ if return_A:
844
+ return A_mm, res
845
+ else:
846
+ return None, res
847
+
848
+ A = A.view(group_size, *A_shard.shape).movedim(gather_dim + 1, 1).flatten(0, 1)
849
+ res = [torch.matmul(A, B).movedim(0, gather_dim) for B in Bs]
850
+ if return_A:
851
+ return A.movedim(0, gather_dim), res
852
+ else:
853
+ return None, res
854
+
855
+
856
+ @torch.library.impl(lib, "fused_all_gather_matmul", "CUDA")
857
+ @torch.library.impl(lib, "fused_all_gather_matmul", "XPU")
858
+ def _fused_all_gather_matmul(
859
+ A_shard: torch.Tensor,
860
+ Bs: list[torch.Tensor],
861
+ gather_dim: int,
862
+ group_name: c10d.GroupName,
863
+ *,
864
+ return_A: bool = True,
865
+ ) -> tuple[torch.Tensor | None, list[torch.Tensor]]:
866
+ """
867
+ Perform the following logic with micro-pipelined computation and
868
+ communication:
869
+
870
+ all_gather_single(A_shard, gather_dim, group_name) @ B
871
+
872
+ Optimal stride order for A_shard - if A_shard.movedim(gather_dim, 0) is
873
+ contiguous, no extra copy is required for input layout transformation.
874
+ Otherwise A_shard needs to be copied once.
875
+ """
876
+ if _is_test_mode:
877
+ return _fused_all_gather_matmul_fallback(
878
+ A_shard, Bs, gather_dim, group_name, return_A=return_A
879
+ )
880
+
881
+ if _should_use_fused_all_gather_matmul_native(A_shard, Bs, gather_dim, group_name):
882
+ group = c10d._resolve_process_group(group_name)
883
+ leading_dims = list(A_shard.shape[:-1])
884
+ leading_dims[0] *= group.size()
885
+ A, out = _fused_all_gather_matmul_native(
886
+ A_shard.flatten(0, -2), Bs[0], group_name
887
+ )
888
+ return A.view(*leading_dims, -1), [out.view(*leading_dims, -1)]
889
+
890
+ if _should_use_multimem_all_gather_matmul(
891
+ A_shard, gather_dim, group_name, return_A
892
+ ):
893
+ return None, _multimem_all_gather_matmul(A_shard, Bs, group_name)
894
+
895
+ with torch.profiler.record_function("fused_all_gather_matmul"):
896
+ return _fused_all_gather_matmul_impl(
897
+ torch.ops.aten.mm.out,
898
+ A_shard,
899
+ Bs,
900
+ None,
901
+ [{} for B in Bs],
902
+ [B.dtype for B in Bs],
903
+ gather_dim,
904
+ group_name,
905
+ return_A,
906
+ )
907
+
908
+
909
+ def _should_use_fused_all_gather_matmul_native(
910
+ A_shard: torch.Tensor,
911
+ Bs: list[torch.Tensor],
912
+ gather_dim: int,
913
+ group_name: c10d.GroupName,
914
+ ) -> bool:
915
+ group = c10d._resolve_process_group(group_name)
916
+ local_M = math.prod(A_shard.shape[:-1])
917
+
918
+ return (
919
+ "TORCH_SYMM_MEM_ENABLE_NATIVE_ASYNC_TP" in os.environ
920
+ and torch.cuda.is_available()
921
+ and A_shard.device.type == "cuda"
922
+ and A_shard.is_contiguous()
923
+ and gather_dim == 0
924
+ # _async_input_mm requires local_M to be divisible by world_size.
925
+ and local_M % group.size() == 0
926
+ # _async_input_mm outperforms the decomposition-based approach when the
927
+ # global M is small.
928
+ and 2048 < local_M * group.size() <= 4096
929
+ # _async_input_mm only supports a single B.
930
+ and len(Bs) == 1
931
+ )
932
+
933
+
934
+ def _fused_all_gather_matmul_native(
935
+ A_shard: torch.Tensor,
936
+ B: torch.Tensor,
937
+ group_name: c10d.GroupName,
938
+ ) -> tuple[torch.Tensor, torch.Tensor]:
939
+ symm_mem = rendezvous(A_shard, group_name)
940
+ if symm_mem is None:
941
+ symm_mem = get_symm_mem_workspace(
942
+ group_name, A_shard.numel() * A_shard.element_size()
943
+ )
944
+ symm_mem.barrier()
945
+ buf = symm_mem.get_buffer(symm_mem.rank, A_shard.shape, A_shard.dtype)
946
+ buf.copy_(A_shard)
947
+ A_shard = buf
948
+
949
+ rank = symm_mem.rank
950
+ world_size = symm_mem.world_size
951
+
952
+ current_stream = torch.cuda.current_stream()
953
+ backend_stream = _get_backend_stream(priority=-1)
954
+
955
+ symm_mem.barrier()
956
+ backend_stream.wait_stream(current_stream)
957
+ current_stream.wait_stream(backend_stream)
958
+
959
+ A = A_shard.new_empty(A_shard.shape[0] * world_size, A_shard.shape[1])
960
+ A_signals = torch.zeros(world_size, dtype=torch.uint32, device=A_shard.device)
961
+ A_shards = A.chunk(world_size)
962
+
963
+ A_shards[rank].copy_(A_shard)
964
+ if not torch.cuda.is_current_stream_capturing():
965
+ _SymmetricMemory.stream_write_value32(A_signals, rank, 1)
966
+ else:
967
+ _SymmetricMemory.memset32(A_signals, offset=rank, val=1, count=1)
968
+
969
+ out = torch.ops.symm_mem._async_input_mm(A, B, A_signals, rank)
970
+ for step in range(1, world_size):
971
+ src_rank = (rank + step) % world_size
972
+ src_buf = symm_mem.get_buffer(src_rank, A_shard.shape, A_shard.dtype)
973
+ with backend_stream:
974
+ A_shards[src_rank].copy_(src_buf)
975
+ if not torch.cuda.is_current_stream_capturing():
976
+ # cuStreamWriteValue32 issues a system level fence before the write
977
+ _SymmetricMemory.stream_write_value32(A_signals, src_rank, 1)
978
+ else:
979
+ _SymmetricMemory.memset32(A_signals, offset=src_rank, val=1, count=1)
980
+
981
+ current_stream.wait_stream(backend_stream)
982
+ backend_stream.wait_stream(current_stream)
983
+
984
+ symm_mem.barrier()
985
+ return A, out
986
+
987
+
988
+ def _should_use_multimem_all_gather_matmul(
989
+ A_shard: torch.Tensor,
990
+ gather_dim: int,
991
+ group_name: c10d.GroupName,
992
+ return_A: bool,
993
+ ) -> bool:
994
+ group = c10d._resolve_process_group(group_name)
995
+ local_M = math.prod(A_shard.shape[:-1])
996
+ has_multicast_support = (
997
+ A_shard.device.type == "cuda"
998
+ and _SymmetricMemory.has_multicast_support(
999
+ DeviceType.CUDA, A_shard.device.index
1000
+ )
1001
+ )
1002
+
1003
+ return (
1004
+ has_multicast_support
1005
+ and not return_A
1006
+ and A_shard.is_contiguous()
1007
+ and gather_dim == 0
1008
+ # The heuristic is empirical. We could refine it with a more
1009
+ # sophisticated perf model.
1010
+ and local_M * group.size() <= 2048
1011
+ )
1012
+
1013
+
1014
+ def _multimem_all_gather_matmul(
1015
+ A_shard: torch.Tensor,
1016
+ Bs: list[torch.Tensor],
1017
+ group_name: c10d.GroupName,
1018
+ ) -> list[torch.Tensor]:
1019
+ group = c10d._resolve_process_group(group_name)
1020
+ A_shape = torch.Size((A_shard.shape[0] * group.size(), *A_shard.shape[1:]))
1021
+ symm_mem = get_symm_mem_workspace(
1022
+ group_name, A_shape.numel() * A_shard.element_size()
1023
+ )
1024
+ A = symm_mem.get_buffer(symm_mem.rank, A_shape, A_shard.dtype)
1025
+ torch.ops.symm_mem.multimem_all_gather_out(A_shard, group_name, A)
1026
+ return [torch.matmul(A, B) for B in Bs]
1027
+
1028
+
1029
+ @torch.library.impl(lib, "fused_all_gather_scaled_matmul", "Meta")
1030
+ def _fused_all_gather_scaled_matmul_fallback(
1031
+ A_shard: torch.Tensor,
1032
+ Bs: list[torch.Tensor],
1033
+ A_scale: torch.Tensor,
1034
+ B_scales: list[torch.Tensor],
1035
+ gather_dim: int,
1036
+ group_name: c10d.GroupName,
1037
+ biases: list[torch.Tensor | None],
1038
+ result_scales: list[torch.Tensor | None],
1039
+ out_dtypes: list[torch.dtype | None],
1040
+ use_fast_accum: list[bool],
1041
+ ) -> tuple[torch.Tensor, list[torch.Tensor]]:
1042
+ out_dtypes = _maybe_convert_scalar_types_to_dtypes(out_dtypes)
1043
+
1044
+ group_size = c10d._get_group_size_by_name(group_name)
1045
+ A = torch.ops._c10d_functional.all_gather_into_tensor(
1046
+ A_shard.contiguous(), group_size, group_name
1047
+ )
1048
+ A = torch.ops._c10d_functional.wait_tensor(A)
1049
+ A = A.view(group_size, *A_shard.shape).movedim(gather_dim + 1, 1).flatten(0, 1)
1050
+
1051
+ scale_mode = _check_and_verify_fp8_all_gather_scale_mode(
1052
+ shard=A_shard, scale=A_scale, gather_dim=gather_dim, group_size=group_size
1053
+ )
1054
+ if scale_mode == _ScaleMode.ROW_WISE_SHARDED:
1055
+ A_scale_shard = A_scale
1056
+ A_scale = torch.ops._c10d_functional.all_gather_into_tensor(
1057
+ A_scale.contiguous(), group_size, group_name
1058
+ )
1059
+ A_scale = torch.ops._c10d_functional.wait_tensor(A_scale)
1060
+ A_scale = (
1061
+ A_scale.view(group_size, *A_scale_shard.shape)
1062
+ .movedim(gather_dim + 1, 1)
1063
+ .flatten(0, -2)
1064
+ )
1065
+ elif scale_mode == _ScaleMode.ROW_WISE_REPLICATED:
1066
+ A_scale = A_scale.movedim(gather_dim, 0).flatten(0, -2)
1067
+ else:
1068
+ if scale_mode != _ScaleMode.TENSOR_WISE:
1069
+ raise AssertionError
1070
+
1071
+ def scaled_matmul(
1072
+ A: torch.Tensor,
1073
+ B: torch.Tensor,
1074
+ A_scale: torch.Tensor,
1075
+ B_scale: torch.Tensor,
1076
+ bias: torch.Tensor | None,
1077
+ result_scale: torch.Tensor | None,
1078
+ out_dtype: torch.dtype | None,
1079
+ use_fast_accum: bool,
1080
+ ) -> torch.Tensor:
1081
+ leading_dims = A.shape[:-1]
1082
+ res = torch.ops.aten._scaled_mm(
1083
+ A.flatten(0, -2),
1084
+ B,
1085
+ A_scale,
1086
+ B_scale,
1087
+ bias,
1088
+ result_scale,
1089
+ out_dtype=out_dtype,
1090
+ use_fast_accum=use_fast_accum,
1091
+ )
1092
+ return res.unflatten(0, leading_dims)
1093
+
1094
+ return A.movedim(0, gather_dim), [
1095
+ scaled_matmul(
1096
+ A, B, A_scale, B_scale, bias, result_scale, out_dtype, fast_accum
1097
+ ).movedim(0, gather_dim)
1098
+ for B, B_scale, bias, result_scale, out_dtype, fast_accum in zip(
1099
+ Bs, B_scales, biases, result_scales, out_dtypes, use_fast_accum
1100
+ )
1101
+ ]
1102
+
1103
+
1104
+ @torch.library.impl(lib, "fused_all_gather_scaled_matmul", "CUDA")
1105
+ @torch.library.impl(lib, "fused_all_gather_scaled_matmul", "XPU")
1106
+ def _fused_all_gather_scaled_matmul(
1107
+ A_shard: torch.Tensor,
1108
+ Bs: list[torch.Tensor],
1109
+ A_scale: torch.Tensor,
1110
+ B_scales: list[torch.Tensor],
1111
+ gather_dim: int,
1112
+ group_name: c10d.GroupName,
1113
+ biases: list[torch.Tensor | None],
1114
+ result_scales: list[torch.Tensor | None],
1115
+ out_dtypes: list[torch.dtype | None],
1116
+ use_fast_accum: list[bool],
1117
+ ) -> tuple[torch.Tensor, list[torch.Tensor]]:
1118
+ """
1119
+ Perform the following logic with micro-pipelined computation and
1120
+ communication:
1121
+
1122
+ A = all_gather_single(A_shard, gather_dim, group_name)
1123
+ leading_dims = A.shape[:-1]
1124
+ res = torch.ops.aten._scaled_mm(A.flatten(0, -2), B, A_scale, B_scale)
1125
+ res = res.unflatten(0, leading_dims)
1126
+
1127
+ The input `A_scale` can be tensor-wise, row-wise-sharded or
1128
+ row-wise-replicated.
1129
+
1130
+ Optimal stride order for `A_shard` - if `A_shard.movedim(gather_dim, 0)` is
1131
+ contiguous, no extra copy is required for input layout transformation.
1132
+ Otherwise A_shard needs to be copied once.
1133
+ """
1134
+ out_dtypes = _maybe_convert_scalar_types_to_dtypes(out_dtypes)
1135
+
1136
+ if len(biases) != len(Bs):
1137
+ raise ValueError("len(biases) must be the same as len(Bs)")
1138
+ if len(result_scales) != len(Bs):
1139
+ raise ValueError("len(result_scales) must be the same as len(Bs)")
1140
+ if len(out_dtypes) != len(Bs):
1141
+ raise ValueError("len(out_dtypes) must be the same as len(Bs)")
1142
+ if len(use_fast_accum) != len(Bs):
1143
+ raise ValueError("len(use_gast_accum_list) must be the same as len(Bs)")
1144
+
1145
+ if _is_test_mode:
1146
+ return _fused_all_gather_scaled_matmul_fallback(
1147
+ A_shard,
1148
+ Bs,
1149
+ A_scale,
1150
+ B_scales,
1151
+ gather_dim,
1152
+ group_name,
1153
+ biases,
1154
+ result_scales,
1155
+ out_dtypes,
1156
+ use_fast_accum,
1157
+ )
1158
+
1159
+ with torch.profiler.record_function("fused_all_gather_scaled_matmul"):
1160
+ A, res = _fused_all_gather_matmul_impl(
1161
+ torch.ops.aten._scaled_mm.out,
1162
+ A_shard,
1163
+ Bs,
1164
+ A_scale,
1165
+ [
1166
+ {
1167
+ "scale_b": B_scale,
1168
+ "bias": bias,
1169
+ "scale_result": result_scale,
1170
+ "out_dtype": out_dtype,
1171
+ "use_fast_accum": fast_accum,
1172
+ }
1173
+ for B_scale, bias, result_scale, out_dtype, fast_accum in zip(
1174
+ B_scales, biases, result_scales, out_dtypes, use_fast_accum
1175
+ )
1176
+ ],
1177
+ out_dtypes,
1178
+ gather_dim,
1179
+ group_name,
1180
+ True,
1181
+ )
1182
+ if A is None:
1183
+ raise AssertionError
1184
+ return A, res
1185
+
1186
+
1187
+ def make_contiguous_for_perm(
1188
+ t: torch.Tensor,
1189
+ perm: list[int],
1190
+ ) -> torch.Tensor:
1191
+ """
1192
+ Restride `t` such that `t.permute(perm)` is contiguous.
1193
+ """
1194
+ inv_perm = [0] * len(perm)
1195
+ for i, p in enumerate(perm):
1196
+ inv_perm[p] = i
1197
+ return t.permute(perm).contiguous().permute(inv_perm)
1198
+
1199
+
1200
+ def restride_A_shard_for_fused_all_gather_matmul(
1201
+ t: torch.Tensor,
1202
+ gather_dim: int,
1203
+ ) -> torch.Tensor:
1204
+ """
1205
+ Restride the `A_shard` arg of `fused_all_gather_matmul` for optimal perf.
1206
+ See the doc for `fused_all_gather_matmul` for detail.
1207
+ """
1208
+ perm = list(range(len(t.shape)))
1209
+ perm.insert(0, perm.pop(gather_dim))
1210
+ return make_contiguous_for_perm(t, perm)
1211
+
1212
+
1213
+ @torch.library.impl(lib, "fused_matmul_reduce_scatter", "CUDA")
1214
+ @torch.library.impl(lib, "fused_matmul_reduce_scatter", "XPU")
1215
+ def _fused_matmul_reduce_scatter(
1216
+ A: torch.Tensor,
1217
+ B: torch.Tensor,
1218
+ reduce_op: str,
1219
+ scatter_dim: int,
1220
+ group_name: c10d.GroupName,
1221
+ ) -> torch.Tensor:
1222
+ """
1223
+ Perform the following logic with micro-pipelined computation and
1224
+ communication:
1225
+
1226
+ reduce_scatter_single(A @ B, reduce_op, scatter_dim, group_name)
1227
+
1228
+ Optimal stride order for A - if A.movedim(scatter_dim, 0) is contiguous, no
1229
+ extra copy is required for input layout transformation. Otherwise A needs
1230
+ to be copied once.
1231
+ """
1232
+ if _is_test_mode:
1233
+ return _fused_matmul_reduce_scatter_fallback(
1234
+ A, B, reduce_op, scatter_dim, group_name
1235
+ )
1236
+
1237
+ with torch.profiler.record_function("fused_matmul_reduce_scatter"):
1238
+ return _fused_matmul_reduce_scatter_impl(
1239
+ mm_out_op=torch.ops.aten.mm.out,
1240
+ A=A,
1241
+ B=B,
1242
+ kwargs={},
1243
+ out_dtype=A.dtype,
1244
+ reduce_op=reduce_op,
1245
+ scatter_dim=scatter_dim,
1246
+ group_name=group_name,
1247
+ )
1248
+
1249
+
1250
+ @torch.library.impl(lib, "fused_matmul_reduce_scatter", "Meta")
1251
+ def _fused_matmul_reduce_scatter_fallback(
1252
+ A: torch.Tensor,
1253
+ B: torch.Tensor,
1254
+ reduce_op: str,
1255
+ scatter_dim: int,
1256
+ group_name: c10d.GroupName,
1257
+ ) -> torch.Tensor:
1258
+ res = funcol.reduce_scatter_single(A @ B, reduce_op, scatter_dim, group_name)
1259
+ res = funcol.wait_tensor(res)
1260
+ return res
1261
+
1262
+
1263
+ def _fused_matmul_reduce_scatter_impl(
1264
+ mm_out_op: torch._ops.OpOverload,
1265
+ A: torch.Tensor,
1266
+ B: torch.Tensor,
1267
+ kwargs: dict[str, Any],
1268
+ out_dtype: torch.dtype | None,
1269
+ reduce_op: str,
1270
+ scatter_dim: int,
1271
+ group_name: c10d.GroupName,
1272
+ ) -> torch.Tensor:
1273
+ if A.dim() < 2:
1274
+ raise ValueError("A_shard must be a matrix")
1275
+ if scatter_dim < 0 or scatter_dim >= A.dim():
1276
+ raise ValueError("Invalid gather_dim")
1277
+ if B.dim() != 2:
1278
+ raise ValueError("B must be a matrix")
1279
+ if reduce_op == "sum":
1280
+ reduce_fn = partial(torch.sum, dim=0)
1281
+ elif reduce_op == "avg":
1282
+ reduce_fn = partial(torch.mean, dim=0)
1283
+ else:
1284
+ raise ValueError("reduce_op must be sum or avg")
1285
+ group = c10d._resolve_process_group(group_name)
1286
+ out_shape = [*A.shape[:-1], B.shape[1]]
1287
+ out_shape[scatter_dim] //= group.size()
1288
+
1289
+ if scatter_dim == A.ndim - 1:
1290
+ B_shards = B.chunk(group.size(), dim=B.ndim - 1)
1291
+ A_flat = A.flatten(0, -2)
1292
+
1293
+ def _chunk_producer(rank: int, out: torch.Tensor) -> None:
1294
+ mm_out_op(A_flat, B_shards[rank], **kwargs, out=out)
1295
+
1296
+ leading_dims = list(A.shape[:-1])
1297
+
1298
+ stacked_partials = torch.empty(
1299
+ (A_flat.shape[0], B.shape[1]),
1300
+ dtype=out_dtype or A.dtype,
1301
+ device=A.device,
1302
+ )
1303
+
1304
+ _pipelined_produce_and_all2all(
1305
+ _chunk_producer,
1306
+ stacked_partials,
1307
+ group_name,
1308
+ out_chunk_dim=1,
1309
+ )
1310
+
1311
+ stacked_partials_view = stacked_partials.reshape(
1312
+ *leading_dims, group.size(), -1
1313
+ )
1314
+ return reduce_fn(
1315
+ stacked_partials_view,
1316
+ dim=-2,
1317
+ )
1318
+
1319
+ # Move the scatter_dim to the front and flatten the tensor into a 2D matrix
1320
+ x = A.movedim(scatter_dim, 0)
1321
+ leading_dims = [group.size()] + list(x.shape[:-1])
1322
+ leading_dims[1] //= group.size()
1323
+ x = x.flatten(0, -2)
1324
+ A_shards = x.chunk(group.size())
1325
+
1326
+ # Computing block-wise matmul along the first dim of A
1327
+ def chunk_producer(rank: int, out: torch.Tensor) -> None:
1328
+ mm_out_op(A_shards[rank], B, **kwargs, out=out)
1329
+
1330
+ stacked_partials = x.new_empty(x.shape[0], B.shape[1], dtype=out_dtype or A.dtype)
1331
+
1332
+ _pipelined_produce_and_all2all(
1333
+ chunk_producer,
1334
+ stacked_partials,
1335
+ group_name,
1336
+ )
1337
+
1338
+ # Ensures that the transpose and reduction produce contiguous result
1339
+ # in a single reduction kernel.
1340
+ return reduce_fn(
1341
+ stacked_partials.view(*leading_dims, -1)
1342
+ .movedim(1, scatter_dim + 1)
1343
+ .movedim(0, scatter_dim),
1344
+ dim=scatter_dim,
1345
+ )
1346
+
1347
+
1348
+ @torch.library.impl(lib, "fused_scaled_matmul_reduce_scatter", "CUDA")
1349
+ @torch.library.impl(lib, "fused_scaled_matmul_reduce_scatter", "XPU")
1350
+ def _fused_scaled_matmul_reduce_scatter(
1351
+ A: torch.Tensor,
1352
+ B: torch.Tensor,
1353
+ A_scale: torch.Tensor,
1354
+ B_scale: torch.Tensor,
1355
+ reduce_op: str,
1356
+ orig_scatter_dim: int,
1357
+ scatter_dim_after_maybe_reshape: int,
1358
+ group_name: c10d.GroupName,
1359
+ output_shape: list[int],
1360
+ bias: torch.Tensor | None = None,
1361
+ result_scale: torch.Tensor | None = None,
1362
+ out_dtype: torch.dtype | None = None,
1363
+ use_fast_accum: bool = False,
1364
+ ) -> torch.Tensor:
1365
+ if _is_test_mode:
1366
+ return _fused_scaled_matmul_reduce_scatter_fallback(
1367
+ A,
1368
+ B,
1369
+ A_scale,
1370
+ B_scale,
1371
+ reduce_op,
1372
+ orig_scatter_dim,
1373
+ scatter_dim_after_maybe_reshape,
1374
+ group_name,
1375
+ output_shape,
1376
+ bias,
1377
+ result_scale,
1378
+ out_dtype,
1379
+ use_fast_accum,
1380
+ )
1381
+ with torch.profiler.record_function("fused_scaled_matmul_reduce_scatter"):
1382
+ return _fused_scaled_matmul_reduce_scatter_impl(
1383
+ mm_out_op=torch.ops.aten._scaled_mm.out,
1384
+ A=A,
1385
+ B=B,
1386
+ A_scale=A_scale,
1387
+ kwargs={
1388
+ "scale_b": B_scale,
1389
+ "bias": bias,
1390
+ "scale_result": result_scale,
1391
+ "out_dtype": out_dtype,
1392
+ "use_fast_accum": use_fast_accum,
1393
+ },
1394
+ out_dtype=out_dtype,
1395
+ reduce_op=reduce_op,
1396
+ orig_scatter_dim=orig_scatter_dim,
1397
+ scatter_dim_after_maybe_reshape=scatter_dim_after_maybe_reshape,
1398
+ group_name=group_name,
1399
+ output_shape=output_shape,
1400
+ )
1401
+
1402
+
1403
+ @torch.library.impl(lib, "fused_scaled_matmul_reduce_scatter", "Meta")
1404
+ def _fused_scaled_matmul_reduce_scatter_fallback(
1405
+ A: torch.Tensor,
1406
+ B: torch.Tensor,
1407
+ A_scale: torch.Tensor,
1408
+ B_scale: torch.Tensor,
1409
+ reduce_op: str,
1410
+ orig_scatter_dim: int,
1411
+ scatter_dim_after_maybe_reshape: int,
1412
+ group_name: c10d.GroupName,
1413
+ output_shape: list[int],
1414
+ bias: torch.Tensor | None = None,
1415
+ result_scale: torch.Tensor | None = None,
1416
+ out_dtype: torch.dtype | None = None,
1417
+ use_fast_accum: bool = False,
1418
+ ) -> torch.Tensor:
1419
+ if A_scale.numel() > 1:
1420
+ if A_scale.shape[:-1] != A.shape[:-1]:
1421
+ raise ValueError(
1422
+ "For row-wise scaling, the leading dims of A_scale "
1423
+ "must match the leading dims of A "
1424
+ f"(A shape: {A.shape}, A_scale shape: {A_scale.shape})"
1425
+ )
1426
+ A_scale = A_scale.flatten(0, -2).contiguous()
1427
+ elif A_scale.numel() != 1:
1428
+ raise ValueError(
1429
+ "Invalid A_scale shape "
1430
+ f"(A shape: {A.shape}, A_scale shape: {A_scale.shape})"
1431
+ )
1432
+
1433
+ C = torch._scaled_mm(
1434
+ A.flatten(0, -2).contiguous(),
1435
+ B,
1436
+ A_scale,
1437
+ B_scale,
1438
+ bias,
1439
+ result_scale,
1440
+ out_dtype,
1441
+ use_fast_accum,
1442
+ )
1443
+ C = C.view(*output_shape[:-1], B.shape[1])
1444
+ res = funcol.reduce_scatter_single(
1445
+ C,
1446
+ reduce_op,
1447
+ orig_scatter_dim, # need original scatter dim for 3D+ output tensor here
1448
+ group_name,
1449
+ )
1450
+ res = funcol.wait_tensor(res)
1451
+ return res
1452
+
1453
+
1454
+ def _fused_scaled_matmul_reduce_scatter_impl(
1455
+ mm_out_op: torch._ops.OpOverload,
1456
+ A: torch.Tensor,
1457
+ B: torch.Tensor,
1458
+ A_scale: torch.Tensor,
1459
+ kwargs: dict[str, Any],
1460
+ out_dtype: torch.dtype | None,
1461
+ reduce_op: str,
1462
+ orig_scatter_dim: int,
1463
+ scatter_dim_after_maybe_reshape: int,
1464
+ group_name: c10d.GroupName,
1465
+ output_shape: list[int],
1466
+ ) -> torch.Tensor:
1467
+ if A.dim() < 2:
1468
+ raise ValueError("A_shard must be a matrix")
1469
+ if (
1470
+ scatter_dim_after_maybe_reshape < 0
1471
+ or scatter_dim_after_maybe_reshape >= A.dim()
1472
+ ):
1473
+ raise ValueError("Invalid scatter dim for 2D tensor input to scaled_mm")
1474
+ if orig_scatter_dim < 0 or orig_scatter_dim >= len(output_shape):
1475
+ raise ValueError("Invalid scatter dim for 3D+ output tensor")
1476
+ if B.dim() != 2:
1477
+ raise ValueError("B must be a matrix")
1478
+ if reduce_op == "sum":
1479
+ reduce_fn = partial(torch.sum, dim=0)
1480
+ elif reduce_op == "avg":
1481
+ reduce_fn = partial(torch.mean, dim=0)
1482
+ else:
1483
+ raise ValueError("reduce_op must be sum or avg")
1484
+
1485
+ group = c10d._resolve_process_group(group_name)
1486
+
1487
+ # Move scatter to first dim, then shard the tensor along the first dim, so the chunk producer
1488
+ # can perform matmuls along the first dim.
1489
+ A_with_scatter_dim_0 = A.movedim(scatter_dim_after_maybe_reshape, 0)
1490
+
1491
+ # To handle case where A is 3D+, reshape to 2D to prepare for mm which requires 2D inputs.
1492
+ A_2D_with_scatter_dim_0 = A_with_scatter_dim_0.flatten(0, -2)
1493
+
1494
+ # Partition A along the first dim to prepare for sharding across TP process group.
1495
+ A_shards = A_2D_with_scatter_dim_0.chunk(group.size())
1496
+
1497
+ # Now that 'A' is sharded along the first dim, we need to update its scale(s) accordingly.
1498
+ # How we do this depends on if we are using tensorwise scaling, rowwise scaling, or no scaling.
1499
+ tensorwise_scaling = A_scale is not None and A_scale.numel() == 1
1500
+ rowwise_scaling = A_scale is not None and A_scale.numel() > 1
1501
+
1502
+ # For tensorwise scaling, the scale should be replicated so each shard has a copy.
1503
+ if tensorwise_scaling:
1504
+ A_scale_shards = [A_scale] * group.size()
1505
+
1506
+ # For rowwise scaling, we need to move the scatter dim to the first dim to match the
1507
+ # dim swap of the 'A' tensor. Then we can shard the scales along the first dim, just like
1508
+ # the 'A' tensor.
1509
+ elif rowwise_scaling:
1510
+ if A_scale.shape[:-1] != A.shape[:-1]:
1511
+ raise ValueError(
1512
+ "For row-wise scaling, the leading dims of A_scale "
1513
+ "must match the leading dims of A "
1514
+ f"(A shape: {A.shape}, A_scale shape: {A_scale.shape})"
1515
+ )
1516
+ A_scale = (
1517
+ A_scale.movedim(scatter_dim_after_maybe_reshape, 0)
1518
+ .contiguous()
1519
+ .flatten(0, -2)
1520
+ )
1521
+ A_scale_shards = list(A_scale.chunk(group.size()))
1522
+ # cuBLAS's row-wise kernel requires scales to be aligned to 16 bytes.
1523
+ # When we slice them we might break this and need to reallocate them.
1524
+ A_scale_shards = [
1525
+ t if t.data_ptr() % 16 == 0 else t.clone() for t in A_scale_shards
1526
+ ]
1527
+ else:
1528
+ raise ValueError("A_scale cannot be none for scaled_mm")
1529
+
1530
+ # Computing block-wise matmul along the first dim of A
1531
+ def chunk_producer(rank: int, out: torch.Tensor) -> None:
1532
+ mm_out_op(A_shards[rank], B, scale_a=A_scale_shards[rank], **kwargs, out=out)
1533
+
1534
+ # Stacked partials will be the 2D outputs of the pipelined scaled mm, and will
1535
+ # have the shape (A_with_scatter_dim_0_tensor.shape[0], B.shape[1]) to align with the formula:
1536
+ # (a*b,c) @ (c,d) = (a*b,d)
1537
+ stacked_partials = A_with_scatter_dim_0.new_empty(
1538
+ A_2D_with_scatter_dim_0.shape[0], B.shape[1], dtype=out_dtype or A.dtype
1539
+ )
1540
+
1541
+ # Execute the pipelined mm/scaled_mm.
1542
+ _pipelined_produce_and_all2all(
1543
+ chunk_producer,
1544
+ stacked_partials,
1545
+ group_name,
1546
+ )
1547
+
1548
+ # We now need to transform the *unreduced* stacked 2D partial mm outputs to an *unreduced* 3D+ output,
1549
+ # then reduce-scatter. To do this, we first need to determine the shape of the unreduced 3D+ output,
1550
+ # to reshape our stacked partials so we can apply the reduce-scatter.
1551
+ #
1552
+ # The *unreduced* 3D+ tensor will have dim 0 = `group_size`, as we have `group_size` instances of
1553
+ # stacked partial outputs. The next dims will be A's leading dims (sharded along the original scatter dim),
1554
+ # as it was the left operand of the mm op. We can use -1 as the final dim of the view to populate the rest.
1555
+ stacked_partials_3D_leading_dims = [group.size()] + list(
1556
+ # We use A from after the dim swap 0<=>scatter_dim, but before the flatten,
1557
+ # to get the leading dims of the 3D+ view of stacked partials.
1558
+ A_with_scatter_dim_0.shape[:-1]
1559
+ )
1560
+
1561
+ # The `group_size` leading dim has been prepended to `stacked_partials_3D_leading_dims`,
1562
+ # to capture the partial output from each rank. We need to divide the sharding/scatter dim
1563
+ # by the group size. If the original scatter dim was 0, then it is now dim 1 in this
1564
+ # tensor, since this new `group_size` dim was prepended.
1565
+ stacked_partial_scatter_dim = orig_scatter_dim if orig_scatter_dim > 0 else 1
1566
+ stacked_partials_3D_leading_dims[stacked_partial_scatter_dim] //= group.size()
1567
+
1568
+ # Ensures that the transpose and reduction produce contiguous result
1569
+ # in a single reduction kernel.
1570
+ reduced_out = reduce_fn(
1571
+ # View 2D stacked partials as 3D+ tensor of shape (`group_size`, ...)
1572
+ stacked_partials.view(*stacked_partials_3D_leading_dims, -1)
1573
+ # We originally swapped 0<=>scatter_dim_after_maybe_reshape. Now after
1574
+ # prepending the `group_size` dim, to undo this original swap, we
1575
+ # must swap 1<=>scatter_dim_after_maybe_reshape+1.
1576
+ .movedim(1, scatter_dim_after_maybe_reshape + 1),
1577
+ # Reduce along the `group_size` dim (0).
1578
+ dim=0,
1579
+ )
1580
+
1581
+ # Output shape must be scattered along original scatter dim as well.
1582
+ output_shape[orig_scatter_dim] //= group.size()
1583
+ out = reduced_out.view(*output_shape)
1584
+ return out
1585
+
1586
+
1587
+ def restride_A_for_fused_matmul_reduce_scatter(
1588
+ t: torch.Tensor,
1589
+ scatter_dim: int,
1590
+ ) -> torch.Tensor:
1591
+ """
1592
+ Restride the `A_shard` arg of `fused_matmul_reduce_scatter` for optimal
1593
+ perf. See the doc for `fused_matmul_reduce_scatter` for detail.
1594
+ """
1595
+ perm = list(range(len(t.shape)))
1596
+ perm.insert(0, perm.pop(scatter_dim))
1597
+ return make_contiguous_for_perm(t, perm)
1598
+
1599
+
1600
+ def _maybe_convert_scalar_types_to_dtypes(
1601
+ scalar_types: list[Any],
1602
+ ) -> list[torch.dtype | None]:
1603
+ """
1604
+ When a list of `torch.dtype`s is passed through the dispatcher as
1605
+ `ScalarType[]`, it is converted to a list of scalar type enum values. This
1606
+ function converts it back to a list of `torch.dtype`s.
1607
+ """
1608
+ # Order defined in https://github.com/pytorch/pytorch/blob/344defc9733a45fee8d0c4d3f5530f631e823196/c10/core/ScalarType.h
1609
+ _SCALAR_TYPE_TO_DTYPE = {
1610
+ 0: torch.uint8,
1611
+ 1: torch.int8,
1612
+ 2: torch.short,
1613
+ 3: torch.int,
1614
+ 4: torch.int64,
1615
+ 5: torch.half,
1616
+ 6: torch.float,
1617
+ 7: torch.double,
1618
+ 8: torch.complex32,
1619
+ 9: torch.complex64,
1620
+ 10: torch.complex128,
1621
+ 11: torch.bool,
1622
+ 12: torch.qint8,
1623
+ 13: torch.quint8,
1624
+ 14: torch.qint32,
1625
+ 15: torch.bfloat16,
1626
+ 16: torch.float8_e5m2,
1627
+ 17: torch.float8_e4m3fn,
1628
+ 18: torch.float8_e5m2fnuz,
1629
+ 19: torch.float8_e4m3fnuz,
1630
+ }
1631
+ if any(not isinstance(x, (type(None), int)) for x in scalar_types):
1632
+ return scalar_types
1633
+
1634
+ dtypes: list[torch.dtype | None] = []
1635
+ for scalar_type in scalar_types:
1636
+ if scalar_type is None:
1637
+ dtypes.append(scalar_type)
1638
+ elif scalar_type not in _SCALAR_TYPE_TO_DTYPE:
1639
+ raise ValueError(f"Unrecognized scalar type {scalar_type}")
1640
+ else:
1641
+ dtypes.append(_SCALAR_TYPE_TO_DTYPE[scalar_type])
1642
+ return dtypes
1643
+
1644
+
1645
+ class Work(_Work):
1646
+ def __init__(self) -> None:
1647
+ super().__init__()
1648
+ self.event = torch.Event()
1649
+ self.event.record()
1650
+
1651
+ def wait(self, timeout: timedelta = timedelta(seconds=0)) -> bool:
1652
+ self.event.wait()
1653
+ return True
1654
+
1655
+
1656
+ """
1657
+ NOTE [low-contention collectives]
1658
+ When a collective is overlapped with abundant compute, it makes sense to
1659
+ prioritize reducing the contention between the collective and the overlapped
1660
+ compute, even at the cost of a slightly slower collective.
1661
+
1662
+ Common collective implementations (e.g., NCCL without user buffer
1663
+ registration) optimize for throughput with no ambient compute. However, such
1664
+ implementations may not be optimal when they are overlapped with compute:
1665
+ - These implementations typically fuse the entire collective into a single
1666
+ kernel and reserve SM resources based on the most demanding portion of the
1667
+ collective, even when a large portion of the collective does not require this
1668
+ much resource.
1669
+ - These implementations often use SM-based P2P copy as opposed to copy
1670
+ engine-based P2P copy. Copy engine-based P2P copy may not have a significant
1671
+ advantage when there's no ambient compute. However, it may significantly
1672
+ improve overall resource utilization in the presence of ambient compute.
1673
+
1674
+ When overlapped with intensive compute (e.g., persistent matmul kernels), the
1675
+ SM-usage of a collective can lead to inefficient overlapping.
1676
+
1677
+ Low-contention collectives achieve their goals with the following strategies:
1678
+ - Use copy engine-based copy whenever possible.
1679
+ - Break down portions of a collective with different resource requirements
1680
+ into multiple kernels. This improves the overlapping efficiency at the cost
1681
+ of additional launching overhead.
1682
+ """
1683
+
1684
+
1685
+ @torch.library.impl(lib, "_low_contention_all_gather", "Meta")
1686
+ def _low_contention_all_gather_meta(
1687
+ tensor: torch.Tensor,
1688
+ group_name: c10d.GroupName,
1689
+ ) -> torch.Tensor:
1690
+ group_size = c10d._get_group_size_by_name(group_name)
1691
+ return tensor.new_empty(tensor.shape[0] * group_size, *tensor.shape[1:])
1692
+
1693
+
1694
+ @torch.library.impl(lib, "_low_contention_all_gather", "CUDA")
1695
+ def _low_contention_all_gather(
1696
+ tensor: torch.Tensor,
1697
+ group_name: c10d.GroupName,
1698
+ ) -> torch.Tensor:
1699
+ """
1700
+ Performs all-gather with symmetric memory in a low-contention fashion.
1701
+
1702
+ When `tensor` is already in symmetric memory:
1703
+ - The collective is carried out without using SMs.
1704
+ - No symmetric memory workspace is required.
1705
+
1706
+ When `tensor` is not in symmetric memory:
1707
+ - An extra SM-based copy is performed to copy the input data into the
1708
+ symmetric memory workspace.
1709
+ - Symmetric memory workspace size requirement: the size of `tensor`.
1710
+ """
1711
+ symm_mem = rendezvous(tensor, group_name)
1712
+ if symm_mem is not None:
1713
+ input_is_symm_mem = True
1714
+ else:
1715
+ symm_mem = get_symm_mem_workspace(
1716
+ group_name, tensor.numel() * tensor.element_size()
1717
+ )
1718
+ input_is_symm_mem = False
1719
+
1720
+ rank = symm_mem.rank
1721
+ world_size = symm_mem.world_size
1722
+
1723
+ output = tensor.new_empty(tensor.shape[0] * world_size, *tensor.shape[1:])
1724
+ chunks = output.chunk(world_size)
1725
+
1726
+ _get_backend_stream().wait_stream(torch.accelerator.current_stream())
1727
+ with _get_backend_stream():
1728
+ if not input_is_symm_mem:
1729
+ local_buf = symm_mem.get_buffer(rank, tensor.shape, tensor.dtype)
1730
+ local_buf.copy_(tensor)
1731
+ # pull
1732
+ symm_mem.barrier()
1733
+ for step in range(world_size):
1734
+ remote_rank = (rank - step) % world_size
1735
+ src_buf = symm_mem.get_buffer(remote_rank, tensor.shape, tensor.dtype)
1736
+ chunks[remote_rank].copy_(src_buf)
1737
+ symm_mem.barrier()
1738
+ torch._C._distributed_c10d._register_work(output, Work())
1739
+ return output
1740
+
1741
+
1742
+ @torch.library.impl(lib, "_low_contention_reduce_scatter", "Meta")
1743
+ def _low_contention_reduce_scatter_meta(
1744
+ tensor: torch.Tensor,
1745
+ reduce_op: str,
1746
+ group_name: c10d.GroupName,
1747
+ ) -> torch.Tensor:
1748
+ group_size = c10d._get_group_size_by_name(group_name)
1749
+ return tensor.unflatten(0, (group_size, -1)).mean(dim=0)
1750
+
1751
+
1752
+ def _low_contention_reduce_scatter_with_symm_mem_input(
1753
+ tensor: torch.Tensor,
1754
+ reduce_op: str,
1755
+ symm_mem: _SymmetricMemory,
1756
+ ) -> torch.Tensor:
1757
+ rank = symm_mem.rank
1758
+ world_size = symm_mem.world_size
1759
+
1760
+ if tensor.shape[0] % world_size != 0:
1761
+ raise AssertionError
1762
+ a2a_res = torch.empty_like(tensor)
1763
+ chunks = a2a_res.chunk(world_size)
1764
+
1765
+ _get_backend_stream().wait_stream(torch.accelerator.current_stream())
1766
+ with _get_backend_stream():
1767
+ # pull + offline reduction
1768
+ symm_mem.barrier()
1769
+ for step in range(world_size):
1770
+ remote_rank = (rank - step) % world_size
1771
+ src_buf = symm_mem.get_buffer(
1772
+ remote_rank,
1773
+ chunks[0].shape,
1774
+ chunks[0].dtype,
1775
+ chunks[0].numel() * rank,
1776
+ )
1777
+ chunks[remote_rank].copy_(src_buf)
1778
+ symm_mem.barrier()
1779
+
1780
+ ret = a2a_res.unflatten(0, (world_size, -1))
1781
+ if reduce_op == "sum":
1782
+ ret = ret.sum(dim=0)
1783
+ elif reduce_op == "avg":
1784
+ ret = ret.mean(dim=0)
1785
+ else:
1786
+ raise ValueError(f"reduce_op ({reduce_op}) is not supported")
1787
+ torch._C._distributed_c10d._register_work(ret, Work())
1788
+ return ret
1789
+
1790
+
1791
+ def _low_contention_reduce_scatter_with_workspace(
1792
+ tensor: torch.Tensor,
1793
+ reduce_op: str,
1794
+ workspace: _SymmetricMemory,
1795
+ ) -> torch.Tensor:
1796
+ rank = workspace.rank
1797
+ world_size = workspace.world_size
1798
+
1799
+ if tensor.shape[0] % world_size != 0:
1800
+ raise AssertionError
1801
+ chunks = tensor.chunk(world_size)
1802
+
1803
+ _get_backend_stream().wait_stream(torch.accelerator.current_stream())
1804
+ with _get_backend_stream():
1805
+ # push + offline reduction
1806
+ workspace.barrier()
1807
+ for step in range(world_size):
1808
+ remote_rank = (rank - step) % world_size
1809
+ dst_buf = workspace.get_buffer(
1810
+ remote_rank, chunks[0].shape, chunks[0].dtype, chunks[0].numel() * rank
1811
+ )
1812
+ dst_buf.copy_(chunks[remote_rank])
1813
+ workspace.barrier()
1814
+
1815
+ buf = workspace.get_buffer(rank, tensor.shape, tensor.dtype)
1816
+ ret = buf.unflatten(0, (world_size, -1))
1817
+ if reduce_op == "sum":
1818
+ ret = ret.sum(dim=0)
1819
+ elif reduce_op == "avg":
1820
+ ret = ret.mean(dim=0)
1821
+ else:
1822
+ raise ValueError(f"reduce_op ({reduce_op}) is not supported")
1823
+ torch._C._distributed_c10d._register_work(ret, Work())
1824
+ return ret
1825
+
1826
+
1827
+ @torch.library.impl(lib, "_low_contention_reduce_scatter", "CUDA")
1828
+ def _low_contention_reduce_scatter(
1829
+ tensor: torch.Tensor,
1830
+ reduce_op: str,
1831
+ group_name: c10d.GroupName,
1832
+ ) -> torch.Tensor:
1833
+ """
1834
+ Performs reduce-scatter with symmetric memory in a low-contention fashion.
1835
+
1836
+ This implementation performs a P2P-based all-to-all followed by an offline
1837
+ reduction.
1838
+
1839
+ When `tensor` is already in symmetric memory:
1840
+ - Pull-based all-to-all is used.
1841
+ - No symmetric memory workspace is required.
1842
+
1843
+ When `tensor` is not in symmetric memory:
1844
+ - Push-based all-to-all is used.
1845
+ - Symmetric memory workspace size requirement: the size of `tensor`.
1846
+
1847
+ SM-usage:
1848
+ - SM-based copy of the rank's own chunk for the all-to-all.
1849
+ - Reduction on the all-to-all result.
1850
+
1851
+ TODO(yifu): the SM-based copy can be avoided with a list-based reduction
1852
+ kernel.
1853
+ """
1854
+ symm_mem = rendezvous(tensor, group_name)
1855
+ if symm_mem is not None:
1856
+ return _low_contention_reduce_scatter_with_symm_mem_input(
1857
+ tensor, reduce_op, symm_mem
1858
+ )
1859
+ else:
1860
+ workspace = get_symm_mem_workspace(
1861
+ group_name, tensor.numel() * tensor.element_size()
1862
+ )
1863
+ return _low_contention_reduce_scatter_with_workspace(
1864
+ tensor, reduce_op, workspace
1865
+ )
1866
+
1867
+
1868
+ @torch.library.impl(lib, "all_to_all_vdev_2d", "Meta")
1869
+ def _all_to_all_vdev_2d_meta(
1870
+ input: torch.Tensor,
1871
+ out: torch.Tensor,
1872
+ in_splits: torch.Tensor,
1873
+ out_splits_offsets: torch.Tensor,
1874
+ group_name: c10d.GroupName,
1875
+ major_align: int | None = None,
1876
+ ) -> None:
1877
+ return None
1878
+
1879
+
1880
+ @torch.library.impl(lib, "all_to_all_vdev_2d_offset", "Meta")
1881
+ def _all_to_all_vdev_2d_offset_meta(
1882
+ input: torch.Tensor,
1883
+ out: torch.Tensor,
1884
+ in_splits_offsets: torch.Tensor,
1885
+ out_splits_offsets: torch.Tensor,
1886
+ group_name: c10d.GroupName,
1887
+ ) -> None:
1888
+ return None
1889
+
1890
+
1891
+ # =============================================================================
1892
+ # User-facing APIs
1893
+ # =============================================================================
1894
+
1895
+
1896
+ from collections.abc import Sequence
1897
+ from typing import overload, TYPE_CHECKING, Union
1898
+
1899
+
1900
+ if TYPE_CHECKING:
1901
+ from torch._C._distributed_c10d import ProcessGroup
1902
+ from torch.types import _device, _dtype, _int
1903
+
1904
+
1905
+ _use_implicit_mempool: bool | None = None # type: ignore[assignment]
1906
+
1907
+
1908
+ def _should_use_implicit_mempool() -> bool:
1909
+ r"""
1910
+ Check if the implicit memory pool should be used for symmetric memory allocations.
1911
+
1912
+ Returns:
1913
+ bool: True if the implicit memory pool should be used, False otherwise.
1914
+
1915
+ By default, use implicit memory pool for `symm_mem.empty`. Users can
1916
+ disable this by setting the environment variable `TORCH_SYMMMEM_IMPLICIT_POOL` to `0`.
1917
+ """
1918
+ global _use_implicit_mempool
1919
+ if _use_implicit_mempool is None:
1920
+ _use_implicit_mempool = os.getenv("TORCH_SYMMMEM_IMPLICIT_POOL", "1") == "1"
1921
+
1922
+ return _use_implicit_mempool
1923
+
1924
+
1925
+ @overload
1926
+ def empty(
1927
+ *size: _int, dtype: _dtype | None = None, device: _device | None = None
1928
+ ) -> torch.Tensor: ...
1929
+
1930
+
1931
+ @overload
1932
+ # pyrefly: ignore [inconsistent-overload]
1933
+ def empty(
1934
+ size: Sequence[_int],
1935
+ *,
1936
+ dtype: _dtype | None = None,
1937
+ device: _device | None = None,
1938
+ ) -> torch.Tensor: ...
1939
+
1940
+
1941
+ def empty( # type: ignore[misc]
1942
+ *size: Any,
1943
+ dtype: _dtype | None = None,
1944
+ device: _device | None = None,
1945
+ ) -> torch.Tensor:
1946
+ r"""
1947
+ Similar to :func:`torch.empty()`. The returned tensor can be used by
1948
+ :func:`torch._distributed._symmetric_memory.rendezvous()` to establish a
1949
+ symmetric memory tensor among participating processes.
1950
+
1951
+ Args:
1952
+ size (int...): a sequence of integers defining the shape of the output tensor.
1953
+ Can be a variable number of arguments or a collection like a list or tuple.
1954
+
1955
+ Keyword args:
1956
+ dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor.
1957
+ Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`).
1958
+ device (:class:`torch.device`, optional): the desired device of returned tensor.
1959
+ Default: if ``None``, uses the current device for the default tensor type
1960
+ (see :func:`torch.set_default_device`). :attr:`device` will be the CPU
1961
+ for CPU tensor types and the current CUDA device for CUDA tensor types.
1962
+ """
1963
+ if len(size) == 1 and isinstance(size[0], Sequence):
1964
+ size = tuple(size[0])
1965
+ else:
1966
+ size = tuple(size)
1967
+
1968
+ if dtype is None:
1969
+ dtype = torch.get_default_dtype()
1970
+
1971
+ if device is None:
1972
+ device = torch.get_default_device()
1973
+ else:
1974
+ device = torch.device(device)
1975
+
1976
+ stride = torch._prims_common.make_contiguous_strides_for(size)
1977
+
1978
+ if _should_use_implicit_mempool() and device.type == "cuda":
1979
+ # Allocate tensor from an implicit memory pool
1980
+ mempool = get_mem_pool(device)
1981
+ # TODO: this path can be made device-agnostic if `use_mem_pool` is
1982
+ # elevated from torch.cuda to torch accelerator.
1983
+ with torch.cuda.use_mem_pool(mempool):
1984
+ return _SymmetricMemory.empty_strided_p2p(size, stride, dtype, device)
1985
+ else:
1986
+ return _SymmetricMemory.empty_strided_p2p(size, stride, dtype, device)
1987
+
1988
+
1989
+ def _resolve_group_name(group: c10d.GroupName | ProcessGroup) -> c10d.GroupName:
1990
+ from torch._C._distributed_c10d import ProcessGroup
1991
+
1992
+ if isinstance(group, str):
1993
+ return c10d.GroupName(group)
1994
+ if isinstance(group, ProcessGroup):
1995
+ return group.group_name
1996
+ raise TypeError(f"unsupported group type: {type(group)}")
1997
+
1998
+
1999
+ def rendezvous(
2000
+ tensor: torch.Tensor, group: c10d.GroupName | ProcessGroup
2001
+ ) -> _SymmetricMemory:
2002
+ r"""
2003
+ rendezvous(tensor, group) -> _SymmetricMemory
2004
+
2005
+ Establish a symmetric memory tensor among participating processes. This is
2006
+ a collective operation.
2007
+
2008
+ Args:
2009
+ tensor (:class:`torch.Tensor`): the local tensor used to establish the symmetric memory tensor.
2010
+ It must be allocated via :func:`torch._distributed._symmetric_memory.empty()`. The shape,
2011
+ dtype, and device type must be identical across all participating processes.
2012
+ group (Union[str, :class:`torch.distributed.ProcessGroup`]): The group identifying the
2013
+ participating processes. This can be either a group name or a process group object.
2014
+ """
2015
+ group_name = _resolve_group_name(group)
2016
+ return _SymmetricMemory.rendezvous(tensor, group_name)
2017
+
2018
+
2019
+ def is_nvshmem_available() -> bool:
2020
+ r"""
2021
+ is_nvshmem_available() -> bool
2022
+
2023
+ Check if NVSHMEM (CUDA) or rocSHMEM (ROCm) is available in the current
2024
+ build and usable at runtime. On ROCm, rocSHMEM ``VERSION`` must be at
2025
+ least 3.3.0 (see ``rocshmem/rocshmem.hpp``).
2026
+ """
2027
+ try:
2028
+ from torch._C._distributed_c10d import _is_nvshmem_available
2029
+ except ImportError:
2030
+ # Not all builds have NVSHMEM support.
2031
+ return False
2032
+
2033
+ # Check if NVSHMEM is available on current system.
2034
+ return _is_nvshmem_available()
2035
+
2036
+
2037
+ def set_backend(name: Literal["NVSHMEM", "CUDA", "NCCL"]) -> None:
2038
+ r"""
2039
+ Set the backend for symmetric memory allocation. This is a global setting
2040
+ and affects all subsequent calls to
2041
+ :func:`torch._distributed._symmetric_memory.empty()`. Note that the backend
2042
+ cannot be changed once a symmetric memory tensor has been allocated.
2043
+
2044
+ Args:
2045
+ backend (str): the backend for symmetric memory allocation. Currently,
2046
+ only `"NVSHMEM"`, `"CUDA"`, `"NCCL"` are supported.
2047
+ """
2048
+ _SymmetricMemory.set_backend(name)
2049
+
2050
+
2051
+ def get_backend(device: _device) -> str | None:
2052
+ r"""
2053
+ Get the backend for symmetric memory allocation for a given device. If not
2054
+ found, return None.
2055
+
2056
+ Args:
2057
+ device (`torch.device` or str): the device for which to get the backend.
2058
+ """
2059
+ return _SymmetricMemory.get_backend(torch.device(device))
2060
+
2061
+
2062
+ def get_mempool_allocator(device: _device): # type: ignore[no-untyped-def]
2063
+ r"""
2064
+ Get the MemPool allocator for symmetric memory for a given device.
2065
+
2066
+ Args:
2067
+ device (`torch.device` or str): the device for which to get the MemPool
2068
+ allocator.
2069
+ """
2070
+ return _SymmetricMemory.get_mempool_allocator(torch.device(device))
2071
+
2072
+
2073
+ def set_signal_pad_size(size: int) -> None:
2074
+ r"""
2075
+ Set the signal pad size for future symmetric memory allocations.
2076
+
2077
+ Signal pads are P2P-accessible memory regions used for synchronization in
2078
+ symmetric memory. This function allows users to configure
2079
+ the signal pad size to be proportional to their workload requirements.
2080
+
2081
+ .. warning::
2082
+ This must be called before any symmetric memory allocations are made.
2083
+ The size cannot be changed after allocations have been performed.
2084
+
2085
+ Args:
2086
+ size (int): the signal pad size in bytes. The size should be
2087
+ proportional to the number of blocks launched and the world size.
2088
+
2089
+ Example::
2090
+
2091
+ >>> # doctest: +SKIP
2092
+ >>> # Set a larger signal pad size before any allocations
2093
+ >>> torch.distributed._symmetric_memory.set_signal_pad_size(1024 * 1024) # 1MB
2094
+ """
2095
+ _SymmetricMemory.signal_pad_size = size
2096
+
2097
+
2098
+ def get_signal_pad_size() -> int:
2099
+ r"""
2100
+ Get the current signal pad size for symmetric memory allocations.
2101
+
2102
+ Returns the user-configured size if set via :func:`set_signal_pad_size`,
2103
+ otherwise returns the default size.
2104
+
2105
+ Returns:
2106
+ int: the signal pad size in bytes.
2107
+
2108
+ Example::
2109
+
2110
+ >>> # doctest: +SKIP
2111
+ >>> size = torch.distributed._symmetric_memory.get_signal_pad_size()
2112
+ >>> print(f"Signal pad size: {size} bytes")
2113
+ """
2114
+ return _SymmetricMemory.signal_pad_size
2115
+
2116
+
2117
+ # An internal map from device to the symmetric memory pool for that device.
2118
+ _symm_mem_pools: dict[_device, torch.cuda.MemPool] = {}
2119
+
2120
+
2121
+ def get_mem_pool(device: _device) -> torch.cuda.MemPool:
2122
+ """
2123
+ Get the symmetric memory pool for a given device. If not found, create a new
2124
+ pool.
2125
+
2126
+ The tensor allocations with this pool must be symmetric across ranks. The
2127
+ allocated tensors can be used with symmetric operations, for example,
2128
+ operations defined under `torch.ops.symm_mem`.
2129
+
2130
+ Args:
2131
+ device (`torch.device` or str): the device for which to get the symmetric memory pool.
2132
+
2133
+ Returns:
2134
+ `torch.cuda.MemPool`: the symmetric memory pool for the given device.
2135
+
2136
+ Example::
2137
+
2138
+ >>> # doctest: +SKIP
2139
+ >>> pool = torch.distributed._symmetric_memory.get_mem_pool("cuda:0")
2140
+ >>> with torch.cuda.use_mem_pool(pool):
2141
+ >>> tensor = torch.randn(1000, device="cuda:0")
2142
+ >>> tensor = torch.ops.symm_mem.one_shot_all_reduce(tensor, "sum", group_name)
2143
+
2144
+ """
2145
+ # This function is a wrapper around the `torch.cuda.MemPool` constructor.
2146
+ # Due to special requirements of SymmetricMemory, we preset certain options for the pool.
2147
+ # - use_on_oom=False: we don't want to lend the space of the pool for
2148
+ # non-symmetric allocations because this could desync the allocation state
2149
+ # across ranks.
2150
+ # - no_split=True: we don't want to split segments, because today a segment
2151
+ # is associated with a signal pad, if two allocated tensors share a segment
2152
+ # and their kernels concurrently use (the same) signal pad, this could cause
2153
+ # undefined behaviors. We could consider relaxing this in the future if we
2154
+ # establish stream tracking and implicit synchronization around an
2155
+ # allocation.
2156
+ if device not in _symm_mem_pools:
2157
+ allocator = get_mempool_allocator(device)
2158
+ # Create a new pool with the given allocator and the preset options.
2159
+ _symm_mem_pools[device] = torch.cuda.MemPool(
2160
+ allocator,
2161
+ use_on_oom=False,
2162
+ no_split=True,
2163
+ )
2164
+
2165
+ return _symm_mem_pools[device]
2166
+
2167
+
2168
+ def _cuda_get_out(
2169
+ dst: torch.Tensor, hdl: _SymmetricMemory, offset: int, size: int, peer: int
2170
+ ) -> None:
2171
+ storage_offset = hdl.offset // dst.element_size() + offset
2172
+ remote_src = hdl.get_buffer(peer, (size,), dst.dtype, storage_offset)
2173
+ dst.view(-1)[:size].copy_(remote_src)
2174
+
2175
+
2176
+ # One-sided communication APIs.
2177
+ def get(
2178
+ dst: torch.Tensor,
2179
+ hdl: _SymmetricMemory,
2180
+ peer: int,
2181
+ offset: int = 0,
2182
+ ) -> None:
2183
+ r"""
2184
+ get(dst, hdl, peer, offset=0) -> ()
2185
+
2186
+ Copy ``dst.numel()`` elements starting at ``offset`` from ``peer``'s
2187
+ symmetric allocation into local ``dst`` using one-sided symmetric memory
2188
+ access.
2189
+
2190
+ ``hdl`` is the symmetric memory handle returned by
2191
+ :func:`torch.distributed._symmetric_memory.rendezvous`; the remote source
2192
+ is ``peer``'s allocation backing that handle. The number of elements copied
2193
+ is inferred from ``dst``; pass a view (e.g. ``dst[:n]``) to fill only part
2194
+ of a tensor. ``offset`` is expressed in elements of ``dst``'s dtype.
2195
+ ``dst`` can be a regular CUDA tensor or a symmetric-memory tensor; it must
2196
+ be on the same device as ``hdl`` and backed by contiguous memory. The copy
2197
+ is issued on the current CUDA stream.
2198
+
2199
+ Args:
2200
+ dst (Tensor): local destination tensor.
2201
+ hdl (SymmetricMemory): handle whose peer allocation is the remote
2202
+ source.
2203
+ peer (int): rank to copy from.
2204
+ offset (int, optional): element offset into the peer allocation to
2205
+ start reading from. Defaults to ``0``.
2206
+ """
2207
+ if dst.device != hdl.device:
2208
+ raise ValueError("get: dst must be on the same device as hdl")
2209
+ if not dst.is_contiguous():
2210
+ raise ValueError("get: dst must be backed by contiguous memory")
2211
+ if offset < 0:
2212
+ raise ValueError("get: offset must be non-negative")
2213
+ if peer < 0 or peer >= hdl.world_size:
2214
+ raise ValueError("get: invalid peer")
2215
+ size = dst.numel()
2216
+ element_size = dst.element_size()
2217
+ if hdl.offset % element_size != 0:
2218
+ raise RuntimeError("get: handle offset is not element-aligned")
2219
+ start = hdl.offset + offset * element_size
2220
+ end = start + size * element_size
2221
+ if start > hdl.buffer_size or end > hdl.buffer_size:
2222
+ raise ValueError("get: requested range exceeds symmetric allocation")
2223
+
2224
+ backend = get_backend(dst.device)
2225
+ if backend == "CUDA":
2226
+ _cuda_get_out(dst, hdl, offset, size, peer)
2227
+ return
2228
+
2229
+ # `hdl` is a pybind `_SymmetricMemory` object. Dispatcher expects the
2230
+ # TorchBind custom class type `__torch__.torch.classes.c10d.SymmetricMemory`.
2231
+ # Convert via `.boxed()`.
2232
+ hdl_boxed = hdl.boxed() if hasattr(hdl, "boxed") else hdl
2233
+ if backend == "NVSHMEM":
2234
+ torch.ops.symm_mem.nvshmem_get_out(dst, hdl_boxed, offset, size, peer)
2235
+ elif backend == "NCCL":
2236
+ torch.ops.symm_mem.nccl_get_out(dst, hdl_boxed, offset, size, peer)
2237
+ else:
2238
+ raise ValueError(f"get: unsupported backend: {backend}")
2239
+
2240
+
2241
+ def put_signal(src: torch.Tensor, hdl: _SymmetricMemory, peer: int) -> None:
2242
+ r"""
2243
+ put_signal(src, hdl, peer) -> None
2244
+
2245
+ Put data to a peer's symmetric memory and signal the peer.
2246
+
2247
+ Args:
2248
+ src (torch.Tensor): the source tensor to read data from.
2249
+ hdl (SymmetricMemory): the symmetric memory to put data to.
2250
+ peer (int): the peer to put data to.
2251
+ """
2252
+ backend = get_backend(src.device)
2253
+ # `hdl` is a pybind `_SymmetricMemory` object. Dispatcher expects the
2254
+ # TorchBind custom class type `__torch__.torch.classes.c10d.SymmetricMemory`.
2255
+ # Convert via `.boxed()`.
2256
+ hdl_boxed = hdl.boxed() if hasattr(hdl, "boxed") else hdl
2257
+ if backend == "NCCL":
2258
+ torch.ops.symm_mem.nccl_put_signal(src, hdl_boxed, peer)
2259
+ # TODO: other backends' dispatch goes here
2260
+ else:
2261
+ raise ValueError(f"put_signal: unsupported backend: {backend}")
2262
+
2263
+
2264
+ def wait_signal(hdl: _SymmetricMemory, peer: int) -> None:
2265
+ r"""
2266
+ wait_signal(hdl, peer) -> None
2267
+
2268
+ Wait for a signal from a peer.
2269
+
2270
+ Args:
2271
+ hdl (SymmetricMemory): the symmetric memory handle on which to wait for a signal.
2272
+ peer (int): the peer to wait for a signal from.
2273
+ """
2274
+ backend = get_backend(hdl.device)
2275
+ # See note in `put_signal` about `_SymmetricMemory` vs TorchBind type.
2276
+ hdl_boxed = hdl.boxed() if hasattr(hdl, "boxed") else hdl
2277
+ if backend == "NCCL":
2278
+ torch.ops.symm_mem.nccl_wait_signal(hdl_boxed, peer)
2279
+ # TODO: other backends' dispatch goes here
2280
+ else:
2281
+ raise ValueError(f"wait_signal: unsupported backend: {backend}")
2282
+
2283
+
2284
+ def reduce_scatter_offset(
2285
+ input: torch.Tensor,
2286
+ out: list[torch.Tensor],
2287
+ group: str,
2288
+ *,
2289
+ dim: int,
2290
+ offsets: list[int] | None = None,
2291
+ dst_ranks: list[int] | None = None,
2292
+ red_op: str = "sum",
2293
+ ) -> None:
2294
+ r"""
2295
+ reduce_scatter_offset(input, out, group, *, dim, offsets, dst_ranks, red_op='sum') -> None
2296
+
2297
+ Simultaneously reduce N blocks of a 2-D ``input`` tensor from a symmetric
2298
+ memory buffer, routing each block to a specific destination rank. Only
2299
+ ``dst_ranks[i]`` writes the reduced result for block ``i``; the result is
2300
+ written to a contiguous output tensor, with the same shape as block ``i``.
2301
+
2302
+ The ``dim`` argument controls which dimension is sharded:
2303
+
2304
+ - ``dim=0`` (row sharding): block ``i`` spans
2305
+ ``input[offsets[i-1] : offsets[i], :]``. Each ``out[j]`` has shape
2306
+ ``(size_j, input.size(1))``.
2307
+ - ``dim=1`` (column sharding): block ``i`` spans
2308
+ ``input[:, offsets[i-1] : offsets[i]]``. Each ``out[j]`` has shape
2309
+ ``(input.size(0), size_j)``.
2310
+
2311
+ Blocks are described by ``offsets``, an inclusive prefix-sum of block sizes
2312
+ along ``dim`` (first block starts at index 0 by convention). Block offsets
2313
+ can be even or uneven; when uneven, the following condition must be met: for
2314
+ each ``j``, the ``j``-th owned block must have the same size across all
2315
+ ranks (so that ``out[j]`` has a uniform shape); different ``j``'s may
2316
+ differ.
2317
+
2318
+ Args:
2319
+ input (Tensor): 2-D tensor allocated via symmetric memory (innermost
2320
+ dimension must be contiguous).
2321
+ out (list[Tensor]): Output tensors for this rank's owned blocks. Must
2322
+ have length equal to the number of blocks owned by this rank (i.e.
2323
+ the count of ``i`` where ``dst_ranks[i] == my_rank``). Each
2324
+ ``out[j]`` must be contiguous with the same dtype as ``input``.
2325
+ group (str): The name of the ``ProcessGroup`` to perform the operation on.
2326
+ dim (int): Dimension along which blocks are defined (0 or 1).
2327
+ offsets (list[int] | None): Inclusive prefix-sum of block sizes along
2328
+ ``dim``, length N. If not provided, ``input.size(dim)`` is divided
2329
+ into equal-size blocks based on the size of the ``group``.
2330
+ dst_ranks (list[int] | None): Destination rank for each block. If not
2331
+ provided, blocks are distributed round-robin across ranks.
2332
+ red_op (str): Reduction operation; currently only ``'sum'`` is supported.
2333
+
2334
+ Example::
2335
+
2336
+ >>> # doctest: +SKIP
2337
+ >>> # Each rank holds a Grouped GEMM gradient buffer in symmetric memory.
2338
+ >>> # The buffer has W experts laid out as equal column blocks; each expert
2339
+ >>> # is reduced to a specific rank (dst_ranks[i] == i % world_size).
2340
+ >>> buf = symm_mem.empty(H, W * C, dtype=torch.bfloat16, device="cuda")
2341
+ >>> symm_mem.rendezvous(buf, group=group_name)
2342
+ >>> offsets = [i * C for i in range(1, W + 1)] # inclusive prefix-sum
2343
+ >>> dst_ranks = [i % world_size for i in range(W)]
2344
+ >>> n_owned = sum(r == rank for r in dst_ranks)
2345
+ >>> out = [torch.empty(H, C, dtype=torch.bfloat16, device="cuda") for _ in range(n_owned)]
2346
+ >>> symm_mem.reduce_scatter_offset(buf, out, group_name, dim=1, offsets=offsets, dst_ranks=dst_ranks)
2347
+ """
2348
+ backend = get_backend(input.device)
2349
+ if backend == "NCCL":
2350
+ torch.ops.symm_mem.nccl_reduce_scatter_offset(
2351
+ input, out, group, dim, offsets, dst_ranks, red_op
2352
+ )
2353
+ else:
2354
+ raise NotImplementedError(
2355
+ f"reduce_scatter_offset: unsupported backend: {backend}"
2356
+ )
2357
+
2358
+
2359
+ def is_symm_mem_tensor(tensor: torch.Tensor) -> bool:
2360
+ r"""
2361
+ is_symm_mem_tensor(tensor) -> bool
2362
+
2363
+ Returns ``True`` if ``tensor`` was allocated via symmetric memory
2364
+ (i.e. via :func:`torch.distributed._symmetric_memory.empty` or
2365
+ :meth:`_SymmetricMemory.empty_strided_p2p`).
2366
+
2367
+ This is a non-collective, O(1) check.
2368
+
2369
+ Args:
2370
+ tensor (:class:`torch.Tensor`): the tensor to check.
2371
+ """
2372
+ return _SymmetricMemory.is_symm_mem_tensor(tensor)
2373
+
2374
+
2375
+ __all__ = [
2376
+ "empty",
2377
+ "is_symm_mem_tensor",
2378
+ "rendezvous",
2379
+ "is_nvshmem_available",
2380
+ "set_backend",
2381
+ "get_backend",
2382
+ "get",
2383
+ "set_signal_pad_size",
2384
+ "get_signal_pad_size",
2385
+ "get_mem_pool",
2386
+ "reduce_scatter_offset",
2387
+ ]
western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_nccl.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING
4
+ from typing_extensions import Self
5
+
6
+ import torch
7
+
8
+
9
+ if TYPE_CHECKING:
10
+ from torch.types import _device
11
+
12
+
13
+ __all__ = ["NcclCommRegistration", "register_external_nccl_comm"]
14
+
15
+
16
+ class NcclCommRegistration:
17
+ r"""
18
+ Handle returned by :func:`register_external_nccl_comm`.
19
+
20
+ Keeps an externally-owned NCCL communicator published in PyTorch's
21
+ symmetric memory registry for as long as this object is alive. Dropping it
22
+ (via ``del``, exiting its ``with`` block, or calling :meth:`unregister`)
23
+ removes the entry so a successor producer can register cleanly under the
24
+ same ``group_name``.
25
+
26
+ The comm's lifetime is *not* owned by this object; the registration only
27
+ publishes a borrowed pointer. Optionally a strong reference to the source
28
+ comm object is held so a stray ``del comm`` cannot dangle the registered
29
+ pointer for the lifetime of the registration.
30
+ """
31
+
32
+ def __init__(
33
+ self,
34
+ group_name: str,
35
+ device: torch.device,
36
+ comm: object | None = None,
37
+ ) -> None:
38
+ self._group_name = group_name
39
+ self._device = device
40
+ self._comm = comm
41
+ self._active = True
42
+
43
+ def unregister(self) -> None:
44
+ """Remove the registration. Idempotent."""
45
+ if not self._active:
46
+ return
47
+ # Imported lazily: only present in NCCL builds with symmetric-memory
48
+ # device support.
49
+ from torch._C._distributed_c10d import _unregister_external_nccl_comm
50
+
51
+ _unregister_external_nccl_comm(self._group_name, self._device)
52
+ self._active = False
53
+ self._comm = None
54
+
55
+ def __enter__(self) -> Self:
56
+ return self
57
+
58
+ def __exit__(self, *exc: object) -> None:
59
+ self.unregister()
60
+
61
+ def __del__(self) -> None:
62
+ try:
63
+ self.unregister()
64
+ except Exception:
65
+ # Best-effort cleanup; never raise from __del__ (e.g. during
66
+ # interpreter teardown the C extension may already be gone).
67
+ pass
68
+
69
+
70
+ def register_external_nccl_comm(
71
+ group_name: str,
72
+ comm_ptr: int,
73
+ device: _device,
74
+ comm: object | None = None,
75
+ ) -> NcclCommRegistration:
76
+ r"""
77
+ Publish an externally-owned ``ncclComm_t`` into PyTorch's symmetric memory
78
+ registry under ``group_name``.
79
+
80
+ This lets producers that are not a ``ProcessGroupNCCL`` (e.g. torchcomms
81
+ backends) supply the host NCCL communicator that
82
+ :func:`rendezvous` needs, without symmetric memory having to
83
+ ``dynamic_cast`` to a specific process-group implementation. After this
84
+ call, ``rendezvous(tensor, group=group_name)`` on a tensor in a process
85
+ group registered under ``group_name`` will find this comm.
86
+
87
+ Args:
88
+ group_name (str): the process-group name to register the comm under.
89
+ comm_ptr (int): the host ``ncclComm_t`` as an opaque integer pointer
90
+ (e.g. from ``torchcomms`` ``get_nccl_comm_ptr()``).
91
+ device (`torch.device` or str): the CUDA device the comm belongs to.
92
+ comm (object, optional): the source comm object. When provided, a
93
+ strong reference is held by the returned registration so the comm
94
+ cannot be garbage-collected while still registered.
95
+
96
+ Returns:
97
+ NcclCommRegistration: keep this alive for as long as symmetric memory
98
+ should use this comm; drop it (or call ``unregister()``) to remove the
99
+ registration.
100
+
101
+ .. note::
102
+ This is only available in NCCL builds with symmetric-memory device
103
+ support; otherwise it raises :class:`ImportError`.
104
+ """
105
+ from torch._C._distributed_c10d import (
106
+ _register_external_nccl_comm as _register_external_nccl_comm_impl,
107
+ )
108
+
109
+ device = torch.device(device)
110
+ _register_external_nccl_comm_impl(group_name, comm_ptr, device)
111
+ return NcclCommRegistration(group_name, device, comm)
western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_nvshmem_triton.py ADDED
@@ -0,0 +1,1220 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import os
3
+ import subprocess
4
+ import sysconfig
5
+ from typing import Any
6
+
7
+ import torch.distributed as dist
8
+ from torch.utils._triton import has_triton
9
+
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ class NvshmemLibFinder:
15
+ """
16
+ A class to find path to the NVSHMEM device library.
17
+
18
+ Environment variable:
19
+
20
+ `NVSHMEM_LIB_DIR` (Optional[str]): The directory where the NVSHMEM device
21
+ library is located. If not provided, it will use the default path where
22
+ NVSHMEM wheel is installed, or search for the library in common system
23
+ paths.
24
+ """
25
+
26
+ # Class variable to store the found library path for reuse
27
+ found_device_lib_path: str | None = None
28
+
29
+ @classmethod
30
+ def find_device_library(cls) -> str:
31
+ """
32
+ Find the path to the NVSHMEM device library.
33
+
34
+ Returns:
35
+ str: The path to libnvshmem_device.bc (included).
36
+ """
37
+ if cls.found_device_lib_path is not None:
38
+ # Return the cached path if it exists
39
+ return cls.found_device_lib_path
40
+
41
+ # First, check if the user has specified a custom library path
42
+ user_lib_dir = os.environ.get("NVSHMEM_LIB_DIR", None)
43
+ if user_lib_dir is not None:
44
+ lib_path = os.path.join(user_lib_dir, "libnvshmem_device.bc")
45
+ if not os.path.exists(lib_path):
46
+ raise RuntimeError(
47
+ f"NVSHMEM device library not found at specified path: {user_lib_dir}"
48
+ )
49
+ cls.found_device_lib_path = lib_path
50
+ return lib_path
51
+
52
+ # Otherwise, search for the library in the default installation paths
53
+ paths = [
54
+ os.path.join(sysconfig.get_path("purelib"), "nvidia", "nvshmem", "lib")
55
+ ]
56
+
57
+ # Add common system installation paths
58
+ common_paths = [
59
+ "/usr/local/lib",
60
+ "/usr/lib",
61
+ "/opt/nvidia/nvshmem/lib",
62
+ ]
63
+ paths.extend(common_paths)
64
+
65
+ try:
66
+ import torch
67
+
68
+ torch_lib = os.path.join(os.path.dirname(torch.__file__), "lib")
69
+ so_path = os.path.join(torch_lib, "libtorch_nvshmem.so")
70
+
71
+ if os.path.exists(so_path):
72
+ try:
73
+ result = subprocess.run(
74
+ ["readelf", "-d", so_path],
75
+ capture_output=True,
76
+ text=True,
77
+ check=True,
78
+ )
79
+
80
+ for line in result.stdout.splitlines():
81
+ if ("RPATH" in line or "RUNPATH" in line) and "[" in line:
82
+ rpath = line.split("[", 1)[1].split("]", 1)[0]
83
+ for p in rpath.split(":"):
84
+ p = p.strip().replace("$ORIGIN", torch_lib)
85
+ if p and p not in paths:
86
+ paths.append(p)
87
+ except subprocess.CalledProcessError:
88
+ pass
89
+
90
+ except ImportError:
91
+ pass
92
+
93
+ for path in paths:
94
+ device_lib = os.path.join(path, "libnvshmem_device.bc")
95
+ if os.path.exists(device_lib):
96
+ cls.found_device_lib_path = device_lib
97
+ return device_lib
98
+
99
+ raise RuntimeError(f"NVSHMEM device library not found. Searched: {paths}")
100
+
101
+
102
+ def enable_triton(lib_dir: str | None = None) -> dict[str, str]:
103
+ raise NotImplementedError(
104
+ "`enable_triton` is deprecated. "
105
+ "If you need NVSHMEM device function support for Triton, "
106
+ "please use `@requires_nvshmem` to decorate your Triton kernel. ",
107
+ )
108
+
109
+
110
+ class NvshmemKernelRegistry:
111
+ """
112
+ A class to register kernel functions that ** require NVSHMEM initialization **
113
+ """
114
+
115
+ # Class variable to store the functions to be initialized
116
+ _to_init: dict[str, Any] = {}
117
+
118
+ @classmethod
119
+ def register(cls, name: str) -> None:
120
+ """
121
+ Register a kernel function with the given name.
122
+
123
+ Args:
124
+ name (str): The name of the kernel function.
125
+ """
126
+ cls._to_init.setdefault(name)
127
+
128
+ @classmethod
129
+ def deregister(cls, name: str) -> None:
130
+ """
131
+ Deregister a kernel function with the given name.
132
+
133
+ Args:
134
+ name (str): The name of the kernel function.
135
+ """
136
+ cls._to_init.pop(name, None)
137
+
138
+ @classmethod
139
+ def has(cls, name: str) -> bool:
140
+ """
141
+ Check if a kernel function with the given name is registered.
142
+
143
+ Args:
144
+ name (str): The name of the kernel function.
145
+
146
+ Returns:
147
+ bool: True if the kernel function is registered, False otherwise.
148
+ """
149
+ return name in cls._to_init
150
+
151
+
152
+ def _nvshmem_init_hook(*args, **kwargs) -> None: # type: ignore[no-untyped-def]
153
+ """
154
+ A hook function to initialize the CUModule created by `triton.jit` with
155
+ NVSHMEM device context
156
+ """
157
+ from torch._C._distributed_c10d import _nvshmemx_cumodule_init
158
+
159
+ jit_function = kwargs["fn"].jit_function
160
+ fn_name = jit_function.fn.__name__
161
+
162
+ # Only initialize NVSHMEM module for kernels registered via @requires_nvshmem
163
+ if NvshmemKernelRegistry.has(fn_name):
164
+ key = kwargs["key"]
165
+ device = kwargs["compile"]["device"]
166
+ jit_function = kwargs["fn"].jit_function
167
+ kernel_cache = jit_function.device_caches[device][0]
168
+ kernel = kernel_cache.get(key, None)
169
+ if kernel is not None:
170
+ kernel.run
171
+ # Initialize NVSHMEM for the CU module
172
+ _nvshmemx_cumodule_init(kernel.module)
173
+ else:
174
+ logger.warning(
175
+ f"It seems Triton hasn't created a kernel for function {fn_name}. " # noqa: G004
176
+ "Please report this issue to Triton."
177
+ )
178
+
179
+
180
+ if has_triton():
181
+ from triton.runtime.jit import JITFunction, KernelInterface
182
+
183
+ # Create a new Callable class that follows the KernelInterface protocol so
184
+ # that the Callable works with the subscript operator, e.g. `foo[(1, 1)]`
185
+ class GridCallableWithExtern(KernelInterface):
186
+ """
187
+ `KernelInterface` invokes `self.run` in `__getitem__`, i.e. []. We
188
+ implement a `run` method by directing the call to `JITFunction.run`,
189
+ with added extern_libs kwarg, so that users don't have to pass it
190
+ """
191
+
192
+ def __init__(self, jit_func: JITFunction, extern_libs: dict[str, str]) -> None:
193
+ self.jit_func = jit_func
194
+ self.extern_libs = extern_libs
195
+
196
+ def run(self, *args, **kwargs): # type: ignore[no-untyped-def]
197
+ # Call the JITFunction.run with added extern_libs kwarg
198
+ return self.jit_func.run(*args, **kwargs, extern_libs=self.extern_libs)
199
+
200
+
201
+ def requires_nvshmem( # type: ignore[no-untyped-def]
202
+ jit_func, # JITFunction created by triton.jit
203
+ ):
204
+ """
205
+ A decorator to register a Triton kernel function that requires NVSHMEM initialization.
206
+
207
+ Example usage:
208
+ ```
209
+ @requires_nvshmem
210
+ @triton.jit
211
+ def foo(...):
212
+ ...
213
+ ```
214
+
215
+ If you would like to specify a path to the NVSHMEM device library other
216
+ than standard search locations, you can use the following environment
217
+ variable:
218
+ ```
219
+ export NVSHMEM_LIB_DIR=/path/to/nvshmem/lib
220
+ ```
221
+ """
222
+
223
+ import triton
224
+ from triton.runtime.jit import JITFunction
225
+
226
+ if not isinstance(jit_func, JITFunction):
227
+ raise TypeError(f"Expected a JITFunction, but got {type(jit_func)}")
228
+
229
+ # Find the NVSHMEM device library
230
+ lib_path = NvshmemLibFinder.find_device_library()
231
+ extern_libs = {"libnvshmem_device": lib_path}
232
+
233
+ # Register the JITFunction with the kernel registry as "to be initialized"
234
+ NvshmemKernelRegistry.register(jit_func.fn.__name__)
235
+
236
+ # Register the NVSHMEM init function as a post-compile hook.
237
+ # [Note] This is a global setting (due to lack of Triton API exposure). To
238
+ # avoid initializing Triton kernels that do not require NVSHMEM, filtering
239
+ # is performed in the hook function itself by checking against
240
+ # NvshmemKernelRegistry.
241
+ triton.knobs.runtime.jit_post_compile_hook = _nvshmem_init_hook
242
+
243
+ return GridCallableWithExtern(jit_func, extern_libs)
244
+
245
+
246
+ if has_triton():
247
+ import triton
248
+ import triton.language as tl
249
+ from triton.language import core
250
+
251
+ @triton.jit # type: ignore[misc]
252
+ def put(dest, source, nelems, pe): # type: ignore[no-untyped-def]
253
+ """
254
+ Put tensor data from local PE to a remote PE.
255
+
256
+ This high-level function provides a tensor-aware interface for NVSHMEM put
257
+ operations. It automatically handles type checking and size calculations, making
258
+ the API more ergonomic and type-safe.
259
+
260
+ Args:
261
+ dest: Destination tensor on the remote PE. Type must match source.
262
+ source: Source tensor on the local PE containing data to be copied.
263
+ nelems: Number of elements to transfer.
264
+ pe: PE number of the remote PE (0 ≤ pe < nvshmem_n_pes()).
265
+
266
+ Notes:
267
+ - Performs compile-time type checking between dest and source tensors.
268
+ - Automatically calculates byte size from tensor type and element count.
269
+ - This is a blocking operation that returns after data has been copied out
270
+ of the source array on the local PE.
271
+ - The operation does not guarantee delivery to the destination PE.
272
+ Use nvshmem_fence() for ordering or nvshmem_quiet() for completion.
273
+
274
+ Example:
275
+ ```
276
+ # Transfer 100 elements to PE 1
277
+ nvshmem.put(dest_tensor, src_tensor, 100, 1)
278
+ ```
279
+ """
280
+ tl.static_assert(dest.type == source.type)
281
+ nbytes = nelems * dest.type.element_ty.itemsize
282
+ return putmem_block_extern_wrapper(
283
+ dest.to(tl.int64), source.to(tl.int64), nbytes.to(tl.int64), pe
284
+ )
285
+
286
+ @core.extern
287
+ def putmem_block_extern_wrapper(dest, source, size_bytes, pe, _semantic=None): # type: ignore[no-untyped-def]
288
+ """Low-level extern wrapper for NVSHMEM put"""
289
+ return core.extern_elementwise(
290
+ "",
291
+ "",
292
+ [dest, source, size_bytes, pe],
293
+ {
294
+ (
295
+ core.dtype("int64"), # dest ptr
296
+ core.dtype("int64"), # source ptr
297
+ core.dtype("int64"), # size in bytes
298
+ core.dtype("int32"), # pe number
299
+ ): ("nvshmemx_putmem_block", core.dtype("int32"))
300
+ },
301
+ is_pure=False,
302
+ _semantic=_semantic,
303
+ )
304
+
305
+ @triton.jit # type: ignore[misc]
306
+ def get(dest, source, nelems, pe): # type: ignore[no-untyped-def]
307
+ """
308
+ Get tensor data from a remote PE to local PE.
309
+
310
+ This high-level function provides a tensor-aware interface for NVSHMEM get
311
+ operations. It automatically handles type checking and size calculations, making
312
+ the API more ergonomic and type-safe.
313
+
314
+ Args:
315
+ dest: Destination tensor on the local PE. Type must match source.
316
+ source: Source tensor on the remote PE containing data to be copied.
317
+ nelems: Number of elements to transfer.
318
+ pe: PE number of the remote PE (0 ≤ pe < nvshmem_n_pes()).
319
+
320
+ Notes:
321
+ - Performs compile-time type checking between dest and source tensors.
322
+ - Automatically calculates byte size from tensor type and element count.
323
+ - This is a blocking operation that returns after data has been delivered
324
+ to the destination array on the local PE.
325
+ - The destination data is guaranteed to be available for use after the call returns.
326
+
327
+ Example:
328
+ ```
329
+ # Get 100 elements from PE 0
330
+ nvshmem.get(dest_tensor, src_tensor, 100, 0)
331
+ ```
332
+ """
333
+ tl.static_assert(dest.type == source.type)
334
+ nbytes = nelems * dest.type.element_ty.itemsize
335
+ return getmem_block_extern_wrapper(
336
+ dest.to(tl.int64), source.to(tl.int64), nbytes.to(tl.int64), pe
337
+ )
338
+
339
+ @core.extern
340
+ def getmem_block_extern_wrapper(dest, source, size_bytes, pe, _semantic=None): # type: ignore[no-untyped-def]
341
+ """Low-level extern wrapper for NVSHMEM get"""
342
+ return core.extern_elementwise(
343
+ "",
344
+ "",
345
+ [dest, source, size_bytes, pe],
346
+ {
347
+ (
348
+ core.dtype("int64"), # dest ptr
349
+ core.dtype("int64"), # source ptr
350
+ core.dtype("int64"), # size in bytes
351
+ core.dtype("int32"), # pe number
352
+ ): ("nvshmemx_getmem_block", core.dtype("int32"))
353
+ },
354
+ is_pure=False,
355
+ _semantic=_semantic,
356
+ )
357
+
358
+ @triton.jit # type: ignore[misc]
359
+ def get_nbi(dest, source, nelems, pe): # type: ignore[no-untyped-def]
360
+ """
361
+ Get tensor data from a remote PE to local PE, non-blocking.
362
+
363
+ Different from the `get` function, this function returns after
364
+ initiating the operation. The operation is considered complete after a
365
+ subsequent call to `quiet`.
366
+
367
+ Args:
368
+ dest: Destination tensor on the local PE. Type must match source.
369
+ source: Source tensor on the remote PE containing data to be copied.
370
+ nelems: Number of elements to transfer.
371
+ pe: PE number of the remote PE (0 ≤ pe < nvshmem_n_pes()).
372
+
373
+ Notes:
374
+ - Performs compile-time type checking between dest and source tensors.
375
+ - Automatically calculates byte size from tensor type and element count.
376
+
377
+ Example:
378
+ ```
379
+ # Get 100 elements from PE 0
380
+ nvshmem.get_nbi(dest, src, 100, 0)
381
+ # Some independent computation which overlaps with the get operation
382
+ ...
383
+ # Wait for completion of the get operation
384
+ nvshmem.quiet()
385
+ ```
386
+ """
387
+ tl.static_assert(dest.type == source.type)
388
+ nbytes = nelems * dest.type.element_ty.itemsize
389
+ return getmem_block_extern_wrapper(
390
+ dest.to(tl.int64), source.to(tl.int64), nbytes.to(tl.int64), pe
391
+ )
392
+
393
+ @core.extern
394
+ def getmem_nbi_block_extern_wrapper(dest, source, size_bytes, pe, _semantic=None): # type: ignore[no-untyped-def]
395
+ """Low-level extern wrapper for NVSHMEM get"""
396
+ return core.extern_elementwise(
397
+ "",
398
+ "",
399
+ [dest, source, size_bytes, pe],
400
+ {
401
+ (
402
+ core.dtype("int64"), # dest ptr
403
+ core.dtype("int64"), # source ptr
404
+ core.dtype("int64"), # size in bytes
405
+ core.dtype("int32"), # pe number
406
+ ): ("nvshmemx_getmem_nbi_block", core.dtype("int32"))
407
+ },
408
+ is_pure=False,
409
+ _semantic=_semantic,
410
+ )
411
+
412
+ @triton.jit # type: ignore[misc]
413
+ def putmem_signal_block( # type: ignore[no-untyped-def]
414
+ dst,
415
+ src,
416
+ size_bytes,
417
+ signal,
418
+ sig_val,
419
+ sig_op,
420
+ pe,
421
+ ): # type: ignore[no-untyped-def]
422
+ """
423
+ Put data to remote PE with atomic signal operation using block-scoped operation.
424
+
425
+ This function copies data from the local PE to the remote PE and then
426
+ atomically updates a signal variable on the remote PE to indicate completion.
427
+ This enables efficient point-to-point synchronization between PEs.
428
+
429
+ Args:
430
+ dst (tensor): A tensor on calling PE symmetric to the destination tensor on remote PE.
431
+ src (tensor): Local tensor containing the source data.
432
+ size_bytes (int64): Number of bytes to transfer. Must be positive.
433
+ signal (tensor): Symmetric signal pad with remote PE.
434
+ Must be 8-byte aligned symmetric memory.
435
+ signal (int64): Value to be used in the signal operation.
436
+ sig_op (int32): Signal operation type. Common values:
437
+ - NVSHMEM_SIGNAL_SET (0): Atomic set operation
438
+ - NVSHMEM_SIGNAL_ADD (5): Atomic add operation
439
+ pe (int32): PE number of the remote PE (0 ≤ pe < nvshmem_n_pes()).
440
+
441
+ Returns:
442
+ int32: Status code (0 for success).
443
+
444
+ Notes:
445
+ - This is a blocking operation that returns after data has been copied out
446
+ of the source array and the signal has been updated on the remote PE.
447
+ - The signal update is performed atomically with respect to other signal
448
+ operations and synchronization routines.
449
+ - The signal variable must be of type uint64_t in symmetric memory.
450
+ - Use with nvshmem_signal_wait_until() for synchronization.
451
+
452
+ Example:
453
+ ```
454
+ # Transfer data and set completion flag to 1
455
+ NVSHMEM_SIGNAL_SET = 0
456
+ nvshmem.putmem_signal_block(
457
+ dst_ptr, src_ptr, 1024, sig_ptr, 1, NVSHMEM_SIGNAL_SET, target_pe
458
+ )
459
+ ```
460
+ """
461
+ # Ensure sig_val is 64 bits
462
+ sig_val = 0 << 32 | sig_val
463
+ return putmem_signal_block_extern_wrapper(
464
+ dst.to(tl.int64),
465
+ src.to(tl.int64),
466
+ size_bytes.to(tl.int64),
467
+ signal.to(tl.int64),
468
+ sig_val.to(tl.uint64),
469
+ sig_op,
470
+ pe,
471
+ )
472
+
473
+ @core.extern
474
+ def putmem_signal_block_extern_wrapper( # type: ignore[no-untyped-def]
475
+ dst,
476
+ src,
477
+ size_bytes,
478
+ signal,
479
+ sig_val,
480
+ sig_op,
481
+ pe,
482
+ _semantic=None,
483
+ ): # type: ignore[no-untyped-def]
484
+ return core.extern_elementwise(
485
+ "",
486
+ "",
487
+ [dst, src, size_bytes, signal, sig_val, sig_op, pe],
488
+ {
489
+ (
490
+ core.dtype("int64"),
491
+ core.dtype("int64"),
492
+ core.dtype("int64"),
493
+ core.dtype("int64"),
494
+ core.dtype("uint64"),
495
+ core.dtype("int32"),
496
+ core.dtype("int32"),
497
+ ): ("nvshmemx_putmem_signal_block", core.dtype("int32"))
498
+ },
499
+ is_pure=False,
500
+ _semantic=_semantic,
501
+ )
502
+
503
+ # Wait and Signal Operations
504
+
505
+ @triton.jit # type: ignore[misc]
506
+ def wait_until(ivar, cmp_op, cmp_val): # type: ignore[no-untyped-def]
507
+ """
508
+ Wait until a tensor variable meets a specified condition.
509
+
510
+ This high-level function provides a tensor-aware interface for NVSHMEM wait_until
511
+ operations. It automatically handles tensor address extraction, making
512
+ the API more ergonomic and type-safe.
513
+
514
+ Args:
515
+ ivar_tensor: Tensor to monitor (typically int64/uint64) in symmetric memory.
516
+ cmp: Comparison operator. Common values:
517
+ - NVSHMEM_CMP_EQ (0): Wait until ivar == cmp_val
518
+ - NVSHMEM_CMP_NE (1): Wait until ivar != cmp_val
519
+ - NVSHMEM_CMP_GT (2): Wait until ivar > cmp_val
520
+ - NVSHMEM_CMP_GE (3): Wait until ivar >= cmp_val
521
+ - NVSHMEM_CMP_LT (4): Wait until ivar < cmp_val
522
+ - NVSHMEM_CMP_LE (5): Wait until ivar <= cmp_val
523
+ cmp_val: Value to compare against.
524
+
525
+ Notes:
526
+ - This is a blocking operation that will wait indefinitely until the
527
+ condition is satisfied.
528
+ - The tensor must be in symmetric memory and accessible from other PEs.
529
+
530
+ Example:
531
+ ```
532
+ # Wait until flag tensor becomes 1 (set by another PE)
533
+ NVSHMEM_CMP_EQ = 0
534
+ nvshmem.wait_until_tensor(flag_tensor, NVSHMEM_CMP_EQ, 1)
535
+ ```
536
+ """
537
+ tl.static_assert(
538
+ ivar.type.element_ty.itemsize == 4,
539
+ "wait_until expects a 32-bit type for the synchronization variable",
540
+ )
541
+ return wait_until_extern_wrapper(ivar.to(tl.int64), cmp_op, cmp_val)
542
+
543
+ @core.extern
544
+ def wait_until_extern_wrapper(ivar, cmp, cmp_val, _semantic=None): # type: ignore[no-untyped-def]
545
+ return core.extern_elementwise(
546
+ "",
547
+ "",
548
+ [ivar, cmp, cmp_val],
549
+ {
550
+ (
551
+ core.dtype("int64"),
552
+ core.dtype("int32"),
553
+ core.dtype("int32"),
554
+ ): ("nvshmem_int_wait_until", core.dtype("int32"))
555
+ },
556
+ is_pure=False,
557
+ _semantic=_semantic,
558
+ )
559
+
560
+ @triton.jit # type: ignore[misc]
561
+ def signal_wait_until(signal, cmp, cmp_val): # type: ignore[no-untyped-def]
562
+ """
563
+ Wait until a signal variable meets a specified condition.
564
+
565
+ This function blocks the calling thread until the value at the specified
566
+ signal variable satisfies the given comparison condition. Signal variables
567
+ are special uint64_t symmetric objects used for efficient synchronization
568
+ with signal operations.
569
+
570
+ Args:
571
+ signal (tensor): Symmetric signal tensor with remote PE.
572
+ Must be 8-byte aligned symmetric memory.
573
+ cmp (int32): Comparison operator. Common values:
574
+ - NVSHMEM_CMP_EQ (0): Wait until signal == cmp_val
575
+ - NVSHMEM_CMP_NE (1): Wait until signal != cmp_val
576
+ - NVSHMEM_CMP_GT (2): Wait until signal > cmp_val
577
+ - NVSHMEM_CMP_GE (3): Wait until signal >= cmp_val
578
+ - NVSHMEM_CMP_LT (4): Wait until signal < cmp_val
579
+ - NVSHMEM_CMP_LE (5): Wait until signal <= cmp_val
580
+ cmp_val (int64): Value to compare against.
581
+
582
+ Returns:
583
+ int32: Status code (0 for success).
584
+
585
+ Notes:
586
+ - This is a blocking operation designed specifically for signal variables.
587
+ - Signal variables are updated atomically by putmem_signal operations.
588
+ - More efficient than wait_until for signal-based synchronization patterns.
589
+ - Ensures the signal update is fully complete before returning.
590
+ - Commonly used with putmem_signal_block for producer-consumer patterns.
591
+
592
+ Example:
593
+ ```
594
+ # Wait for signal to be set to completion value
595
+ NVSHMEM_CMP_EQ = 0
596
+ nvshmem.signal_wait_until(signal_ptr, NVSHMEM_CMP_EQ, 42)
597
+ ```
598
+ """
599
+ cmp_val = 0 << 32 | cmp_val
600
+ return signal_wait_until_extern_wrapper(
601
+ signal.to(tl.int64), cmp, cmp_val.to(tl.uint64)
602
+ )
603
+
604
+ @core.extern
605
+ def signal_wait_until_extern_wrapper(signal, cmp, cmp_val, _semantic=None): # type: ignore[no-untyped-def]
606
+ return core.extern_elementwise(
607
+ "",
608
+ "",
609
+ [signal, cmp, cmp_val],
610
+ {
611
+ (
612
+ core.dtype("int64"),
613
+ core.dtype("int32"),
614
+ core.dtype("uint64"),
615
+ ): ("nvshmem_signal_wait_until", core.dtype("int32"))
616
+ },
617
+ is_pure=False,
618
+ _semantic=_semantic,
619
+ )
620
+
621
+ @core.extern
622
+ def signal_op(sig_addr, signal, sig_op, pe, _semantic=None): # type: ignore[no-untyped-def]
623
+ """
624
+ Perform an atomic signal operation on a remote PE.
625
+
626
+ This function atomically updates a signal variable on the specified remote PE
627
+ using the given operation and value. This enables efficient point-to-point
628
+ synchronization and notification between PEs.
629
+
630
+ Args:
631
+ sig_addr (int64): Symmetric address of the signal variable (uint64_t) on the remote PE.
632
+ Must be 8-byte aligned symmetric memory.
633
+ signal (int64): Value to be used in the signal operation.
634
+ sig_op (int32): Signal operation type. Common values:
635
+ - NVSHMEM_SIGNAL_SET (0): Atomically set sig_addr = signal
636
+ - NVSHMEM_SIGNAL_ADD (5): Atomically set sig_addr += signal
637
+ pe (int32): PE number of the remote PE (0 ≤ pe < nvshmem_n_pes()).
638
+ _semantic: Optional semantic information for Triton compilation.
639
+
640
+ Returns:
641
+ int32: Status code (0 for success).
642
+
643
+ Notes:
644
+ - This is a one-sided operation - the remote PE does not need to participate.
645
+ - The signal operation is performed atomically on the remote PE.
646
+ - Can be used with signal_wait_until() on the remote PE for synchronization.
647
+ - Provides low-overhead notification mechanism between PEs.
648
+ - The signal variable must be of type uint64_t in symmetric memory.
649
+
650
+ Example:
651
+ ```python
652
+ # Atomically set remote signal to 1 to notify completion
653
+ NVSHMEM_SIGNAL_SET = 0
654
+ nvshmem.signal_op(remote_signal_ptr, 1, NVSHMEM_SIGNAL_SET, target_pe)
655
+ ```
656
+ """
657
+ return core.extern_elementwise(
658
+ "",
659
+ "",
660
+ [sig_addr, signal, sig_op, pe],
661
+ {
662
+ (
663
+ core.dtype("int64"),
664
+ core.dtype("int64"),
665
+ core.dtype("int32"),
666
+ core.dtype("int32"),
667
+ ): ("nvshmemx_signal_op", core.dtype("int32"))
668
+ },
669
+ is_pure=False,
670
+ _semantic=_semantic,
671
+ )
672
+
673
+ # Memory Ordering Operations
674
+ @core.extern
675
+ def fence(_semantic=None): # type: ignore[no-untyped-def]
676
+ """
677
+ Ensure ordering of put operations to each remote PE.
678
+
679
+ This function provides a memory fence that ensures point-to-point ordering
680
+ of remote memory operations. Put operations issued before the fence are
681
+ guaranteed to be ordered before put operations issued after the fence,
682
+ when targeting the same remote PE.
683
+
684
+ Args:
685
+ _semantic: Optional semantic information for Triton compilation.
686
+
687
+ Returns:
688
+ int32: Status code (0 for success).
689
+
690
+ Notes:
691
+ - This provides weaker ordering guarantees than quiet().
692
+ - Operations to each PE are ordered, but operations to different PEs
693
+ may still be reordered relative to each other.
694
+ - Does not guarantee completion of operations, only ordering.
695
+ - Non-blocking operations are not ordered by fence - use quiet() instead.
696
+ - Essential for ensuring correct ordering in communication patterns.
697
+
698
+ Memory Ordering Guarantees:
699
+ - Put operations before fence() → ordered before → Put operations after fence()
700
+ - Ordering is maintained per-destination-PE basis
701
+ - Remote PEs can observe the enforced ordering
702
+
703
+ Example:
704
+ ```
705
+ # Ensure first put completes before second put to same PE
706
+ nvshmem.put(dst, src, nelems, target_pe)
707
+ nvshmem.fence() # Enforce ordering
708
+ nvshmem.put(dst2, src2, nelems, target_pe)
709
+ ```
710
+ """
711
+ return core.extern_elementwise(
712
+ "",
713
+ "",
714
+ [],
715
+ {
716
+ (): ("nvshmem_fence", core.dtype("int32")),
717
+ },
718
+ is_pure=False,
719
+ _semantic=_semantic,
720
+ )
721
+
722
+ @core.extern
723
+ def quiet(_semantic=None): # type: ignore[no-untyped-def]
724
+ """
725
+ Wait for completion of all outstanding put operations.
726
+
727
+ This function blocks until all outstanding remote memory operations issued
728
+ by the calling PE have completed. It provides stronger guarantees than
729
+ fence() by ensuring both ordering and completion of all operations.
730
+
731
+ Args:
732
+ _semantic: Optional semantic information for Triton compilation.
733
+
734
+ Returns:
735
+ int32: Status code (0 for success).
736
+
737
+ Notes:
738
+ - This is a blocking operation that waits for completion.
739
+ - Ensures all previous put operations have been delivered to their destinations.
740
+ - Provides global ordering - operations to ALL PEs are ordered.
741
+ - Required to complete non-blocking operations.
742
+ - More expensive than fence() but provides stronger guarantees.
743
+
744
+ Memory Ordering Guarantees:
745
+ - All put operations before quiet() are completed before any operations after quiet()
746
+ - Operations are visible to all PEs as having occurred before subsequent operations
747
+ - Both blocking and non-blocking operations are completed
748
+
749
+ Example:
750
+ ```
751
+ # Ensure all data transfers complete before setting completion flag
752
+ nvshmem.putmem_block(data_ptr, src_ptr, data_size, target_pe)
753
+ nvshmem.quiet() # Wait for data transfer completion
754
+ nvshmem.putmem_block(
755
+ flag_ptr, flag_src_ptr, 8, target_pe
756
+ ) # Signal completion
757
+ ```
758
+ """
759
+ return core.extern_elementwise(
760
+ "",
761
+ "",
762
+ [],
763
+ {
764
+ (): ("nvshmem_quiet", core.dtype("int32")),
765
+ },
766
+ is_pure=False,
767
+ _semantic=_semantic,
768
+ )
769
+
770
+ # PE Information Operations
771
+ @core.extern
772
+ def my_pe(_semantic=None): # type: ignore[no-untyped-def]
773
+ """
774
+ Get the PE number of the calling PE.
775
+
776
+ This function returns the unique identifier (PE number) of the current
777
+ processing element within the NVSHMEM job. PE numbers range from 0 to
778
+ nvshmem_n_pes() - 1.
779
+
780
+ Args:
781
+ _semantic: Optional semantic information for Triton compilation.
782
+
783
+ Returns:
784
+ int32: PE number of the calling PE (0 ≤ pe < nvshmem_n_pes()).
785
+
786
+ Notes:
787
+ - This is a pure function that returns the same value throughout execution.
788
+ - PE numbering starts from 0 and is contiguous.
789
+ - Each PE has a unique identifier within the NVSHMEM job.
790
+ - Can be called from both host and device code.
791
+ - Essential for implementing PE-specific logic and communication patterns.
792
+
793
+ Example:
794
+ ```
795
+ # Get current PE number for conditional logic
796
+ pe = nvshmem.my_pe()
797
+ if pe == 0:
798
+ # Root PE logic
799
+ pass
800
+ else:
801
+ # Non-root PE logic
802
+ pass
803
+ ```
804
+ """
805
+ return core.extern_elementwise(
806
+ "",
807
+ "",
808
+ [],
809
+ {(): ("nvshmem_my_pe", core.dtype("int32"))},
810
+ is_pure=True,
811
+ _semantic=_semantic,
812
+ )
813
+
814
+ @core.extern
815
+ def n_pes(_semantic=None): # type: ignore[no-untyped-def]
816
+ """
817
+ Get the total number of PEs in the NVSHMEM job.
818
+
819
+ This function returns the total count of processing elements (PEs)
820
+ participating in the current NVSHMEM job. This value remains constant
821
+ throughout the execution of the program.
822
+
823
+ Args:
824
+ _semantic: Optional semantic information for Triton compilation.
825
+
826
+ Returns:
827
+ int32: Total number of PEs in the job (always ≥ 1).
828
+
829
+ Notes:
830
+ - This is a pure function that returns the same value throughout execution.
831
+ - The value is determined at NVSHMEM initialization and never changes.
832
+ - Valid PE numbers range from 0 to n_pes() - 1.
833
+ - Can be called from both host and device code.
834
+ - Essential for implementing collective operations and communication patterns.
835
+
836
+ Example:
837
+ ```
838
+ # Broadcast from root to all other PEs
839
+ total_pes = nvshmem.n_pes()
840
+ my_rank = nvshmem.my_pe()
841
+
842
+ if my_rank == 0:
843
+ # Send to all other PEs
844
+ for peer in range(1, total_pes):
845
+ nvshmem.putmem_block(dst_ptr, src_ptr, size, peer)
846
+ ```
847
+ """
848
+ return core.extern_elementwise(
849
+ "",
850
+ "",
851
+ [],
852
+ {(): ("nvshmem_n_pes", core.dtype("int32"))},
853
+ is_pure=True,
854
+ _semantic=_semantic,
855
+ )
856
+
857
+ # Synchronization Operations
858
+ @core.extern
859
+ def barrier_all(_semantic=None): # type: ignore[no-untyped-def]
860
+ """
861
+ Synchronize all PEs with completion guarantee.
862
+
863
+ This function creates a barrier across all PEs in the NVSHMEM job. It ensures
864
+ that all local and remote memory updates issued before the barrier by any PE
865
+ are completed before any PE exits the barrier. This provides both
866
+ synchronization and memory consistency.
867
+
868
+ Args:
869
+ _semantic: Optional semantic information for Triton compilation.
870
+
871
+ Returns:
872
+ int32: Status code (0 for success).
873
+
874
+ Notes:
875
+ - This is a collective operation - all PEs must participate.
876
+ - Stronger guarantee than sync_all() - ensures completion of remote operations.
877
+ - Blocks until all PEs reach the barrier AND all memory operations complete.
878
+ - Must be called from kernels launched with cooperative launch.
879
+ - Provides full memory consistency across all PEs.
880
+ - More expensive than sync_all() due to completion guarantees.
881
+
882
+ Memory Consistency Guarantees:
883
+ - All memory updates before barrier_all() are visible to all PEs
884
+ - All remote memory operations are completed before any PE continues
885
+ - Provides a global synchronization point with memory ordering
886
+
887
+ Example:
888
+ ```
889
+ # Ensure all PEs complete their work before proceeding
890
+ # All PEs execute this - it's a collective operation
891
+ nvshmem.barrier_all()
892
+ # At this point, all previous operations are complete on all PEs
893
+ ```
894
+ """
895
+ return core.extern_elementwise(
896
+ "",
897
+ "",
898
+ [],
899
+ {(): ("nvshmem_barrier_all", core.dtype("int32"))},
900
+ is_pure=False,
901
+ _semantic=_semantic,
902
+ )
903
+
904
+ @core.extern
905
+ def sync_all(_semantic=None): # type: ignore[no-untyped-def]
906
+ """
907
+ Synchronize all PEs with local completion guarantee.
908
+
909
+ This function creates a lightweight synchronization barrier across all PEs.
910
+ It ensures that all local store operations issued before the sync are
911
+ visible to other PEs, but does not guarantee completion of remote memory
912
+ operations initiated by the calling PE.
913
+
914
+ Args:
915
+ _semantic: Optional semantic information for Triton compilation.
916
+
917
+ Returns:
918
+ int32: Status code (0 for success).
919
+
920
+ Notes:
921
+ - This is a collective operation - all PEs must participate.
922
+ - Lighter weight than barrier_all() - only ensures local store visibility.
923
+ - Does not guarantee completion of remote memory updates initiated locally.
924
+ - Must be called from kernels launched with cooperative launch.
925
+ - Suitable when only synchronization (not completion) is needed.
926
+ - More efficient than barrier_all() for synchronization-only patterns.
927
+
928
+ Memory Consistency Guarantees:
929
+ - Local store operations are visible to other PEs
930
+ - Does NOT ensure completion of outgoing remote operations
931
+ - Provides synchronization point without full completion overhead
932
+
933
+ Example:
934
+ ```
935
+ # Lightweight synchronization between PEs
936
+ # All PEs execute this - it's a collective operation
937
+ nvshmem.sync_all()
938
+ # Local stores are visible, but remote ops may still be in flight
939
+ ```
940
+ """
941
+ return core.extern_elementwise(
942
+ "",
943
+ "",
944
+ [],
945
+ {(): ("nvshmem_sync_all", core.dtype("int32"))},
946
+ is_pure=False,
947
+ _semantic=_semantic,
948
+ )
949
+
950
+ # Collective Operations (mem-based APIs - sizes in bytes)
951
+ @triton.jit # type: ignore[misc]
952
+ def alltoall(team, dest, source, nelems_per_pe): # type: ignore[no-untyped-def]
953
+ """
954
+ All-to-all tensor exchange between PEs in a team.
955
+
956
+ This high-level function provides a tensor-aware interface for NVSHMEM alltoall
957
+ operations. Each PE sends nelems_per_pe elements to every other PE and receives
958
+ the same amount from every other PE.
959
+
960
+ Args:
961
+ team: Team handle for the collective operation. Use 0 for NVSHMEM_TEAM_WORLD.
962
+ dest: Destination tensor. Must be large enough for nelems_per_pe * n_pes elements.
963
+ source: Source tensor containing data for all PEs. Must contain nelems_per_pe * n_pes elements.
964
+ nelems_per_pe: Number of elements to exchange with each PE.
965
+
966
+ Notes:
967
+ - Performs compile-time type checking between dest and source tensors.
968
+ - Automatically calculates byte size from tensor type and element count.
969
+ - This is a collective operation - all PEs in the team must participate.
970
+ - Data layout: source=[data_for_pe0, data_for_pe1, ...], dest=[data_from_pe0, data_from_pe1, ...]
971
+
972
+ Example:
973
+ ```
974
+ # Each PE exchanges 10 elements with every other PE
975
+ nvshmem.alltoall(0, dest_tensor, src_tensor, 10)
976
+ ```
977
+ """
978
+ tl.static_assert(dest.type == source.type)
979
+ size_bytes_per_pe = nelems_per_pe * dest.type.element_ty.itemsize
980
+ return alltoallmem_block_extern_wrapper(
981
+ team, dest.to(tl.int64), source.to(tl.int64), size_bytes_per_pe.to(tl.int64)
982
+ )
983
+
984
+ @core.extern # type: ignore[misc]
985
+ def alltoallmem_block_extern_wrapper(
986
+ team: Any, dest: Any, source: Any, size_bytes: Any, _semantic: Any = None
987
+ ) -> None:
988
+ """Low-level extern wrapper for NVSHMEM alltoall"""
989
+ return core.extern_elementwise(
990
+ "",
991
+ "",
992
+ [team, dest, source, size_bytes],
993
+ {
994
+ (
995
+ core.dtype("int32"), # team handle
996
+ core.dtype("int64"), # dest ptr
997
+ core.dtype("int64"), # source ptr
998
+ core.dtype("int64"), # size in bytes
999
+ ): ("nvshmemx_alltoallmem_block", core.dtype("int32"))
1000
+ },
1001
+ is_pure=False,
1002
+ _semantic=_semantic,
1003
+ )
1004
+
1005
+ @triton.jit # type: ignore[misc]
1006
+ def broadcast(team, dest, source, nelems, pe_root): # type: ignore[no-untyped-def]
1007
+ """
1008
+ Broadcast tensor data from a root PE to all other PEs in a team.
1009
+
1010
+ This high-level function provides a tensor-aware interface for NVSHMEM broadcast
1011
+ operations. It automatically handles type checking and size calculations, making
1012
+ the API more ergonomic and type-safe.
1013
+
1014
+ Args:
1015
+ team: Team handle for the collective operation. Use 0 for NVSHMEM_TEAM_WORLD.
1016
+ dest: Destination tensor with type information. All PEs receive data here.
1017
+ source: Source tensor on the root PE. Type must match dest.
1018
+ nelems: Number of elements to broadcast.
1019
+ pe_root: PE number of the root PE that provides the source data.
1020
+
1021
+ Notes:
1022
+ - Performs compile-time type checking between dest and source tensors.
1023
+ - Automatically calculates byte size from tensor type and element count.
1024
+ - This is a collective operation - all PEs in the team must participate.
1025
+ - Must be called from kernels launched with cooperative launch.
1026
+
1027
+ Example:
1028
+ ```
1029
+ # Broadcast 100 elements from PE 0 to all PEs
1030
+ nvshmem.broadcast(0, dest_tensor, src_tensor, 100, 0)
1031
+ ```
1032
+ """
1033
+ tl.static_assert(dest.type == source.type)
1034
+ nbytes = nelems * dest.type.element_ty.itemsize
1035
+ return broadcastmem_block_extern_wrapper(
1036
+ team, dest.to(tl.int64), source.to(tl.int64), nbytes.to(tl.int64), pe_root
1037
+ )
1038
+
1039
+ @core.extern # type: ignore[misc]
1040
+ def broadcastmem_block_extern_wrapper(
1041
+ team: Any,
1042
+ dest: Any,
1043
+ source: Any,
1044
+ size_bytes: Any,
1045
+ pe_root: Any,
1046
+ _semantic: Any = None,
1047
+ ) -> None:
1048
+ """Low-level extern wrapper for NVSHMEM broadcast"""
1049
+ return core.extern_elementwise(
1050
+ "",
1051
+ "",
1052
+ [team, dest, source, size_bytes, pe_root],
1053
+ {
1054
+ (
1055
+ core.dtype("int32"), # team handle
1056
+ core.dtype("int64"), # dest ptr
1057
+ core.dtype("int64"), # source ptr
1058
+ core.dtype("int64"), # size in bytes
1059
+ core.dtype("int32"), # pe_root
1060
+ ): ("nvshmemx_broadcastmem_block", core.dtype("int32"))
1061
+ },
1062
+ is_pure=False,
1063
+ _semantic=_semantic,
1064
+ )
1065
+
1066
+ # Reduction Operation
1067
+ @triton.jit # type: ignore[misc]
1068
+ def reduce(team, dest, source, nreduce, operation: tl.constexpr): # type: ignore[no-untyped-def]
1069
+ """
1070
+ Performs a collective reduction on tensors across a team of PEs.
1071
+
1072
+ This high-level function provides a tensor-aware interface for NVSHMEM
1073
+ reduction operations. It automatically infers the data type from the
1074
+ input tensors and calls the appropriate underlying NVSHMEM function.
1075
+
1076
+ Args:
1077
+ team: The team handle for the collective (0 for NVSHMEM_TEAM_WORLD).
1078
+ dest: Destination tensor for the reduction results.
1079
+ source: Source tensor containing data to be reduced. Must be the same type as dest.
1080
+ nreduce: The number of elements in the source tensor to reduce.
1081
+ operation: The reduction operation to perform ("sum", "max", "min", "prod").
1082
+
1083
+ Notes:
1084
+ - Performs compile-time type checking between dest and source tensors.
1085
+ - This is a collective operation that must be called by all PEs in the team.
1086
+ - Requires a cooperative grid launch.
1087
+
1088
+ Example:
1089
+ ```
1090
+ # Perform a sum reduction on two tensors
1091
+ nvshmem.reduce(0, dest_tensor, src_tensor, 100, "sum")
1092
+ ```
1093
+ """
1094
+ tl.static_assert(dest.type == source.type)
1095
+ dtype = dest.type.element_ty
1096
+ return reduce_extern_wrapper(
1097
+ team,
1098
+ dest.to(tl.int64),
1099
+ source.to(tl.int64),
1100
+ nreduce.to(tl.int64),
1101
+ operation,
1102
+ dtype,
1103
+ )
1104
+
1105
+ @core.extern # type: ignore[misc]
1106
+ def reduce_extern_wrapper(
1107
+ team: Any,
1108
+ dest: Any,
1109
+ source: Any,
1110
+ nreduce: Any,
1111
+ operation: str,
1112
+ dtype: Any,
1113
+ _semantic: Any = None,
1114
+ ) -> None:
1115
+ """
1116
+ Low-level extern wrapper for NVSHMEM reduction operations.
1117
+
1118
+ This function provides a generic interface to NVSHMEM reduction operations,
1119
+ automatically selecting the appropriate NVSHMEM function based on the data type
1120
+ and operation specified.
1121
+ Args:
1122
+ team (int64): The team handle (0 for NVSHMEM_TEAM_WORLD).
1123
+ dest (pointer): Destination pointer where reduction results are stored.
1124
+ source (pointer): Source pointer containing data to be reduced.
1125
+ nreduce (int64): Number of elements to reduce.
1126
+ operation (str): Reduction operation ("sum", "max", "min", "prod").
1127
+ dtype: Data type specification - accepts torch.dtype, tl.dtype, str, or constexpr.
1128
+ _semantic: Optional semantic information for Triton compilation.
1129
+
1130
+ Raises:
1131
+ ValueError: If the operation is not supported.
1132
+ TypeError: If the data type is not supported.
1133
+
1134
+ Example:
1135
+ nvshmem.reduce(0, dest_ptr, src_ptr, 100, "sum", torch.float32)
1136
+ """
1137
+ # Mapping from Triton dtype names to NVSHMEM typenames
1138
+ DTYPE_TO_NVSHMEM_MAP = {
1139
+ "int8": "int8",
1140
+ "int16": "int16",
1141
+ "int32": "int32",
1142
+ "int64": "int64",
1143
+ "uint8": "uint8",
1144
+ "uint16": "uint16",
1145
+ "uint32": "uint32",
1146
+ "uint64": "uint64",
1147
+ "fp16": "half",
1148
+ "bf16": "bfloat16",
1149
+ "fp32": "float",
1150
+ "fp64": "double",
1151
+ }
1152
+
1153
+ # Triton dtype names are standardized as fp16, bf16, fp32, etc.
1154
+ dtype_name = str(dtype).replace("tl.", "")
1155
+
1156
+ if dtype_name not in DTYPE_TO_NVSHMEM_MAP:
1157
+ raise TypeError(
1158
+ f"Unsupported reduction dtype: {dtype_name}. Supported dtypes: {list(DTYPE_TO_NVSHMEM_MAP.keys())}"
1159
+ )
1160
+
1161
+ # Extract operation name from constexpr if needed
1162
+ op_name = operation.value if hasattr(operation, "value") else operation
1163
+
1164
+ # Validate operation is supported
1165
+ supported_ops = {"sum", "max", "min", "prod"}
1166
+ if op_name not in supported_ops:
1167
+ raise ValueError(
1168
+ f"Unsupported reduction operation: '{op_name}'. Supported ops are {supported_ops}"
1169
+ )
1170
+
1171
+ # Map to NVSHMEM typename and validate dtype is supported
1172
+ nvshmem_typename = DTYPE_TO_NVSHMEM_MAP.get(dtype_name)
1173
+ if nvshmem_typename is None:
1174
+ raise TypeError(
1175
+ f"Unsupported reduction dtype: {dtype_name}. Supported dtypes are {list(DTYPE_TO_NVSHMEM_MAP.keys())}"
1176
+ )
1177
+
1178
+ # Generate NVSHMEM function name
1179
+ nvshmem_func = f"nvshmem_{nvshmem_typename}_{op_name}_reduce"
1180
+
1181
+ # Define function signature - all parameters are int64 in Triton (they are just ptrs)
1182
+ signature = (
1183
+ core.dtype("int32"), # team handle
1184
+ core.dtype("int64"), # destination pointer
1185
+ core.dtype("int64"), # source pointer
1186
+ core.dtype("int64"), # number of elements
1187
+ )
1188
+
1189
+ return core.extern_elementwise(
1190
+ "",
1191
+ "",
1192
+ [team, dest, source, nreduce],
1193
+ {signature: (nvshmem_func, core.dtype("int32"))},
1194
+ is_pure=False,
1195
+ _semantic=_semantic,
1196
+ )
1197
+
1198
+ # Utility for inspecting Triton kernels
1199
+
1200
+ triton_kernels: dict = {}
1201
+
1202
+ def _log_triton_kernel(kernel) -> None: # type: ignore[no-untyped-def]
1203
+ import atexit
1204
+ import tempfile
1205
+
1206
+ if dist.is_initialized() and dist.get_rank() != 0:
1207
+ return
1208
+
1209
+ def on_exit() -> None:
1210
+ logger.info("PTX files:")
1211
+ for kernel in triton_kernels:
1212
+ with tempfile.NamedTemporaryFile(delete=False) as f:
1213
+ f.write(kernel.asm["ptx"].encode("utf-8"))
1214
+ logger.info(f"+- {kernel.name}: {f.name}") # noqa: G004
1215
+
1216
+ if len(triton_kernels) == 0:
1217
+ atexit.register(on_exit)
1218
+
1219
+ if kernel not in triton_kernels:
1220
+ triton_kernels[kernel] = None
western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_rocshmem_triton.py ADDED
@@ -0,0 +1,459 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ rocSHMEM Triton integration for AMD GPU (ROCm) builds.
3
+
4
+ This module provides the rocSHMEM-specific counterpart to the NVSHMEM Triton
5
+ integration in _nvshmem_triton.py. It is only imported on ROCm builds
6
+ (i.e. when ``torch.version.hip is not None``).
7
+ """
8
+
9
+ import logging
10
+ import os
11
+ import sysconfig
12
+ from typing import Any
13
+
14
+ import torch
15
+ from torch.distributed._symmetric_memory._shmem_triton_utils import (
16
+ run_shmem_init_hook,
17
+ ShmemKernelRegistry,
18
+ )
19
+ from torch.utils._triton import has_triton
20
+
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+
25
+ class RocshmemLibFinder:
26
+ """
27
+ Find the architecture-specific rocSHMEM device bitcode library.
28
+
29
+ Environment variable:
30
+ ``ROCSHMEM_LIB_DIR`` (Optional[str]): directory containing
31
+ ``librocshmem_device_{arch}.bc``. When not set, the standard
32
+ ROCm installation at ``/opt/rocm/lib`` is searched.
33
+
34
+ Example::
35
+ export ROCSHMEM_LIB_DIR=/opt/rocm/lib
36
+ """
37
+
38
+ found_device_lib_path: str | None = None
39
+
40
+ @classmethod
41
+ def find_device_library(cls) -> str:
42
+ if cls.found_device_lib_path is not None:
43
+ return cls.found_device_lib_path
44
+
45
+ if not torch.cuda.is_available():
46
+ raise RuntimeError(
47
+ "ROCm/CUDA not available — cannot detect GPU architecture"
48
+ )
49
+
50
+ props = torch.cuda.get_device_properties(0)
51
+ # gcnArchName returns e.g. "gfx942:sramecc+:xnack-"
52
+ arch = props.gcnArchName.split(":")[0]
53
+ logger.info("Detected GPU architecture: %s", arch)
54
+
55
+ lib_name = f"librocshmem_device_{arch}.bc"
56
+
57
+ search_paths = [
58
+ os.path.join(sysconfig.get_path("purelib"), "amd", "rocshmem", "lib"),
59
+ "/opt/rocm/lib",
60
+ "/usr/local/lib",
61
+ "/usr/lib",
62
+ ]
63
+
64
+ user_lib_dir = os.environ.get("ROCSHMEM_LIB_DIR")
65
+ if user_lib_dir is not None:
66
+ lib_path = os.path.join(user_lib_dir, lib_name)
67
+ if not os.path.exists(lib_path):
68
+ raise RuntimeError(
69
+ f"rocSHMEM device library not found at ROCSHMEM_LIB_DIR: {lib_path}"
70
+ )
71
+ logger.info("Found rocSHMEM device library: %s", lib_path)
72
+ cls.found_device_lib_path = lib_path
73
+ return lib_path
74
+
75
+ lib_path = None
76
+ for path in search_paths:
77
+ candidate = os.path.join(path, lib_name)
78
+ if os.path.exists(candidate):
79
+ lib_path = candidate
80
+ break
81
+
82
+ if lib_path is None:
83
+ raise RuntimeError(
84
+ f"rocSHMEM device library '{lib_name}' not found.\n"
85
+ f"Searched: {search_paths}\n"
86
+ "Set ROCSHMEM_LIB_DIR to the directory containing it."
87
+ )
88
+ logger.info("Found rocSHMEM device library: %s", lib_path)
89
+ cls.found_device_lib_path = lib_path
90
+ return lib_path
91
+
92
+
93
+ class RocshmemKernelRegistry(ShmemKernelRegistry):
94
+ """Track Triton kernels that need rocSHMEM HIP-module initialization."""
95
+
96
+ _to_init: dict[str, Any] = {}
97
+
98
+
99
+ def _rocshmem_init_hook(*args, **kwargs) -> None: # type: ignore[no-untyped-def]
100
+ """
101
+ Post-compile hook that initializes rocSHMEM device context in the
102
+ compiled HIP module using the existing c10d NVSHMEM entrypoint.
103
+ """
104
+ from torch._C._distributed_c10d import _nvshmemx_cumodule_init
105
+
106
+ run_shmem_init_hook(
107
+ kwargs=kwargs,
108
+ registry=RocshmemKernelRegistry,
109
+ module_init=_nvshmemx_cumodule_init,
110
+ logger=logger,
111
+ )
112
+
113
+
114
+ def requires_rocshmem( # type: ignore[no-untyped-def]
115
+ jit_func,
116
+ ):
117
+ """
118
+ Decorator to mark a Triton kernel as requiring rocSHMEM device APIs.
119
+
120
+ Finds the architecture-specific rocSHMEM bitcode library, registers
121
+ the kernel for post-compile HIP-module initialization, and wraps the
122
+ function so that ``extern_libs`` is injected automatically.
123
+
124
+ Example::
125
+
126
+ @requires_rocshmem
127
+ @triton.jit
128
+ def my_kernel(...):
129
+ pe = rocshmem_my_pe()
130
+ rocshmem_putmem_wg(dest, src, nbytes, target_pe)
131
+
132
+ Set ``ROCSHMEM_LIB_DIR`` to override the default library search path.
133
+ """
134
+ from torch.distributed._symmetric_memory._shmem_triton_utils import (
135
+ build_requires_shmem_decorator,
136
+ )
137
+
138
+ return build_requires_shmem_decorator(
139
+ jit_func=jit_func,
140
+ find_device_library=RocshmemLibFinder.find_device_library,
141
+ extern_libs_key="rocshmem",
142
+ registry=RocshmemKernelRegistry,
143
+ init_hook=_rocshmem_init_hook,
144
+ error_prefix="@requires_rocshmem",
145
+ )
146
+
147
+
148
+ if has_triton():
149
+ import triton
150
+ import triton.language as tl
151
+ from triton.language import core
152
+
153
+ # -----------------------------------------------------------------------
154
+ # rocSHMEM device API — Triton-callable device functions.
155
+ #
156
+ # All symbol names are the direct rocSHMEM device-bitcode names; no
157
+ # NVSHMEM → rocSHMEM translation layer is needed here.
158
+ #
159
+ # RMA (block/wg-scoped):
160
+ # nvshmemx_putmem_block → rocshmem_putmem_wg
161
+ # nvshmemx_getmem_block → rocshmem_getmem_wg
162
+ # nvshmemx_getmem_nbi_block → rocshmem_getmem_nbi_wg
163
+ # nvshmemx_putmem_signal_block → rocshmem_putmem_signal_wg
164
+ # Wait / signal:
165
+ # nvshmem_int_wait_until → rocshmem_int_wait_until
166
+ # nvshmem_signal_wait_until → rocshmem_uint64_wait_until
167
+ # Memory ordering, PE info, barriers: names are identical.
168
+ # -----------------------------------------------------------------------
169
+
170
+ @triton.jit
171
+ def put(dest, source, nelems, pe): # type: ignore[no-untyped-def]
172
+ """Put *nelems* elements from local *source* to *dest* on remote *pe*."""
173
+ tl.static_assert(dest.type == source.type)
174
+ nbytes = nelems * dest.type.element_ty.itemsize
175
+ return _putmem_wg(
176
+ dest.to(tl.int64), source.to(tl.int64), nbytes.to(tl.int64), pe
177
+ )
178
+
179
+ @core.extern
180
+ def _putmem_wg(dest, source, size_bytes, pe, _semantic=None): # type: ignore[no-untyped-def]
181
+ return core.extern_elementwise(
182
+ "",
183
+ "",
184
+ [dest, source, size_bytes, pe],
185
+ {
186
+ (
187
+ core.dtype("int64"),
188
+ core.dtype("int64"),
189
+ core.dtype("int64"),
190
+ core.dtype("int32"),
191
+ ): ("rocshmem_putmem_wg", core.dtype("int32"))
192
+ },
193
+ is_pure=False,
194
+ _semantic=_semantic,
195
+ )
196
+
197
+ @triton.jit
198
+ def get(dest, source, nelems, pe): # type: ignore[no-untyped-def]
199
+ """Get *nelems* elements from *source* on remote *pe* into local *dest* (blocking)."""
200
+ tl.static_assert(dest.type == source.type)
201
+ nbytes = nelems * dest.type.element_ty.itemsize
202
+ return _getmem_wg(
203
+ dest.to(tl.int64), source.to(tl.int64), nbytes.to(tl.int64), pe
204
+ )
205
+
206
+ @core.extern
207
+ def _getmem_wg(dest, source, size_bytes, pe, _semantic=None): # type: ignore[no-untyped-def]
208
+ return core.extern_elementwise(
209
+ "",
210
+ "",
211
+ [dest, source, size_bytes, pe],
212
+ {
213
+ (
214
+ core.dtype("int64"),
215
+ core.dtype("int64"),
216
+ core.dtype("int64"),
217
+ core.dtype("int32"),
218
+ ): ("rocshmem_getmem_wg", core.dtype("int32"))
219
+ },
220
+ is_pure=False,
221
+ _semantic=_semantic,
222
+ )
223
+
224
+ @triton.jit
225
+ def get_nbi(dest, source, nelems, pe): # type: ignore[no-untyped-def]
226
+ """Non-blocking get; call quiet() for completion."""
227
+ tl.static_assert(dest.type == source.type)
228
+ nbytes = nelems * dest.type.element_ty.itemsize
229
+ return _getmem_nbi_wg(
230
+ dest.to(tl.int64), source.to(tl.int64), nbytes.to(tl.int64), pe
231
+ )
232
+
233
+ @core.extern
234
+ def _getmem_nbi_wg(dest, source, size_bytes, pe, _semantic=None): # type: ignore[no-untyped-def]
235
+ return core.extern_elementwise(
236
+ "",
237
+ "",
238
+ [dest, source, size_bytes, pe],
239
+ {
240
+ (
241
+ core.dtype("int64"),
242
+ core.dtype("int64"),
243
+ core.dtype("int64"),
244
+ core.dtype("int32"),
245
+ ): ("rocshmem_getmem_nbi_wg", core.dtype("int32"))
246
+ },
247
+ is_pure=False,
248
+ _semantic=_semantic,
249
+ )
250
+
251
+ @triton.jit
252
+ def putmem_signal_block( # type: ignore[no-untyped-def]
253
+ dst,
254
+ src,
255
+ size_bytes,
256
+ signal,
257
+ sig_val,
258
+ sig_op,
259
+ pe,
260
+ ):
261
+ """Put data to remote PE and atomically update a signal variable."""
262
+ sig_val = 0 << 32 | sig_val
263
+ return _putmem_signal_wg(
264
+ dst.to(tl.int64),
265
+ src.to(tl.int64),
266
+ size_bytes.to(tl.int64),
267
+ signal.to(tl.int64),
268
+ sig_val.to(tl.uint64),
269
+ sig_op,
270
+ pe,
271
+ )
272
+
273
+ @core.extern
274
+ def _putmem_signal_wg( # type: ignore[no-untyped-def]
275
+ dst,
276
+ src,
277
+ size_bytes,
278
+ signal,
279
+ sig_val,
280
+ sig_op,
281
+ pe,
282
+ _semantic=None,
283
+ ):
284
+ return core.extern_elementwise(
285
+ "",
286
+ "",
287
+ [dst, src, size_bytes, signal, sig_val, sig_op, pe],
288
+ {
289
+ (
290
+ core.dtype("int64"),
291
+ core.dtype("int64"),
292
+ core.dtype("int64"),
293
+ core.dtype("int64"),
294
+ core.dtype("uint64"),
295
+ core.dtype("int32"),
296
+ core.dtype("int32"),
297
+ ): ("rocshmem_putmem_signal_wg", core.dtype("int32"))
298
+ },
299
+ is_pure=False,
300
+ _semantic=_semantic,
301
+ )
302
+
303
+ @triton.jit
304
+ def wait_until(ivar, cmp_op, cmp_val): # type: ignore[no-untyped-def]
305
+ """Block until *ivar* satisfies the comparison condition."""
306
+ tl.static_assert(
307
+ ivar.type.element_ty.itemsize == 4,
308
+ "wait_until expects a 32-bit type for the synchronization variable",
309
+ )
310
+ return _int_wait_until(ivar.to(tl.int64), cmp_op, cmp_val)
311
+
312
+ @core.extern
313
+ def _int_wait_until(ivar, cmp, cmp_val, _semantic=None): # type: ignore[no-untyped-def]
314
+ return core.extern_elementwise(
315
+ "",
316
+ "",
317
+ [ivar, cmp, cmp_val],
318
+ {
319
+ (core.dtype("int64"), core.dtype("int32"), core.dtype("int32")): (
320
+ "rocshmem_int_wait_until",
321
+ core.dtype("int32"),
322
+ )
323
+ },
324
+ is_pure=False,
325
+ _semantic=_semantic,
326
+ )
327
+
328
+ @triton.jit
329
+ def signal_wait_until(signal, cmp, cmp_val): # type: ignore[no-untyped-def]
330
+ """Block until a uint64 signal variable satisfies the comparison condition."""
331
+ cmp_val = 0 << 32 | cmp_val
332
+ return _uint64_wait_until(signal.to(tl.int64), cmp, cmp_val.to(tl.uint64))
333
+
334
+ @core.extern
335
+ def _uint64_wait_until(signal, cmp, cmp_val, _semantic=None): # type: ignore[no-untyped-def]
336
+ return core.extern_elementwise(
337
+ "",
338
+ "",
339
+ [signal, cmp, cmp_val],
340
+ {
341
+ (core.dtype("int64"), core.dtype("int32"), core.dtype("uint64")): (
342
+ "rocshmem_uint64_wait_until",
343
+ core.dtype("int32"),
344
+ )
345
+ },
346
+ is_pure=False,
347
+ _semantic=_semantic,
348
+ )
349
+
350
+ @triton.jit
351
+ def signal_op(sig_addr, signal, sig_op, pe): # type: ignore[no-untyped-def]
352
+ """Not available in current rocSHMEM device bitcode."""
353
+ tl.static_assert(
354
+ False,
355
+ "rocshmem has no device-bitcode equivalent for signal_op. "
356
+ "Use rocshmem_uint64_atomic_set or rocshmem_uint64_atomic_add instead.",
357
+ )
358
+
359
+ @core.extern
360
+ def fence(_semantic=None): # type: ignore[no-untyped-def]
361
+ """Ensure ordering of issued remote-memory operations to each target PE."""
362
+ return core.extern_elementwise(
363
+ "",
364
+ "",
365
+ [],
366
+ {(): ("rocshmem_fence", core.dtype("int32"))},
367
+ is_pure=False,
368
+ _semantic=_semantic,
369
+ )
370
+
371
+ @core.extern
372
+ def quiet(_semantic=None): # type: ignore[no-untyped-def]
373
+ """Wait for completion of all outstanding remote-memory operations."""
374
+ return core.extern_elementwise(
375
+ "",
376
+ "",
377
+ [],
378
+ {(): ("rocshmem_quiet", core.dtype("int32"))},
379
+ is_pure=False,
380
+ _semantic=_semantic,
381
+ )
382
+
383
+ @core.extern
384
+ def my_pe(_semantic=None): # type: ignore[no-untyped-def]
385
+ """Return the PE number of the calling PE."""
386
+ return core.extern_elementwise(
387
+ "",
388
+ "",
389
+ [],
390
+ {(): ("rocshmem_my_pe", core.dtype("int32"))},
391
+ is_pure=True,
392
+ _semantic=_semantic,
393
+ )
394
+
395
+ @core.extern
396
+ def n_pes(_semantic=None): # type: ignore[no-untyped-def]
397
+ """Return the total number of PEs."""
398
+ return core.extern_elementwise(
399
+ "",
400
+ "",
401
+ [],
402
+ {(): ("rocshmem_n_pes", core.dtype("int32"))},
403
+ is_pure=True,
404
+ _semantic=_semantic,
405
+ )
406
+
407
+ @core.extern
408
+ def barrier_all(_semantic=None): # type: ignore[no-untyped-def]
409
+ """Barrier across all PEs with completion guarantee (workgroup-scoped)."""
410
+ return core.extern_elementwise(
411
+ "",
412
+ "",
413
+ [],
414
+ {(): ("rocshmem_barrier_all_wg", core.dtype("int32"))},
415
+ is_pure=False,
416
+ _semantic=_semantic,
417
+ )
418
+
419
+ @core.extern
420
+ def sync_all(_semantic=None): # type: ignore[no-untyped-def]
421
+ """Lightweight synchronization barrier across all PEs (workgroup-scoped)."""
422
+ return core.extern_elementwise(
423
+ "",
424
+ "",
425
+ [],
426
+ {(): ("rocshmem_sync_all_wg", core.dtype("int32"))},
427
+ is_pure=False,
428
+ _semantic=_semantic,
429
+ )
430
+
431
+ # Collective stubs: rocSHMEM *_wg collectives are not exposed in current
432
+ # device bitcode, so Triton kernels cannot call them yet.
433
+
434
+ @triton.jit
435
+ def alltoall(team, dest, source, nelems_per_pe): # type: ignore[no-untyped-def]
436
+ """Not available: rocshmem_alltoallmem_wg is not in current device bitcode."""
437
+ tl.static_assert(
438
+ False,
439
+ "rocshmem_alltoallmem_wg is not available in current device bitcode. "
440
+ "Use host-side rocshmem_alltoallmem_on_stream instead.",
441
+ )
442
+
443
+ @triton.jit
444
+ def broadcast(team, dest, source, nelems, pe_root): # type: ignore[no-untyped-def]
445
+ """Not available: rocshmem_broadcastmem_wg is not in current device bitcode."""
446
+ tl.static_assert(
447
+ False,
448
+ "rocshmem_broadcastmem_wg is not available in current device bitcode. "
449
+ "Use host-side rocshmem_broadcastmem_on_stream instead.",
450
+ )
451
+
452
+ @triton.jit
453
+ def reduce(team, dest, source, nreduce, operation: tl.constexpr): # type: ignore[no-untyped-def]
454
+ """Not available: rocshmem team reduce wg ops are not in current device bitcode."""
455
+ tl.static_assert(
456
+ False,
457
+ "rocshmem team reduce is not available in current device bitcode. "
458
+ "Use host-side rocshmem reduce API instead.",
459
+ )
western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_shmem_triton.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ SHMEM Triton backend dispatch.
3
+
4
+ Thin abstraction layer that routes to the correct platform-specific module:
5
+ - ROCm - _rocshmem_triton
6
+ - CUDA - _nvshmem_triton
7
+
8
+
9
+ End users import from here to get backend-agnostic functionality:
10
+ - get_shmem_backend_module(): returns the active backend module
11
+ - requires_shmem(): backend-agnostic @requires_shmem decorator
12
+ """
13
+
14
+ import torch
15
+
16
+
17
+ def get_shmem_backend_module():
18
+ if torch.version.hip is not None:
19
+ from torch.distributed._symmetric_memory import _rocshmem_triton
20
+
21
+ return _rocshmem_triton
22
+ from torch.distributed._symmetric_memory import _nvshmem_triton
23
+
24
+ return _nvshmem_triton
25
+
26
+
27
+ def requires_shmem(jit_func): # type: ignore[no-untyped-def]
28
+ """
29
+ Backend-agnostic Triton decorator for SHMEM kernels.
30
+
31
+ Delegates to ``@requires_rocshmem`` on ROCm and ``@requires_nvshmem``
32
+ on CUDA.
33
+ """
34
+ backend = get_shmem_backend_module()
35
+ if torch.version.hip is not None:
36
+ return backend.requires_rocshmem(jit_func)
37
+ return backend.requires_nvshmem(jit_func)
western_ai/usa/pytorch/torch/distributed/_symmetric_memory/_shmem_triton_utils.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Shared utilities for SHMEM Triton backends (_nvshmem_triton, _rocshmem_triton).
3
+
4
+ This is a leaf module — it does NOT import any backend or dispatch module,
5
+ so backend modules can safely import from here without circular dependencies.
6
+ """
7
+
8
+ from collections.abc import Callable
9
+ from typing import Any
10
+
11
+ from torch.utils._triton import has_triton
12
+
13
+
14
+ class ShmemKernelRegistry:
15
+ """Base registry tracking Triton kernels that need SHMEM module init."""
16
+
17
+ _to_init: dict[str, Any] = {}
18
+
19
+ @classmethod
20
+ def register(cls, name: str) -> None:
21
+ cls._to_init.setdefault(name)
22
+
23
+ @classmethod
24
+ def deregister(cls, name: str) -> None:
25
+ cls._to_init.pop(name, None)
26
+
27
+ @classmethod
28
+ def has(cls, name: str) -> bool:
29
+ return name in cls._to_init
30
+
31
+
32
+ def run_shmem_init_hook(
33
+ *,
34
+ kwargs: dict[str, Any],
35
+ registry: type[ShmemKernelRegistry],
36
+ module_init: Callable[[Any], None],
37
+ logger: Any,
38
+ ) -> None:
39
+ """Common post-compile hook logic shared by NVSHMEM and rocSHMEM backends."""
40
+ jit_function = kwargs["fn"].jit_function
41
+ fn_name = jit_function.fn.__name__
42
+
43
+ if not registry.has(fn_name):
44
+ return
45
+
46
+ key = kwargs["key"]
47
+ device = kwargs["compile"]["device"]
48
+ kernel_cache = jit_function.device_caches[device][0]
49
+ kernel = kernel_cache.get(key, None)
50
+ if kernel is not None:
51
+ kernel.run
52
+ module_init(kernel.module)
53
+ else:
54
+ logger.warning(
55
+ "It seems Triton hasn't created a kernel for function %s. "
56
+ "Please report this issue to Triton.",
57
+ fn_name,
58
+ )
59
+
60
+
61
+ if has_triton():
62
+ from triton.runtime.jit import JITFunction, KernelInterface
63
+
64
+ class GridCallableWithExtern(KernelInterface):
65
+ """
66
+ KernelInterface invokes self.run in __getitem__, i.e. [].
67
+ We direct the call to JITFunction.run with extern_libs injected,
68
+ so that users don't have to pass it.
69
+ """
70
+
71
+ def __init__(self, jit_func: JITFunction, extern_libs: dict[str, str]) -> None:
72
+ self.jit_func = jit_func
73
+ self.extern_libs = extern_libs
74
+
75
+ def run(self, *args, **kwargs): # type: ignore[no-untyped-def]
76
+ return self.jit_func.run(*args, **kwargs, extern_libs=self.extern_libs)
77
+
78
+ def build_requires_shmem_decorator( # type: ignore[no-untyped-def]
79
+ *,
80
+ jit_func,
81
+ find_device_library: Callable[[], str],
82
+ extern_libs_key: str,
83
+ registry: type[ShmemKernelRegistry],
84
+ init_hook: Callable[..., None],
85
+ error_prefix: str,
86
+ ):
87
+ """Build the @requires_<backend> decorator for a Triton JIT function."""
88
+ import triton
89
+ from triton.runtime.jit import JITFunction
90
+
91
+ if not isinstance(jit_func, JITFunction):
92
+ raise TypeError(
93
+ f"{error_prefix} must be applied to a @triton.jit function, "
94
+ f"got {type(jit_func)}"
95
+ )
96
+
97
+ lib_path = find_device_library()
98
+ extern_libs = {extern_libs_key: lib_path}
99
+ registry.register(jit_func.fn.__name__)
100
+ triton.knobs.runtime.jit_post_compile_hook = init_hook
101
+ return GridCallableWithExtern(jit_func, extern_libs)
western_ai/usa/pytorch/torch/distributed/_tensor/__init__.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NOTICE: DTensor has moved to torch.distributed.tensor
3
+
4
+ This file is a shim to redirect to the new location, and
5
+ we keep the old import path starts with `_tensor` for
6
+ backward compatibility. We will remove this folder once
7
+ we resolve all the BC issues.
8
+ """
9
+
10
+ import sys
11
+ from importlib import import_module
12
+
13
+
14
+ submodules = [
15
+ # TODO: _shards_wrapper/_utils here mainly for checkpoint BC, remove them
16
+ "_shards_wrapper",
17
+ "_utils",
18
+ "experimental",
19
+ "device_mesh",
20
+ ]
21
+
22
+ # Redirect imports
23
+ for submodule in submodules:
24
+ full_module_name = f"torch.distributed.tensor.{submodule}"
25
+ sys.modules[f"torch.distributed._tensor.{submodule}"] = import_module(
26
+ full_module_name
27
+ )
28
+
29
+ from torch.distributed.tensor import (
30
+ DeviceMesh,
31
+ distribute_module,
32
+ distribute_tensor,
33
+ DTensor,
34
+ empty,
35
+ full,
36
+ init_device_mesh,
37
+ ones,
38
+ Partial,
39
+ Placement,
40
+ rand,
41
+ randn,
42
+ Replicate,
43
+ Shard,
44
+ zeros,
45
+ )
western_ai/usa/pytorch/torch/distributed/_tensor/api.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NOTE: torch.distributed._tensor has been moved to torch.distributed.tensor.
3
+ The imports here are purely for backward compatibility. We will remove these
4
+ imports in a few releases
5
+
6
+ TODO: throw warnings when this module imported
7
+ """
8
+
9
+ from torch.distributed.tensor._api import * # noqa: F403
western_ai/usa/pytorch/torch/distributed/_tensor/placement_types.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NOTE: torch.distributed._tensor has been moved to torch.distributed.tensor.
3
+ The imports here are purely for backward compatibility. We will remove these
4
+ imports in a few releases
5
+
6
+ TODO: throw warnings when this module imported
7
+ """
8
+
9
+ from torch.distributed.tensor._dtensor_spec import * # noqa: F403
10
+ from torch.distributed.tensor.placement_types import * # noqa: F403
western_ai/usa/pytorch/torch/distributed/_token_switch.py ADDED
@@ -0,0 +1,343 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from __future__ import annotations
3
+
4
+ import abc
5
+ import ctypes
6
+ import importlib.util
7
+ import os
8
+ from dataclasses import dataclass
9
+ from typing import Any, TYPE_CHECKING
10
+
11
+ import torch
12
+
13
+
14
+ if TYPE_CHECKING:
15
+ from torch.distributed.distributed_c10d import ProcessGroup
16
+
17
+
18
+ # Keeps preloaded shared libraries resident for the process lifetime.
19
+ _NCCL_EP_KEEPALIVE: list[ctypes.CDLL] = []
20
+
21
+
22
+ def _find_pkg_dir(name: str) -> str | None:
23
+ # Locate an installed package's directory without importing it. find_spec
24
+ # raises (rather than returning None) when a dotted name's parent namespace
25
+ # is absent, so guard that.
26
+ try:
27
+ spec = importlib.util.find_spec(name)
28
+ except ModuleNotFoundError:
29
+ return None
30
+ if spec is None or not spec.submodule_search_locations:
31
+ return None
32
+ return spec.submodule_search_locations[0]
33
+
34
+
35
+ def _prepare_nccl4py() -> None:
36
+ # Dynamic (USE_SYSTEM_NCCL=ON / wheel) build only: the extension NEEDED-links
37
+ # libnccl_ep.so, which the nccl4py wheel provides at runtime. Point nccl-ep's
38
+ # JIT at nccl4py's EP headers (NCCL_EP_HOME) and the nvidia.nccl wheel's NCCL
39
+ # headers (NCCL_HOME), and make libnccl_ep resolvable, before importing the
40
+ # extension (its libnccl dependency is already loaded by torch). The static
41
+ # build bakes all of this in and never reaches here.
42
+ nccl_pkg = _find_pkg_dir("nccl")
43
+ if nccl_pkg is None:
44
+ raise ImportError(
45
+ "TokenSwitchNCCL needs the 'nccl4py' package for this build's "
46
+ "libnccl_ep.so and runtime JIT headers. Install it with "
47
+ "`pip install nccl4py==0.3.1`."
48
+ )
49
+ ep_dir = os.path.join(nccl_pkg, "ep")
50
+ if "NCCL_EP_HOME" not in os.environ:
51
+ ep_headers = os.path.join(ep_dir, "include", "nccl_ep")
52
+ if not os.path.isdir(ep_headers):
53
+ raise ImportError(
54
+ f"nccl4py at {nccl_pkg} is missing the EP JIT headers expected "
55
+ f"at {ep_headers}; reinstall nccl4py or set NCCL_EP_HOME."
56
+ )
57
+ os.environ["NCCL_EP_HOME"] = ep_dir
58
+
59
+ # Point the JIT at NCCL headers. libnccl.so itself needs no preload here: a
60
+ # USE_SYSTEM_NCCL=ON torch (the only build that reaches this path) already
61
+ # dynamically loaded the system libnccl, so libnccl_ep.so's NEEDED
62
+ # libnccl.so.2 resolves to that already-loaded copy.
63
+ nvidia_nccl = _find_pkg_dir("nvidia.nccl")
64
+ if nvidia_nccl is not None and os.path.isdir(os.path.join(nvidia_nccl, "include")):
65
+ os.environ.setdefault("NCCL_HOME", nvidia_nccl)
66
+
67
+ # nccl4py ships libnccl_ep unversioned (libnccl_ep.so) while the extension's
68
+ # NEEDED entry is its SONAME libnccl_ep.so.0; load it by path to register
69
+ # that SONAME for the import below.
70
+ lib = os.path.join(ep_dir, "lib", "libnccl_ep.so")
71
+ if os.path.isfile(lib):
72
+ _NCCL_EP_KEEPALIVE.append(ctypes.CDLL(lib, mode=ctypes.RTLD_LOCAL))
73
+
74
+
75
+ def _import_nccl_ep() -> Any:
76
+ # The EP bindings live in the optional torch._nccl_ep extension (USE_NCCL_EP).
77
+ # A USE_SYSTEM_NCCL=OFF build links libnccl_ep statically and bakes its JIT
78
+ # header paths, so the extension imports directly -- self-contained, no
79
+ # nccl4py. A USE_SYSTEM_NCCL=ON build NEEDED-links libnccl_ep from the nccl4py
80
+ # wheel, so the first import fails until nccl4py is set up; fall back to that
81
+ # and retry.
82
+ try:
83
+ # pyrefly: ignore [missing-import] # built only with USE_NCCL_EP
84
+ import torch._nccl_ep as _ep
85
+
86
+ return _ep
87
+ except ImportError:
88
+ pass
89
+
90
+ _prepare_nccl4py()
91
+ try:
92
+ # pyrefly: ignore [missing-import] # built only with USE_NCCL_EP
93
+ import torch._nccl_ep as _ep
94
+ except ImportError as e:
95
+ raise ImportError(
96
+ "torch._nccl_ep is unavailable; this PyTorch was not built with "
97
+ "USE_NCCL_EP (or, for a USE_SYSTEM_NCCL=ON build, the nccl4py wheel "
98
+ "is missing)."
99
+ ) from e
100
+
101
+ return _ep
102
+
103
+
104
+ @dataclass(frozen=True, slots=True)
105
+ class Routing:
106
+ handle: object
107
+ topk_idx: torch.Tensor
108
+
109
+
110
+ class _DispatchAutograd(torch.autograd.Function):
111
+ @staticmethod
112
+ def forward(
113
+ ctx: Any,
114
+ ts: TokenSwitch,
115
+ routing: Routing,
116
+ tokens: torch.Tensor,
117
+ topk_weights: torch.Tensor,
118
+ max_recv_tokens: int,
119
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
120
+ _N, H = tokens.shape
121
+ K = topk_weights.shape[1]
122
+ out_tokens = tokens.new_zeros(max_recv_tokens, H)
123
+ out_topk_weights = topk_weights.new_zeros(max_recv_tokens, K)
124
+ out_topk_idx = routing.topk_idx.new_zeros(max_recv_tokens, K)
125
+ ts._dispatch(
126
+ routing, tokens, topk_weights, out_tokens, out_topk_weights, out_topk_idx
127
+ )
128
+ ctx.ts = ts
129
+ ctx.routing = routing
130
+ ctx.tokens_shape = tokens.shape
131
+ return out_tokens, out_topk_weights, out_topk_idx
132
+
133
+ @staticmethod
134
+ # pyrefly: ignore [bad-override]
135
+ def backward(
136
+ ctx: Any,
137
+ grad_out_tokens: torch.Tensor,
138
+ grad_out_topk_weights: torch.Tensor,
139
+ grad_out_topk_idx: torch.Tensor,
140
+ ) -> tuple[None, None, torch.Tensor, None, None]:
141
+ grad_tokens = grad_out_tokens.new_zeros(ctx.tokens_shape)
142
+ ctx.ts._combine(ctx.routing, grad_out_tokens.contiguous(), grad_tokens)
143
+ return None, None, grad_tokens, None, None
144
+
145
+
146
+ class _CombineAutograd(torch.autograd.Function):
147
+ @staticmethod
148
+ def forward(
149
+ ctx: Any,
150
+ ts: TokenSwitch,
151
+ routing: Routing,
152
+ expert_tokens: torch.Tensor,
153
+ ) -> torch.Tensor:
154
+ N = routing.topk_idx.shape[0]
155
+ H = expert_tokens.shape[1]
156
+ out_tokens = expert_tokens.new_zeros(N, H)
157
+ ts._combine(routing, expert_tokens, out_tokens)
158
+ ctx.ts = ts
159
+ ctx.routing = routing
160
+ ctx.expert_shape = expert_tokens.shape
161
+ ctx.expert_dtype = expert_tokens.dtype
162
+ ctx.top_k = routing.topk_idx.shape[1]
163
+ return out_tokens
164
+
165
+ @staticmethod
166
+ # pyrefly: ignore [bad-override]
167
+ def backward(
168
+ ctx: Any, grad_out_tokens: torch.Tensor
169
+ ) -> tuple[None, None, torch.Tensor]:
170
+ M, H = ctx.expert_shape
171
+ N = grad_out_tokens.shape[0]
172
+ K = ctx.top_k
173
+ dtype = ctx.expert_dtype
174
+ # ncclEpDispatch requires the output buffer sized to the group's
175
+ # max_recv_tokens_per_rank, regardless of what shape expert_tokens had
176
+ # in forward (often a slice like out_tokens[:M]). Allocate full-size,
177
+ # run dispatch, then slice to ctx.expert_shape so the returned grad
178
+ # matches the input that produced it.
179
+ max_recv = ctx.ts._max_recv_tokens_per_rank
180
+ grad_expert_full = grad_out_tokens.new_zeros(max_recv, H).to(dtype)
181
+ dummy_weights = grad_out_tokens.new_zeros(N, K, dtype=torch.float32)
182
+ dummy_out_weights = grad_out_tokens.new_zeros(max_recv, K, dtype=torch.float32)
183
+ dummy_out_idx = ctx.routing.topk_idx.new_zeros(max_recv, K)
184
+ ctx.ts._dispatch(
185
+ ctx.routing,
186
+ grad_out_tokens.to(dtype).contiguous(),
187
+ dummy_weights,
188
+ grad_expert_full,
189
+ dummy_out_weights,
190
+ dummy_out_idx,
191
+ )
192
+ return None, None, grad_expert_full[:M].contiguous()
193
+
194
+
195
+ class TokenSwitch(abc.ABC):
196
+ """Abstract token routing switch (e.g. expert-parallel dispatch / combine).
197
+
198
+ Typical usage: :meth:`create_routing`, then :meth:`dispatch` / :meth:`combine`.
199
+ """
200
+
201
+ @abc.abstractmethod
202
+ def create_routing(
203
+ self,
204
+ topk_idx: torch.Tensor,
205
+ per_expert_token_counts: torch.Tensor | None = None,
206
+ ) -> Routing:
207
+ """Create expert routing for the current phase (e.g. top-k indices).
208
+
209
+ ``per_expert_token_counts`` is optional 1D int32, length >= local experts:
210
+ output buffer for per-expert receive counts (NCCL EP ``RECV_EXPERT_COUNTER``).
211
+ """
212
+ raise NotImplementedError
213
+
214
+ @abc.abstractmethod
215
+ def _dispatch(
216
+ self,
217
+ routing: Routing,
218
+ tokens: torch.Tensor,
219
+ topk_weights: torch.Tensor,
220
+ out_tokens: torch.Tensor,
221
+ out_topk_weights: torch.Tensor,
222
+ out_topk_idx: torch.Tensor,
223
+ ) -> None:
224
+ raise NotImplementedError
225
+
226
+ @abc.abstractmethod
227
+ def _combine(
228
+ self,
229
+ routing: Routing,
230
+ expert_tokens: torch.Tensor,
231
+ out_tokens: torch.Tensor,
232
+ ) -> None:
233
+ raise NotImplementedError
234
+
235
+ def dispatch(
236
+ self,
237
+ routing: Routing,
238
+ tokens: torch.Tensor,
239
+ topk_weights: torch.Tensor,
240
+ max_recv_tokens: int | None = None,
241
+ *,
242
+ out: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
243
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
244
+ """Route tokens to experts.
245
+
246
+ With ``out=(out_tokens, out_topk_weights, out_topk_idx)``: writes to the provided
247
+ buffers and returns them; no autograd support.
248
+ Without ``out``: allocates output buffers and returns
249
+ ``(out_tokens, out_topk_weights, out_topk_idx)`` with autograd support.
250
+ ``max_recv_tokens`` is required when ``out`` is not provided.
251
+ ``topk_weights`` receives no gradient (routing metadata).
252
+ """
253
+ if out is not None:
254
+ self._dispatch(routing, tokens, topk_weights, *out)
255
+ return out
256
+ if max_recv_tokens is None:
257
+ raise ValueError("max_recv_tokens is required when out= is not provided")
258
+ return _DispatchAutograd.apply(
259
+ self, routing, tokens, topk_weights, max_recv_tokens
260
+ ) # type: ignore[return-value]
261
+
262
+ def combine(
263
+ self,
264
+ routing: Routing,
265
+ expert_tokens: torch.Tensor,
266
+ *,
267
+ out: torch.Tensor | None = None,
268
+ ) -> torch.Tensor:
269
+ """Gather expert outputs back to token order.
270
+
271
+ With ``out=out_tokens``: writes to the provided buffer and returns it;
272
+ no autograd support.
273
+ Without ``out``: allocates an output buffer and returns it with autograd support.
274
+ """
275
+ if out is not None:
276
+ self._combine(routing, expert_tokens, out)
277
+ return out
278
+ return _CombineAutograd.apply(self, routing, expert_tokens) # type: ignore[return-value]
279
+
280
+
281
+ class TokenSwitchNCCL(TokenSwitch):
282
+ """Token switch backed by NCCL EP (:func:`ncclEpCreateGroup` / dispatch / combine)."""
283
+
284
+ def __init__(
285
+ self,
286
+ process_group: ProcessGroup,
287
+ num_experts: int,
288
+ max_dispatch_tokens_per_rank: int,
289
+ max_recv_tokens_per_rank: int,
290
+ max_token_bytes: int,
291
+ ) -> None:
292
+ self._ep = _import_nccl_ep()
293
+ self._max_recv_tokens_per_rank = max_recv_tokens_per_rank
294
+ self._group = self._ep._NcclEpGroup.create(
295
+ process_group,
296
+ num_experts,
297
+ max_dispatch_tokens_per_rank,
298
+ max_recv_tokens_per_rank,
299
+ max_token_bytes,
300
+ )
301
+
302
+ def create_routing(
303
+ self,
304
+ topk_idx: torch.Tensor,
305
+ per_expert_token_counts: torch.Tensor | None = None,
306
+ ) -> Routing:
307
+ """Create expert routing for this phase; pass to :meth:`dispatch` / :meth:`combine`."""
308
+ handle = self._ep._NcclEpHandle.create(
309
+ self._group,
310
+ topk_idx,
311
+ per_expert_token_counts,
312
+ )
313
+ return Routing(handle=handle, topk_idx=topk_idx)
314
+
315
+ def _dispatch(
316
+ self,
317
+ routing: Routing,
318
+ tokens: torch.Tensor,
319
+ topk_weights: torch.Tensor,
320
+ out_tokens: torch.Tensor,
321
+ out_topk_weights: torch.Tensor,
322
+ out_topk_idx: torch.Tensor,
323
+ ) -> None:
324
+ self._ep._nccl_ep_dispatch(
325
+ routing.handle,
326
+ tokens,
327
+ topk_weights,
328
+ out_tokens,
329
+ out_topk_weights,
330
+ out_topk_idx,
331
+ )
332
+
333
+ def _combine(
334
+ self,
335
+ routing: Routing,
336
+ expert_tokens: torch.Tensor,
337
+ out_tokens: torch.Tensor,
338
+ ) -> None:
339
+ self._ep._nccl_ep_combine(
340
+ routing.handle,
341
+ expert_tokens,
342
+ out_tokens,
343
+ )
western_ai/usa/pytorch/torch/distributed/_tools/__init__.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .fsdp2_mem_tracker import FSDPMemTracker
2
+ from .mem_tracker import MemTracker
3
+ from .memory_tracker import MemoryTracker
4
+ from .mod_tracker import ModTracker
5
+ from .runtime_estimator import RuntimeEstimator
6
+ from .sac_estimator import (
7
+ MSPS,
8
+ SACEstimator,
9
+ SACGreedyOrderMeta,
10
+ SACStats,
11
+ SACTradeOffStats,
12
+ )
western_ai/usa/pytorch/torch/distributed/_tools/common_utils.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+
3
+ import torch
4
+ from torch._custom_class_base import CustomClassBase
5
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
6
+
7
+
8
+ def get_untyped_storages(t: torch.Tensor) -> set[torch.UntypedStorage]:
9
+ """
10
+ Recursively extracts untyped storages from a tensor or its subclasses.
11
+
12
+ Args:
13
+ t (torch.Tensor): The tensor to extract storages from.
14
+
15
+ Returns:
16
+ Set[torch.UntypedStorage]: A set of untyped storages.
17
+ """
18
+ unflattened_tensors = [t]
19
+ flattened_tensor_storages = set()
20
+ while len(unflattened_tensors) > 0:
21
+ obj = unflattened_tensors.pop()
22
+ if is_traceable_wrapper_subclass(obj):
23
+ attrs, _ = obj.__tensor_flatten__()
24
+ for attr in attrs:
25
+ match getattr(obj, attr):
26
+ case torch.Tensor() as v:
27
+ unflattened_tensors.append(v)
28
+ case CustomClassBase():
29
+ pass
30
+ case unexpected:
31
+ raise AssertionError(
32
+ f"expected Tensor or CustomClassBase, got {type(unexpected)}"
33
+ )
34
+ else:
35
+ if not hasattr(obj, "untyped_storage"):
36
+ warnings.warn(
37
+ f"Expected a tensor or a traceable wrapper-subclass of tensor, but got {type(obj)}",
38
+ category=UserWarning,
39
+ stacklevel=2,
40
+ )
41
+ else:
42
+ flattened_tensor_storages.add(obj.untyped_storage())
43
+ return flattened_tensor_storages
western_ai/usa/pytorch/torch/distributed/_tools/fake_collectives.py ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+
3
+ import torch
4
+ from torch._C._distributed_c10d import (
5
+ _resolve_process_group,
6
+ FakeWork,
7
+ ProcessGroup,
8
+ Work,
9
+ )
10
+ from torch.utils._pytree import tree_map_only
11
+
12
+
13
+ c10d = torch.ops.c10d
14
+ _c10d_functional = torch.ops._c10d_functional
15
+ _c10d_functional_autograd = torch.ops._c10d_functional_autograd
16
+ _dtensor = torch.ops._dtensor
17
+
18
+ # List of collective operation functions including functional collectives
19
+ # Note: The following collectives might be deprecated soon hence not adding them
20
+ # depcreated_non_functional_collectives = [
21
+ # c10d.allreduce_coalesced_.default,
22
+ # c10d.reduce_scatter_tensor_coalesced_.default,
23
+ # c10d.allgather_into_tensor_coalesced_.default,
24
+ # c10d.allgather_coalesced_.default,
25
+ # ]
26
+ non_functional_collectives: set[torch._ops.OpOverload] = {
27
+ c10d.broadcast_.default,
28
+ c10d.allreduce_.default,
29
+ c10d.reduce_.default,
30
+ c10d.send.default,
31
+ c10d.recv_.default,
32
+ c10d.recv_any_source_.default,
33
+ c10d.allgather_.default,
34
+ c10d.reduce_scatter_.default,
35
+ c10d._reduce_scatter_base_.default,
36
+ c10d._allgather_base_.default,
37
+ c10d.gather_.default,
38
+ c10d.scatter_.default,
39
+ c10d.alltoall_.default,
40
+ c10d.alltoall_base_.default,
41
+ c10d.barrier.default,
42
+ c10d.monitored_barrier_.default,
43
+ }
44
+ functional_collectives: set[torch._ops.OpOverload] = {
45
+ _c10d_functional.broadcast.default,
46
+ _c10d_functional.all_reduce.default,
47
+ _c10d_functional.all_gather_into_tensor.default,
48
+ _c10d_functional.reduce_scatter_tensor.default,
49
+ _c10d_functional.reduce_scatter_tensor_out.default,
50
+ _c10d_functional.all_to_all_single.default,
51
+ _c10d_functional_autograd.all_to_all_single.default,
52
+ _c10d_functional.wait_tensor.default,
53
+ _c10d_functional.all_reduce_.default,
54
+ _c10d_functional.all_reduce_coalesced.default,
55
+ _c10d_functional.all_reduce_coalesced_.default,
56
+ _c10d_functional.all_gather_into_tensor_out.default,
57
+ _c10d_functional.all_gather_into_tensor_coalesced.default,
58
+ _c10d_functional_autograd.all_gather_into_tensor.default,
59
+ _c10d_functional.reduce_scatter_tensor_coalesced.default,
60
+ _c10d_functional_autograd.reduce_scatter_tensor.default,
61
+ _c10d_functional.broadcast_.default,
62
+ _c10d_functional.isend.default,
63
+ _c10d_functional.irecv.default,
64
+ _c10d_functional.batch_p2p_ops.default,
65
+ _dtensor.shard_dim_alltoall.default,
66
+ }
67
+
68
+ sync_ops: set[torch._ops.OpOverload] = {
69
+ c10d.barrier.default,
70
+ c10d.monitored_barrier_.default,
71
+ _c10d_functional.wait_tensor.default,
72
+ }
73
+
74
+ collective_ops = set.union(functional_collectives, non_functional_collectives)
75
+
76
+
77
+ class CollectiveOp:
78
+ # Static sets for performance optimization
79
+ PG_ARG_1 = {
80
+ c10d.broadcast_.default,
81
+ c10d.allreduce_.default,
82
+ c10d.reduce_.default,
83
+ c10d.send.default,
84
+ c10d.recv_.default,
85
+ c10d.recv_any_source_.default,
86
+ c10d.barrier.default,
87
+ # c10d.allreduce_coalesced_.default
88
+ }
89
+
90
+ PG_ARG_2 = {
91
+ c10d.allgather_.default,
92
+ c10d._allgather_base_.default,
93
+ c10d.reduce_scatter_.default,
94
+ c10d._reduce_scatter_base_.default,
95
+ c10d.gather_.default,
96
+ c10d.scatter_.default,
97
+ c10d.alltoall_.default,
98
+ c10d.alltoall_base_.default,
99
+ # c10d.allgather_coalesced_.default,
100
+ # c10d.allgather_into_tensor_coalesced_.default
101
+ # c10d.reduce_scatter_tensor_coalesced_.default
102
+ }
103
+
104
+ PG_ARG_3 = {
105
+ _c10d_functional.broadcast.default,
106
+ _c10d_functional.broadcast_.default,
107
+ _c10d_functional.all_reduce.default,
108
+ _c10d_functional.all_reduce_.default,
109
+ _c10d_functional.all_reduce_coalesced.default,
110
+ _c10d_functional.all_reduce_coalesced_.default,
111
+ _c10d_functional.all_gather_into_tensor.default,
112
+ _c10d_functional.all_gather_into_tensor_out.default,
113
+ _c10d_functional_autograd.all_gather_into_tensor.default,
114
+ _c10d_functional.all_gather_into_tensor_coalesced.default,
115
+ }
116
+
117
+ PG_ARG_4 = {
118
+ _c10d_functional.reduce_scatter_tensor.default,
119
+ _c10d_functional.reduce_scatter_tensor_coalesced.default,
120
+ _c10d_functional_autograd.reduce_scatter_tensor.default,
121
+ _c10d_functional.all_to_all_single.default,
122
+ _c10d_functional_autograd.all_to_all_single.default,
123
+ _c10d_functional.isend.default,
124
+ _c10d_functional.irecv.default,
125
+ _dtensor.shard_dim_alltoall.default,
126
+ }
127
+
128
+ PG_ARG_5 = {
129
+ _c10d_functional.batch_p2p_ops.default,
130
+ }
131
+
132
+ WK_ARG_1 = {
133
+ c10d.broadcast_.default,
134
+ c10d.allreduce_.default,
135
+ c10d.allgather_.default,
136
+ c10d.reduce_scatter_.default,
137
+ c10d._reduce_scatter_base_.default,
138
+ c10d._allgather_base_.default,
139
+ c10d.scatter_.default,
140
+ c10d.alltoall_.default,
141
+ _c10d_functional.isend.default,
142
+ _c10d_functional.irecv.default,
143
+ }
144
+
145
+ WK = {
146
+ c10d.send.default,
147
+ c10d.recv_.default,
148
+ c10d.recv_any_source_.default,
149
+ c10d.reduce_.default,
150
+ c10d.gather_.default,
151
+ c10d.alltoall_base_.default,
152
+ c10d.barrier.default,
153
+ }
154
+
155
+ COMM_TENSOR_ARG_0 = {
156
+ c10d.allreduce_.default,
157
+ c10d.send.default,
158
+ c10d.recv_.default,
159
+ c10d.recv_any_source_.default,
160
+ c10d.allgather_.default,
161
+ c10d.gather_.default,
162
+ c10d.reduce_.default,
163
+ c10d.broadcast_.default,
164
+ _c10d_functional.all_reduce_coalesced.default,
165
+ _c10d_functional.all_reduce_coalesced_.default,
166
+ # c10d.allreduce_coalesced_.default
167
+ # c10d.allgather_coalesced_.default
168
+ # c10d.allgather_into_tensor_coalesced_.default,
169
+ }
170
+
171
+ COMM_TENSOR_ARG_1 = {
172
+ c10d.reduce_scatter_.default,
173
+ c10d.scatter_.default,
174
+ # c10d.reduce_scatter_tensor_coalesced_.default,
175
+ }
176
+
177
+ COMM_TENSOR_ARG_RES = {
178
+ _c10d_functional.all_gather_into_tensor.default,
179
+ _c10d_functional_autograd.all_gather_into_tensor.default,
180
+ }
181
+
182
+ COMM_TENSOR_SINGLE_UNTYPED_STORAGE = {
183
+ c10d._allgather_base_.default,
184
+ _c10d_functional.broadcast.default,
185
+ _c10d_functional.broadcast_.default,
186
+ _c10d_functional.all_reduce.default,
187
+ _c10d_functional.all_reduce_.default,
188
+ _c10d_functional.reduce_scatter_tensor.default,
189
+ _c10d_functional_autograd.reduce_scatter_tensor.default,
190
+ }
191
+
192
+ COMM_TENSOR_ARG_0_AND_RES = {
193
+ _c10d_functional.all_to_all_single.default,
194
+ _c10d_functional_autograd.all_to_all_single.default,
195
+ _dtensor.shard_dim_alltoall.default,
196
+ }
197
+
198
+ COMM_TENSOR_RES_SUM = {
199
+ _c10d_functional.all_gather_into_tensor_coalesced.default,
200
+ _c10d_functional.reduce_scatter_tensor_coalesced.default,
201
+ }
202
+
203
+ @staticmethod
204
+ def sum_tensors(arg: Any) -> int:
205
+ """Calculate total memory consumed by the tensors in the argument."""
206
+ total_memory = 0
207
+
208
+ def sum_bytes(t: torch.Tensor) -> None:
209
+ nonlocal total_memory
210
+ total_memory += t.untyped_storage().nbytes()
211
+
212
+ tree_map_only(torch.Tensor, sum_bytes, arg)
213
+ return total_memory
214
+
215
+ @staticmethod
216
+ def get_process_group(func, args) -> ProcessGroup: # type: ignore[no-untyped-def]
217
+ """Retrieve the process group for collective operations, except `wait_tensor`."""
218
+ if func in CollectiveOp.PG_ARG_1:
219
+ return ProcessGroup.unbox(args[1])
220
+ if func in CollectiveOp.PG_ARG_2:
221
+ return ProcessGroup.unbox(args[2])
222
+ if func in CollectiveOp.PG_ARG_3:
223
+ return _resolve_process_group(args[2])
224
+ if func in CollectiveOp.PG_ARG_4:
225
+ return _resolve_process_group(args[3])
226
+ if func in CollectiveOp.PG_ARG_5:
227
+ return _resolve_process_group(args[4])
228
+ raise TypeError(f"Func {func} not found in {collective_ops}")
229
+
230
+ @staticmethod
231
+ def get_comm_tensor_size(func, res, args, kwargs) -> int: # type: ignore[no-untyped-def]
232
+ """Compute the communication tensor size, except for `wait_tensor`, `barrier`, and `monitored_barrier`."""
233
+ if func in CollectiveOp.COMM_TENSOR_ARG_0:
234
+ return CollectiveOp.sum_tensors(args[0])
235
+ if func in CollectiveOp.COMM_TENSOR_ARG_1:
236
+ return CollectiveOp.sum_tensors(args[1])
237
+ if func in CollectiveOp.COMM_TENSOR_ARG_RES:
238
+ return res.untyped_storage().nbytes()
239
+ if func in CollectiveOp.COMM_TENSOR_SINGLE_UNTYPED_STORAGE:
240
+ return args[0].untyped_storage().nbytes()
241
+ if func is c10d._reduce_scatter_base_.default:
242
+ return args[1].untyped_storage().nbytes()
243
+ if func is c10d.alltoall_.default:
244
+ # TODO(@sanketpurandare) - Confirm size computation
245
+ return max(
246
+ CollectiveOp.sum_tensors(args[0]), CollectiveOp.sum_tensors(args[1])
247
+ )
248
+ if func is c10d.alltoall_base_.default:
249
+ # TODO(@sanketpurandare) - Confirm size computation
250
+ return max(
251
+ args[0].untyped_storage().nbytes(), args[1].untyped_storage().nbytes()
252
+ )
253
+ if func == _c10d_functional.all_gather_into_tensor_out.default:
254
+ return args[-1].untyped_storage().nbytes()
255
+ if func in CollectiveOp.COMM_TENSOR_RES_SUM:
256
+ return CollectiveOp.sum_tensors(res)
257
+ if func in CollectiveOp.COMM_TENSOR_ARG_0_AND_RES:
258
+ # TODO(@sanketpurandare) - Confirm size computation
259
+ return args[0].untyped_storage().nbytes() + res.untyped_storage().nbytes()
260
+ if func is _c10d_functional.batch_p2p_ops.default:
261
+ return CollectiveOp.sum_tensors(args[3])
262
+ raise TypeError(f"Unknown function: {func} in {collective_ops}")
263
+
264
+ @staticmethod
265
+ def get_work(func, res) -> Work: # type: ignore[no-untyped-def]
266
+ if func in CollectiveOp.WK:
267
+ return FakeWork.unbox(res)
268
+ elif func in CollectiveOp.WK_ARG_1:
269
+ return FakeWork.unbox(res[1])
270
+ raise TypeError(f"Func {func} not found in {collective_ops}")
western_ai/usa/pytorch/torch/distributed/_tools/fsdp2_mem_tracker.py ADDED
@@ -0,0 +1,595 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections.abc import Callable
2
+ from copy import deepcopy
3
+ from enum import auto, Enum
4
+ from functools import partial, wraps
5
+ from typing import Any, NamedTuple, TYPE_CHECKING, TypeVar
6
+ from typing_extensions import ParamSpec, TypeVarTuple, Unpack
7
+
8
+ import torch
9
+ import torch.distributed._tools.fake_collectives
10
+ from torch import nn, optim
11
+ from torch._guards import active_fake_mode
12
+ from torch.distributed._tools.mem_tracker import _RefType, _State, MemTracker
13
+ from torch.distributed.fsdp import FSDPModule
14
+ from torch.distributed.fsdp._fully_shard._fsdp_param_group import FSDPParamGroup
15
+ from torch.distributed.tensor import DTensor
16
+ from torch.utils._python_dispatch import TorchDispatchMode
17
+ from torch.utils._pytree import tree_map_only
18
+ from torch.utils.weak import WeakIdKeyDictionary, weakref
19
+
20
+
21
+ if TYPE_CHECKING:
22
+ from torch.utils.hooks import RemovableHandle
23
+
24
+ _TOTAL_KEY = "Total"
25
+
26
+ __all__ = ["FSDPMemTracker"]
27
+
28
+ _P = ParamSpec("_P")
29
+ _R = TypeVar("_R")
30
+ _Ts = TypeVarTuple("_Ts")
31
+
32
+ c10d = torch.ops.c10d
33
+
34
+
35
+ class _FSDPRefType(_RefType):
36
+ """
37
+ Enumerates categories of memory usage in FSDP modules, including parameters, gradients, activations,
38
+ and optimizer states.
39
+
40
+ Attributes:
41
+ SHARDED_PARAM (str): Memory usage of sharded parameters.
42
+ UNSHARDED_PARAM (str): Memory usage of unsharded parameters.
43
+ SHARDED_GRAD (str): Memory usage of sharded gradients corresponding to the sharded parameters.
44
+ UNSHARDED_GRAD (str): Memory usage of unsharded gradients corresponding to the unsharded parameters.
45
+ ACT (str): Memory usage of activations and tensors from forward and AC recomputation.
46
+ TEMP (str): Memory usage of temporary tensors during the backward pass including gradients of activations.
47
+ ALL_GATHER (str): Memory usage of all_gather output tensor.
48
+ REDUCE_SCATTER (str): Memory usage of reduce_scatter input tensor.
49
+ OPT (str): Memory usage of tensors storing optimizer states.
50
+ INP (str): Memory usage of input tensors.
51
+ """
52
+
53
+ SHARDED_PARAM = "Sharded Param"
54
+ UNSHARDED_PARAM = "Unsharded Param"
55
+ BUFFER = "Buffer"
56
+ SHARDED_GRAD = "Sharded Grad"
57
+ UNSHARDED_GRAD = "Unsharded Grad"
58
+ ACT = "Activation"
59
+ TEMP = "Temp"
60
+ ALL_GATHER = "All Gather"
61
+ REDUCE_SCATTER = "Reduce Scatter"
62
+ OPT = "OptState"
63
+ INP = "Inputs"
64
+
65
+
66
+ class _SavedFSDPMethods(NamedTuple):
67
+ pre_backward: Callable
68
+ post_backward: Callable
69
+
70
+
71
+ class _FSDPModState(_State):
72
+ """
73
+ Enumerates the states of FSDP modules during the forward and backward passes.
74
+ """
75
+
76
+ BEF_PRE_FW = "Before Pre-Forward"
77
+ AFT_PRE_FW = "After Pre-Forward"
78
+ BEF_POST_FW = "Before Post-Forward"
79
+ AFT_POST_FW = "After Post-Forward"
80
+ BEF_PRE_BW = "Before Pre-Backward"
81
+ AFT_PRE_BW = "After Pre-Backward"
82
+ BEF_POST_BW = "Before Post-Backward"
83
+ AFT_POST_BW = "After Post-Backward"
84
+ PRE_FW_AC = "Pre-Forward AC"
85
+ POST_FW_AC = "Post-Forward AC"
86
+ PEAK_FW = "Peak Forward"
87
+ PEAK_BW = "Peak Backward"
88
+
89
+
90
+ class _FSDPModMemStats:
91
+ """
92
+ A class to store the memory statistics of an FSDP module.
93
+
94
+ Args:
95
+ mod_fqn (str): The fully qualified name of the FSDP module.
96
+
97
+ Attributes:
98
+ snapshots (Dict[_FSDPModState, Dict[torch.device, Dict[str, int]]]): A dictionary of memory snapshots
99
+ of the module at different states as defined by ``_FSDPModState``. Each key is a device, and
100
+ each value is another dictionary with keys as memory reference types defined by ``_FSDPRefType`` and
101
+ values as the memory consumed in bytes.
102
+
103
+ """
104
+
105
+ def __init__(self, mod_fqn: str) -> None:
106
+ self.mod_fqn = mod_fqn
107
+ self.local_peak: dict[torch.device, int] = {}
108
+ self.snapshots: dict[
109
+ _FSDPModState, list[dict[torch.device, dict[str, int]]]
110
+ ] = {}
111
+
112
+
113
+ class _FSDPState(Enum):
114
+ PRE_FW = auto()
115
+ FW = auto()
116
+ POST_FW = auto()
117
+ PRE_BW = auto()
118
+ BW = auto()
119
+ POST_BW = auto()
120
+
121
+
122
+ class FSDPMemTracker(MemTracker):
123
+ """
124
+ A ``TorchDispatchMode`` based context manager that extends ``torch.distributed._tools.mem_tracker.MemTracker`` to track
125
+ and categorize the peak memory and module-wise memory usage of FSDP modules.
126
+
127
+ It tracks the peak memory usage across all the devices of all the FSDP modules in the module tree and categorizes
128
+ the tensor memory usage as defined by ``_FSDPRefType``. Further, it captures memory `snapshots` at different stages of
129
+ the module execution defined by ``_FSDPModState``.
130
+
131
+ Attributes:
132
+ memory_tracking: A weakref key dictionary to store the memory statistics of each module. Each key is a reference
133
+ to a module, and each value is a ``_FSDPModMemStats`` object that stores the memory statistics of the module.
134
+
135
+ Args:
136
+ mod (torch.nn.Module): The root FSDP module to be tracked.
137
+ optm (torch.optim.Optimizer, optional): The optimizer to be tracked.
138
+
139
+ Note: Please refer to ``torch.distributed._tools.mem_tracker.MemTracker`` to learn about the limitations.
140
+
141
+ Example usage
142
+
143
+ .. code-block:: python
144
+
145
+ module = ...
146
+ optimizer = ...
147
+ inp = ...
148
+ fmt = FSDPMemTracker(module, optimizer)
149
+ fmt.track_inputs((inp,))
150
+ with fmt:
151
+ optimizer.zero_grad()
152
+ loss = module(inp)
153
+ print("After Forward:")
154
+ fmt.display_snapshot("current")
155
+ loss.backward()
156
+ optimizer.step()
157
+ fmt.display_snapshot("peak")
158
+ fmt.display_modulewise_snapshots(depth=3, units="MB")
159
+
160
+ """
161
+
162
+ def __init__(
163
+ self,
164
+ mod: torch.nn.Module,
165
+ optm: torch.optim.Optimizer | None = None,
166
+ ) -> None:
167
+ super().__init__()
168
+ if not isinstance(mod, FSDPModule):
169
+ raise AssertionError("FSDPMemTracker only supports FSDP modules")
170
+ self._root_mod = mod
171
+ self._optm = optm
172
+ self._fsdp_mod_to_saved_methods: WeakIdKeyDictionary = WeakIdKeyDictionary()
173
+ self._fsdp_state: _FSDPState = _FSDPState.PRE_FW
174
+ self._ref_class: type[_RefType] = _FSDPRefType
175
+
176
+ def _instrument_fsdp_sharded_params_grads(
177
+ self, fsdp_param_group: FSDPParamGroup
178
+ ) -> None:
179
+ # Track sharded params and grads after initialization
180
+ for fsdp_param in fsdp_param_group.fsdp_params:
181
+ self._update_and_maybe_create_winfos(
182
+ fsdp_param.sharded_param,
183
+ _FSDPRefType.SHARDED_PARAM,
184
+ )
185
+ sharded_grad = fsdp_param.sharded_param.grad
186
+ if sharded_grad is not None:
187
+ self._update_and_maybe_create_winfos(
188
+ sharded_grad,
189
+ _FSDPRefType.SHARDED_GRAD,
190
+ )
191
+
192
+ def _fsdp_state_pre_forward(
193
+ self,
194
+ fsdp_mod: FSDPModule,
195
+ orig_fsdp_state_pre_fw: Callable[_P, tuple[tuple[Unpack[_Ts]], dict[str, Any]]],
196
+ ) -> Callable[_P, tuple[tuple[Unpack[_Ts]], dict[str, Any]]]:
197
+ # We capture memory snapshots before and after ``FSDPState._pre_forward`` to attribute the `unsharded` params
198
+ # and `all_gather` buffers. There are three cases:
199
+ # Case 1: If the module is not in the ``memory_tracking`` dictionary, create a new ``_FSDPModMemStats``
200
+ # instance for the module and add it to the ``memory_tracking`` dictionary.
201
+ # Case 2: If the module is already in the ``memory_tracking`` dictionary and we are in backward, this means
202
+ # we are in the AC region. We check if this is the top most module in the AC region. If it is,
203
+ # we store a weak reference and set the flag ``_in_ac`` to True.
204
+ # Case 3: If the module is already in the ``memory_tracking`` dictionary and we are in forward, this means
205
+ # this module is called for the second time. If it is a root module, that means we are in the next
206
+ # iteration and we error out. If it is not a root module, that means it's a submodule that is being
207
+ # used multiple times in the same iteration, which we allow and track.
208
+ # For Case 1 and 3, we also initialize the ``local_peak`` and ``PEAK_FW`` snapshot for the module.
209
+ # For Case 2 we only capture 1 snapshot after ``FSDPState._pre_forward`` runs because it is a no-op.
210
+ @wraps(orig_fsdp_state_pre_fw)
211
+ def inner(
212
+ *args: _P.args, **kwargs: _P.kwargs
213
+ ) -> tuple[tuple[Unpack[_Ts]], dict[str, Any]]:
214
+ self._fsdp_state = _FSDPState.PRE_FW
215
+ mod_fqn = self._mod_tracker.get_known_fqn(fsdp_mod)
216
+ if mod_fqn is None:
217
+ raise AssertionError
218
+ if fsdp_mod not in self.memory_tracking:
219
+ mod_stat = _FSDPModMemStats(mod_fqn)
220
+ self.memory_tracking[fsdp_mod] = mod_stat
221
+ snapshot = self.get_tracker_snapshot()
222
+ mod_stat.local_peak = {
223
+ dev: dev_snap[_TOTAL_KEY] for dev, dev_snap in snapshot.items()
224
+ }
225
+ mod_stat.snapshots.setdefault(_FSDPModState.PEAK_FW, []).append(
226
+ snapshot
227
+ )
228
+ mod_stat.snapshots.setdefault(_FSDPModState.BEF_PRE_FW, []).append(
229
+ deepcopy(snapshot)
230
+ )
231
+ elif not self._mod_tracker.is_bw:
232
+ parents = self._mod_tracker.parents - {mod_fqn}
233
+ if len(parents) == 1 and "Global" in parents:
234
+ raise NotImplementedError(
235
+ "FSDPMemTracker does not support memory tracking for multiple iterative calls."
236
+ " Either use ``reset_mod_stats`` to clear module memory stats for the previous iteration"
237
+ " or file a github issue if you need this feature."
238
+ )
239
+
240
+ # pyrefly: ignore [bad-assignment]
241
+ args, kwargs = orig_fsdp_state_pre_fw(*args, **kwargs)
242
+
243
+ fsdp_state = fsdp_mod._get_fsdp_state()
244
+ if fsdp_param_group := fsdp_state._fsdp_param_group:
245
+ for fsdp_param in fsdp_param_group.fsdp_params:
246
+ self._update_and_maybe_create_winfos(
247
+ fsdp_param.unsharded_param,
248
+ _FSDPRefType.UNSHARDED_PARAM,
249
+ )
250
+ mod_stat = self.memory_tracking[fsdp_mod]
251
+ if self._mod_tracker.is_bw:
252
+ state = _FSDPModState.PRE_FW_AC
253
+ if self._ac_mod is None:
254
+ self._ac_mod = weakref.ref(fsdp_mod)
255
+ self._in_ac = True
256
+ else:
257
+ state = _FSDPModState.AFT_PRE_FW
258
+ mod_stat.snapshots.setdefault(state, []).append(self.get_tracker_snapshot())
259
+ self._fsdp_state = _FSDPState.FW
260
+ # pyrefly: ignore [bad-return]
261
+ return args, kwargs
262
+
263
+ return inner
264
+
265
+ def _fsdp_state_post_forward(
266
+ self,
267
+ fsdp_mod: FSDPModule,
268
+ orig_fsdp_state_post_fw: Callable[_P, _R],
269
+ ) -> Callable[_P, _R]:
270
+ # We capture memory snapshots before and after ``FSDPState._post_forward`` to capture the resharded state
271
+ # if ``reshard_after_forward`` is not ``False``. There are two cases:
272
+ # Case 1: This is called in backward, which means we are in the AC region. If this is the top most module
273
+ # in the AC region, we set the flag ``_in_ac`` to False.
274
+ # Case 2: This is called in forward.
275
+ @wraps(orig_fsdp_state_post_fw)
276
+ def inner(*args: _P.args, **kwargs: _P.kwargs) -> _R:
277
+ mod_stat = self.memory_tracking[fsdp_mod]
278
+ if self._mod_tracker.is_bw:
279
+ state = _FSDPModState.POST_FW_AC
280
+ if self._ac_mod is not None and self._ac_mod() is fsdp_mod:
281
+ self._ac_mod = None
282
+ self._in_ac = False
283
+ else:
284
+ state = _FSDPModState.BEF_POST_FW
285
+ mod_stat.snapshots.setdefault(state, []).append(self.get_tracker_snapshot())
286
+ self._fsdp_state = _FSDPState.POST_FW
287
+
288
+ output = orig_fsdp_state_post_fw(*args, **kwargs)
289
+
290
+ if not self._mod_tracker.is_bw:
291
+ mod_stat.snapshots.setdefault(_FSDPModState.AFT_POST_FW, []).append(
292
+ self.get_tracker_snapshot()
293
+ )
294
+ return output
295
+
296
+ return inner
297
+
298
+ def _fsdp_param_group_pre_backward(
299
+ self,
300
+ fsdp_mod: FSDPModule,
301
+ orig_fsdp_param_group_pre_backward: Callable[_P, Any],
302
+ ) -> Callable[_P, None]:
303
+ # We capture memory snapshots before and after ``FSDPParamGroup.pre_backward`` to capture the pre-fetching
304
+ # and unsharding of params. We also initialize ``local_peak`` and ``PEAK_BW`` snapshot for the module.
305
+ @wraps(orig_fsdp_param_group_pre_backward)
306
+ def inner(*args: _P.args, **kwargs: _P.kwargs) -> None:
307
+ self._fsdp_state = _FSDPState.PRE_BW
308
+ mod_stat = self.memory_tracking[fsdp_mod]
309
+ snapshot = self.get_tracker_snapshot()
310
+ mod_stat.local_peak = {
311
+ dev: dev_snap[_TOTAL_KEY] for dev, dev_snap in snapshot.items()
312
+ }
313
+ mod_stat.snapshots.setdefault(_FSDPModState.PEAK_BW, []).append(snapshot)
314
+ mod_stat.snapshots.setdefault(_FSDPModState.BEF_PRE_BW, []).append(
315
+ deepcopy(snapshot)
316
+ )
317
+ orig_fsdp_param_group_pre_backward(*args, **kwargs)
318
+
319
+ mod_stat.snapshots.setdefault(_FSDPModState.AFT_PRE_BW, []).append(
320
+ self.get_tracker_snapshot()
321
+ )
322
+ self._fsdp_state = _FSDPState.BW
323
+
324
+ return inner
325
+
326
+ def _fsdp_param_group_post_backward(
327
+ self,
328
+ fsdp_mod: FSDPModule,
329
+ orig_fsdp_param_group_post_backward: Callable[_P, Any],
330
+ ) -> Callable[_P, None]:
331
+ # We capture the memory snapshots before and after ``FSDPParamGroup.post_backward`` to track and attribute
332
+ # the `unsharded` grads before the post backward and then `sharded` grads and `reduce_scatter` buffers
333
+ # after the post backward.
334
+ @wraps(orig_fsdp_param_group_post_backward)
335
+ def inner(*args: _P.args, **kwargs: _P.kwargs) -> None:
336
+ fsdp_state = fsdp_mod._get_fsdp_state()
337
+ if fsdp_param_group := fsdp_state._fsdp_param_group:
338
+ for fsdp_param in fsdp_param_group.fsdp_params:
339
+ unsharded_grad = fsdp_param._unsharded_param.grad
340
+ if unsharded_grad is not None:
341
+ self._update_and_maybe_create_winfos(
342
+ unsharded_grad,
343
+ _FSDPRefType.UNSHARDED_GRAD,
344
+ update_existing=True,
345
+ )
346
+
347
+ mod_stat = self.memory_tracking[fsdp_mod]
348
+ mod_stat.snapshots.setdefault(_FSDPModState.BEF_POST_BW, []).append(
349
+ self.get_tracker_snapshot()
350
+ )
351
+ self._fsdp_state = _FSDPState.POST_BW
352
+ orig_fsdp_param_group_post_backward(*args, **kwargs)
353
+
354
+ if fsdp_param_group := fsdp_state._fsdp_param_group:
355
+ for fsdp_param in fsdp_param_group.fsdp_params:
356
+ sharded_grad = fsdp_param.sharded_param.grad
357
+ if sharded_grad is not None:
358
+ self._update_and_maybe_create_winfos(
359
+ sharded_grad,
360
+ _FSDPRefType.SHARDED_GRAD,
361
+ )
362
+
363
+ mod_stat.snapshots.setdefault(_FSDPModState.AFT_POST_BW, []).append(
364
+ self.get_tracker_snapshot()
365
+ )
366
+
367
+ return inner
368
+
369
+ def _instrument_fsdp_module(self) -> None:
370
+ # We uninstall the existing `FSDPState._pre_forward` and `FSDPState._post_forward` hooks and install
371
+ # our own hooks that wrap them. We choose this over monkey-patching `FSDPParamGroup.pre_forward` and
372
+ # `FSDPParamGroup.post_forward` because during AC these won't be called.
373
+ # TODO(@sanketpurandare): This will need to be modified after this PR (https://github.com/pytorch/pytorch/pull/127786)
374
+ # lands. For backward we monkey-patch the `FSDPParamGroup.pre_backward` and `FSDPParamGroup.post_backward`.
375
+
376
+ # get the unique _MultiHandlers/RemoveHandlers and store in dictionary
377
+ # the _MultiHandlers object will only need to be grabbed once.
378
+ unique_handlers: dict[RemovableHandle, bool] = {}
379
+
380
+ for module in self._root_mod.modules():
381
+ if isinstance(module, FSDPModule):
382
+ fsdp_state = module._get_fsdp_state()
383
+ if fsdp_param_group := fsdp_state._fsdp_param_group:
384
+ if not unique_handlers.get(fsdp_state._pre_forward_hook_handle):
385
+ unique_handlers[fsdp_state._pre_forward_hook_handle] = True
386
+ if not unique_handlers.get(fsdp_state._post_forward_hook_handle):
387
+ unique_handlers[fsdp_state._post_forward_hook_handle] = True
388
+ # call remove on the handles once
389
+ for f_hook_handle in unique_handlers:
390
+ f_hook_handle.remove()
391
+
392
+ for module in self._root_mod.modules():
393
+ if isinstance(module, FSDPModule):
394
+ fsdp_state = module._get_fsdp_state()
395
+ if fsdp_param_group := fsdp_state._fsdp_param_group:
396
+ self._instrument_fsdp_sharded_params_grads(fsdp_param_group)
397
+ fsdp_state._pre_forward_hook_handle = (
398
+ module.register_forward_pre_hook(
399
+ self._fsdp_state_pre_forward(
400
+ module, fsdp_state._pre_forward
401
+ ),
402
+ prepend=True,
403
+ with_kwargs=True,
404
+ )
405
+ )
406
+
407
+ fsdp_state._post_forward_hook_handle = module.register_forward_hook(
408
+ self._fsdp_state_post_forward(module, fsdp_state._post_forward),
409
+ prepend=False,
410
+ always_call=True,
411
+ )
412
+ self._fsdp_mod_to_saved_methods[module] = _SavedFSDPMethods(
413
+ fsdp_param_group.pre_backward,
414
+ fsdp_param_group.post_backward,
415
+ )
416
+ fsdp_param_group.pre_backward = self._fsdp_param_group_pre_backward( # type: ignore[assignment]
417
+ module, fsdp_param_group.pre_backward
418
+ )
419
+ fsdp_param_group.post_backward = ( # type: ignore[assignment]
420
+ self._fsdp_param_group_post_backward(
421
+ module, fsdp_param_group.post_backward
422
+ )
423
+ )
424
+
425
+ for buffer in self._root_mod.buffers():
426
+ self._update_and_maybe_create_winfos(
427
+ buffer,
428
+ _FSDPRefType.BUFFER,
429
+ )
430
+
431
+ def _instrument_optimizer(self) -> None:
432
+ # Register a hook on the optimizer step to track the optimizer states.
433
+ # The pre-hook is to set the flag ``_in_opt`` to True. The post-hook unsets the flag,
434
+ # and also tracks any optimizer states that are created during the optimizer step.
435
+ if self._optm is not None:
436
+ self._track_optimizer_states(_FSDPRefType.OPT, self._optm)
437
+
438
+ def _opt_step_pre_hook(
439
+ optimizer: optim.Optimizer, args: Any, kwargs: Any
440
+ ) -> None:
441
+ self._in_opt = True
442
+
443
+ def _opt_step_post_hook(
444
+ optimizer: optim.Optimizer, args: Any, kwargs: Any
445
+ ) -> None:
446
+ self._track_optimizer_states(_FSDPRefType.OPT, optimizer)
447
+ self._in_opt = False
448
+
449
+ self._optimizer_hook_handles = (
450
+ self._optm.register_step_pre_hook(_opt_step_pre_hook),
451
+ self._optm.register_step_post_hook(_opt_step_post_hook),
452
+ )
453
+
454
+ def _register_module_and_optimizer_hooks(self) -> None:
455
+ self._instrument_fsdp_module()
456
+ self._instrument_optimizer()
457
+
458
+ def _deregister_module_and_optimizer_hooks(self) -> None:
459
+ for (
460
+ fsdp_mod,
461
+ saved_methods,
462
+ ) in self._fsdp_mod_to_saved_methods.items():
463
+ fsdp_state = fsdp_mod._get_fsdp_state()
464
+ fsdp_state._pre_forward_hook_handle.remove()
465
+ fsdp_state._post_forward_hook_handle.remove()
466
+ fsdp_state._pre_forward_hook_handle = fsdp_mod.register_forward_pre_hook(
467
+ fsdp_state._pre_forward, prepend=True, with_kwargs=True
468
+ )
469
+ fsdp_state._post_forward_hook_handle = fsdp_mod.register_forward_hook(
470
+ fsdp_state._post_forward, prepend=False
471
+ )
472
+ if fsdp_param_group := fsdp_state._fsdp_param_group:
473
+ fsdp_param_group.pre_backward = saved_methods.pre_backward
474
+ fsdp_param_group.post_backward = saved_methods.post_backward
475
+ self._fsdp_mod_to_saved_methods.clear()
476
+
477
+ if self._optimizer_hook_handles is not None:
478
+ for handle in self._optimizer_hook_handles:
479
+ handle.remove()
480
+ self._optimizer_hook_handles = None
481
+
482
+ def track_inputs(self, inputs: tuple[Any, ...]) -> None:
483
+ """
484
+ This is used to track the input tensors to the model and annotate them as ``Inputs``.
485
+ Args:
486
+ inputs (Tuple[Any]): A tuple containing the input data. This can include tensors
487
+ as well as other data types. Only tensors will be tracked.
488
+ """
489
+
490
+ def _track_inputs(t: torch.Tensor) -> None:
491
+ self._update_and_maybe_create_winfos(
492
+ t,
493
+ _FSDPRefType.INP,
494
+ )
495
+
496
+ tree_map_only(torch.Tensor, _track_inputs, inputs)
497
+
498
+ def track_external(
499
+ self, *external: nn.Module | optim.Optimizer | torch.Tensor
500
+ ) -> None:
501
+ """This is no-op for ``FSDPMemTracker``"""
502
+
503
+ def __enter__(self) -> "FSDPMemTracker":
504
+ if self._depth == 0:
505
+ # None in eager, a FakeTensorMode instance in SAC. Used in
506
+ # __torch_dispatch__ to skip DTensor propagation ops.
507
+ self._fake_mode_on_entry = active_fake_mode()
508
+ self._register_module_and_optimizer_hooks()
509
+ self._track_resize()
510
+ self._peak_mem_snap = self.get_tracker_snapshot()
511
+ self._peak_mem = {
512
+ dev: dev_snap[_TOTAL_KEY]
513
+ for dev, dev_snap in self._peak_mem_snap.items()
514
+ }
515
+ self._mod_tracker.__enter__()
516
+ TorchDispatchMode.__enter__(self)
517
+ self._depth += 1
518
+ return self
519
+
520
+ def __exit__(self, *args: Any) -> None:
521
+ self._depth -= 1
522
+ if self._depth == 0:
523
+ self._deregister_module_and_optimizer_hooks()
524
+ self._restore_resize()
525
+ self._mod_tracker.__exit__(*args)
526
+ TorchDispatchMode.__exit__(self, *args)
527
+
528
+ def __torch_dispatch__(self, func, types, args=..., kwargs=None): # type: ignore[no-untyped-def]
529
+ # When running this mode with DTensor, ordinarily all modes will
530
+ # run **before** subclasses get a chance to run.
531
+ # Returning NotImplemented here gives us a chance to let DTensor
532
+ # run and desugar into local tensor ops, before `MemTracker` sees them.
533
+ if any(t == DTensor for t in types):
534
+ return NotImplemented
535
+ if (
536
+ func is torch.ops._c10d_functional.wait_tensor.default
537
+ and active_fake_mode()
538
+ ):
539
+ # N.B: This is a hacky way to override the Meta IMPL of wait_tensor. The original impl returns
540
+ # a new tensor which does not happen in eager mode, when a wait_tensor is called.
541
+ # pyrefly: ignore [unsupported-operation]
542
+ res = args[0]
543
+ else:
544
+ res = func(*args, **kwargs or {})
545
+ # DTensor sharding propagation may use a nested FakeTensorMode to
546
+ # compute output metadata (shapes, placements). The real memory
547
+ # usage comes from the local op on shard data, which
548
+ # runs outside DTensor's fake mode. We use identity comparison
549
+ # against the fake mode at tracker entry to skip propagation ops
550
+ # while still tracking local ops in both eager and SAC contexts.
551
+ # - Eager (entry mode is None): real ops have no fake mode
552
+ # (None is None), DTensor propagation ops are skipped (B is not None).
553
+ # - SAC (entry mode is A): SAC ops run under mode A (A is A),
554
+ # DTensor propagation ops are skipped (B is not A).
555
+ if active_fake_mode() is self._fake_mode_on_entry:
556
+ # If we are tracking an optimizer state, we use the optimizer reference type.
557
+ # If we are in backward region and not in AC region, we use the backward reference type.
558
+ # Else we use the forward reference type.
559
+ if self._in_opt:
560
+ reftype = _FSDPRefType.OPT
561
+ elif self._mod_tracker.is_bw and not self._in_ac:
562
+ reftype = _FSDPRefType.TEMP
563
+ else:
564
+ reftype = _FSDPRefType.ACT
565
+ if func is c10d._allgather_base_.default and self._fsdp_state in [
566
+ _FSDPState.PRE_FW,
567
+ _FSDPState.PRE_BW,
568
+ ]:
569
+ # pyrefly: ignore [unsupported-operation]
570
+ output_tensor = args[0]
571
+ self._update_and_maybe_create_winfos(
572
+ output_tensor,
573
+ _FSDPRefType.ALL_GATHER,
574
+ update_existing=True,
575
+ )
576
+ if (
577
+ func is c10d._reduce_scatter_base_.default
578
+ and self._fsdp_state == _FSDPState.POST_BW
579
+ ):
580
+ # pyrefly: ignore [unsupported-operation]
581
+ input_tensor = args[1]
582
+ self._update_and_maybe_create_winfos(
583
+ input_tensor,
584
+ _FSDPRefType.REDUCE_SCATTER,
585
+ update_existing=True,
586
+ )
587
+
588
+ tree_map_only(torch.Tensor, partial(self._track, reftype), res)
589
+ peak_state = (
590
+ _FSDPModState.PEAK_BW
591
+ if self._mod_tracker.is_bw
592
+ else _FSDPModState.PEAK_FW
593
+ )
594
+ self._update_peak_stats(peak_state)
595
+ return res
western_ai/usa/pytorch/torch/distributed/_tools/ilp_utils.py ADDED
@@ -0,0 +1,293 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ from collections import OrderedDict
3
+ from typing import cast, TypedDict
4
+
5
+ import numpy as np
6
+
7
+ import torch
8
+ from torch.distributed._tools.mem_tracker import (
9
+ _MemRefType,
10
+ _ModMemStats,
11
+ _ModState,
12
+ MemTracker,
13
+ )
14
+ from torch.distributed._tools.runtime_estimator import RuntimeEstimator
15
+ from torch.distributed._tools.sac_estimator import SACEstimator, SACTradeOffStats
16
+
17
+
18
+ class ModOrder(TypedDict):
19
+ fw_pre_order: list[str]
20
+ bw_pre_order: list[str]
21
+ fw_post_order: list[str]
22
+ bw_post_order: list[str]
23
+
24
+
25
+ class ModRuntime(TypedDict):
26
+ fw: float
27
+ bw: float
28
+
29
+
30
+ class ModStats(TypedDict):
31
+ fqn: str
32
+ # per-module params
33
+ param_per_module: int
34
+ # per-module grads
35
+ grad_per_module: int
36
+ # total accumulated gradients up to and including this module
37
+ grad_total: int
38
+ # per module fw activation size (excluding input and output)
39
+ act_fw_per_module: int
40
+ # per module bw activation size during peak_bw
41
+ act_bw_per_module: int
42
+ # per module activation grad size during peak_bw
43
+ act_grad_per_module: int
44
+ # total activation size up to but excluding the current module
45
+ # includes input of the current module (i.e., output of previous module)
46
+ act_total: int
47
+ # Inputs to the module
48
+ input_per_module: int
49
+ # Outputs of the module
50
+ output_per_module: int
51
+ # Total fw run-time of the module
52
+ fw_runtime_per_module: float
53
+ # Total bw run-time of the module
54
+ bw_runtime_per_module: float
55
+ # Is this module a leaf module
56
+ is_leaf: bool
57
+ # Total ac run-time of the module
58
+ sac_runtime: float
59
+ # Total ac_memory for the module
60
+ sac_memory: int
61
+ # Number of piecewise-linear functions used for approximating ac tradeoff curve
62
+ n_segments: int
63
+ # Slopes of the of piecewise-linear functions
64
+ slopes: list[float]
65
+ # Intercepts of the of piecewise-linear functions
66
+ intercepts: list[float]
67
+ # X breakpoints of the of piecewise-linear functions
68
+ breakpoints: list[float]
69
+ # Original trade-off curves
70
+ tradeoff_curve: OrderedDict[float, float]
71
+
72
+
73
+ class ModuleInfo(TypedDict):
74
+ mod_order: ModOrder
75
+ mod_stats: list[ModStats]
76
+
77
+
78
+ def aggregate_stats(
79
+ model: torch.nn.Module,
80
+ mem_tracker: MemTracker,
81
+ runtime_estimator: RuntimeEstimator,
82
+ sac_estimator: SACEstimator,
83
+ dev: torch.device,
84
+ ) -> ModuleInfo:
85
+ """
86
+ Collect modulewise stats for a given model, including memory, runtime, and AC tradeoff stats.
87
+
88
+ Args:
89
+ model: nn.Module object
90
+ runtime_estimator: RuntimeEstimator object with runtime stats
91
+ mem_tracker: MemTracker object with memory stats
92
+ sac_estimator: SACEstimator object with AC tradeoff stats
93
+ dev: device the model was run on (used to extract memory stats from MemTracker)
94
+
95
+ Returns:
96
+ ModuleInfo: A dictionary with module order and module stats.
97
+ """
98
+
99
+ # Memory stats
100
+ mod_mem_stats: dict[torch.nn.Module, _ModMemStats] = dict(
101
+ copy.deepcopy(mem_tracker.memory_tracking)
102
+ )
103
+
104
+ # Runtime stats
105
+ mod_runtime_stats: dict[str, ModRuntime] = {
106
+ fqn: {"fw": v["fw"], "bw": v["bw"]}
107
+ for fqn, v in runtime_estimator.mod_runtimes.items()
108
+ }
109
+
110
+ # Module order
111
+ mod_order: ModOrder = {
112
+ "fw_pre_order": list(runtime_estimator.mod_fw_pre_order),
113
+ "bw_pre_order": list(runtime_estimator.mod_bw_pre_order),
114
+ "fw_post_order": list(runtime_estimator.mod_fw_post_order),
115
+ "bw_post_order": list(runtime_estimator.mod_bw_post_order),
116
+ }
117
+
118
+ # Selective Activation Checkpointing stats
119
+ sac_estimator.pwlf_sac_tradeoff_curve()
120
+ mod_sac_tradeoff_stats: dict[str, SACTradeOffStats] = copy.deepcopy(
121
+ sac_estimator.sac_mod_tradeoff_stats
122
+ )
123
+
124
+ module_info: ModuleInfo = {
125
+ "mod_order": mod_order,
126
+ "mod_stats": [],
127
+ }
128
+
129
+ for mod in model.modules():
130
+ if mod_mem_stat := mod_mem_stats.get(mod):
131
+ if tradeoff_stats := mod_sac_tradeoff_stats.get(mod_mem_stat.mod_fqn):
132
+ sac_runtime = tradeoff_stats.sac_runtime
133
+ sac_memory = tradeoff_stats.sac_memory
134
+ n_segments = tradeoff_stats.n_segments
135
+ slopes = tradeoff_stats.slopes
136
+ intercepts = tradeoff_stats.intercepts
137
+ breakpoints = tradeoff_stats.fit_breaks
138
+ tradeoff_curve = tradeoff_stats.tradeoff_curve
139
+ is_leaf = False
140
+ else:
141
+ sac_runtime = sac_memory = n_segments = 0
142
+ slopes = intercepts = breakpoints = []
143
+ tradeoff_curve: OrderedDict[float, float] = OrderedDict() # type: ignore[no-redef]
144
+ is_leaf = True
145
+ mod_stat: ModStats = {
146
+ "fqn": mod_mem_stat.mod_fqn,
147
+ "param_per_module": mod_mem_stat.parameter_mem,
148
+ "grad_per_module": mod_mem_stat.parameter_mem,
149
+ "grad_total": mod_mem_stat.snapshots[_ModState.PRE_BW][-1][dev][
150
+ _MemRefType.GRAD
151
+ ],
152
+ "act_fw_per_module": max(
153
+ 0,
154
+ mod_mem_stat.snapshots[_ModState.POST_FW][-1][dev][_MemRefType.ACT]
155
+ - mod_mem_stat.snapshots[_ModState.PRE_FW][-1][dev][_MemRefType.ACT]
156
+ - mod_mem_stat.output_mem,
157
+ ),
158
+ "act_bw_per_module": max(
159
+ 0,
160
+ mod_mem_stat.snapshots[_ModState.PEAK_BW][-1][dev][_MemRefType.ACT],
161
+ ),
162
+ "act_grad_per_module": (
163
+ mod_mem_stat.snapshots[_ModState.PEAK_BW][-1][dev][_MemRefType.TEMP]
164
+ - mod_mem_stat.snapshots[_ModState.PRE_BW][-1][dev][
165
+ _MemRefType.TEMP
166
+ ]
167
+ ),
168
+ "act_total": mod_mem_stat.snapshots[_ModState.POST_FW][-1][dev][
169
+ _MemRefType.ACT
170
+ ],
171
+ "input_per_module": mod_mem_stat.input_mem,
172
+ "output_per_module": mod_mem_stat.output_mem,
173
+ "fw_runtime_per_module": mod_runtime_stats[mod_mem_stat.mod_fqn]["fw"],
174
+ "bw_runtime_per_module": mod_runtime_stats[mod_mem_stat.mod_fqn]["bw"],
175
+ "is_leaf": is_leaf,
176
+ "sac_runtime": sac_runtime,
177
+ "sac_memory": sac_memory,
178
+ "n_segments": n_segments,
179
+ "slopes": slopes,
180
+ "intercepts": intercepts,
181
+ "breakpoints": breakpoints,
182
+ "tradeoff_curve": tradeoff_curve,
183
+ }
184
+ module_info["mod_stats"].append(mod_stat)
185
+
186
+ return module_info
187
+
188
+
189
+ class Node(ModStats):
190
+ index: int # index according to forward pre-order
191
+ pos_fw_post_order: int # index according to forward post-order
192
+
193
+
194
+ class Graph:
195
+ def __init__(self, n: int) -> None:
196
+ self.nodes: list[Node] = []
197
+ self.name2node: dict[str, Node] = {}
198
+ self.ad_matrix = np.zeros((n, n))
199
+ self.fw_post_order: list[str] = []
200
+
201
+ def add_node(self, node: Node) -> None:
202
+ self.nodes.append(node)
203
+ self.name2node[node["fqn"]] = node
204
+
205
+
206
+ def parse_module_info(module_info: ModuleInfo) -> Graph:
207
+ """
208
+ Parse module info and create a graph (tree) of modules. The graph will be
209
+ used by MILP solver to find optimal SAC and/or FSDP configurations.
210
+ """
211
+ mod_stats = module_info["mod_stats"]
212
+ fw_pre_order = module_info["mod_order"]["fw_pre_order"]
213
+ # assertion and number of nodes
214
+ if len(mod_stats) != len(fw_pre_order):
215
+ raise AssertionError
216
+ n_nodes = len(mod_stats)
217
+
218
+ # create graph
219
+ g = Graph(n_nodes)
220
+ g.fw_post_order = module_info["mod_order"]["fw_post_order"]
221
+
222
+ # sort the modules by pre-order and add them to the graph
223
+ module_info["mod_stats"] = sorted(
224
+ mod_stats, key=lambda x: fw_pre_order.index(x["fqn"])
225
+ )
226
+ for i, one_mod_stats in enumerate(mod_stats):
227
+ node: Node = cast(Node, one_mod_stats)
228
+ node["index"] = i
229
+ node["pos_fw_post_order"] = g.fw_post_order.index(node["fqn"])
230
+ g.add_node(node)
231
+
232
+ # set up ancestor-descendant matrix
233
+ for i in range(n_nodes):
234
+ for j in range(i, n_nodes):
235
+ if is_self_or_submodule(g.nodes[j]["fqn"], g.nodes[i]["fqn"]):
236
+ g.ad_matrix[i][j] = 1
237
+ else:
238
+ break
239
+
240
+ return g
241
+
242
+
243
+ def is_self_or_submodule(name_descendant: str, name_ancestor: str) -> bool:
244
+ """
245
+ check if name_descendant is a submodule of name_ancestor, or if they are the same
246
+ """
247
+ return name_descendant == name_ancestor or name_ancestor + "." in name_descendant
248
+
249
+
250
+ def is_submodule(name_descendant: str, name_ancestor: str) -> bool:
251
+ """
252
+ if name_descendant is a submodule of name_ancestor, but not the same
253
+ """
254
+ return name_ancestor + "." in name_descendant
255
+
256
+
257
+ def display_bytes(b: int, unit: str = "MiB") -> str:
258
+ """
259
+ return a string that represent the number of bytes in a desired unit
260
+ """
261
+ if unit == "KiB":
262
+ return f"{b / 2**10:.2f} KiB"
263
+ if unit == "MiB":
264
+ return f"{b / 2**20:.2f} MiB"
265
+ if unit == "GiB":
266
+ return f"{b / 2**30:.2f} GiB"
267
+ return f"{b:.2f} bytes"
268
+
269
+
270
+ def get_peak_memory_runtime_baseline(graph: Graph) -> tuple[int, float]:
271
+ """
272
+ Get the baseline peak memory and runtime.
273
+ Baseline here means there is no FSDP or AC.
274
+ Memory includes the parameters, gradients, activations, and activation gradients.
275
+ Memory does not include e.g., optimizer states, embedding tables, etc.
276
+
277
+ Returns:
278
+ int: peak memory in bytes
279
+ float: compute time in ms
280
+ """
281
+ P_1 = graph.nodes[0]["param_per_module"]
282
+ num_nodes = len(graph.nodes)
283
+ peak_mem = 0
284
+ for i in range(num_nodes):
285
+ TG_i = graph.nodes[i]["grad_total"]
286
+ AG_i = graph.nodes[i]["act_grad_per_module"]
287
+ TA_i = graph.nodes[i]["act_total"]
288
+ peak_mem = max(peak_mem, P_1 + TG_i + AG_i + TA_i)
289
+ compute_time = (
290
+ graph.nodes[0]["fw_runtime_per_module"]
291
+ + graph.nodes[0]["bw_runtime_per_module"]
292
+ )
293
+ return (peak_mem, compute_time)
western_ai/usa/pytorch/torch/distributed/_tools/mem_tracker.py ADDED
@@ -0,0 +1,956 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os
3
+ import re
4
+ import warnings
5
+ from collections.abc import Callable
6
+ from copy import deepcopy
7
+ from enum import auto, Enum
8
+ from functools import partial, wraps
9
+ from typing import Any, TYPE_CHECKING
10
+ from typing_extensions import Self
11
+
12
+ import torch
13
+ import torch.distributed._tools.fake_collectives
14
+ from torch import nn, optim
15
+ from torch._guards import active_fake_mode
16
+ from torch.distributed._tools.common_utils import get_untyped_storages
17
+ from torch.distributed._tools.mod_tracker import ModTracker
18
+ from torch.distributed.tensor import DTensor
19
+ from torch.optim.optimizer import (
20
+ register_optimizer_step_post_hook,
21
+ register_optimizer_step_pre_hook,
22
+ )
23
+ from torch.utils._python_dispatch import TorchDispatchMode
24
+ from torch.utils._pytree import tree_flatten, tree_map_only
25
+ from torch.utils.weak import WeakIdKeyDictionary, weakref
26
+
27
+
28
+ if TYPE_CHECKING:
29
+ from torch.utils.hooks import RemovableHandle
30
+
31
+ # This value is hard-coded here:
32
+ # https://github.com/pytorch/pytorch/blob/5fba5d83f0703ff8077ab65448a998e9ad6598fd/c10/cuda/CUDACachingAllocator.cpp#L117
33
+ _PYTORCH_MIN_ALLOCATE = (
34
+ 2**9 if int(os.environ.get("PYTORCH_NO_CUDA_MEMORY_CACHING", 0)) == 0 else 1
35
+ )
36
+ _TOTAL_KEY = "Total"
37
+
38
+ __all__ = ["MemTracker"]
39
+
40
+
41
+ class _RefType(str, Enum):
42
+ """Base Class for defining memory reference types, categorizing tensors based on their usage within a model."""
43
+
44
+
45
+ class _State(str, Enum):
46
+ """Base Class for defining module state to capture snapshots ."""
47
+
48
+
49
+ class _MemRefType(_RefType):
50
+ """
51
+ An enum to define memory reference types, categorizing tensors based on their usage within a model.
52
+
53
+ - PARAM: Tensors registered as nn.Parameter within modules.
54
+ - BUFFER: Tensors registered as nn.Buffer within modules.
55
+ - GRAD: Gradients associated with parameters.
56
+ - ACT: Tensors produced during the forward pass and recomputation in activation checkpointing.
57
+ - TMP: Temporary memory used during the backward pass, including gradients of activations.
58
+ - OPT: Tensors holding optimizer states.
59
+ - OTH: Tensors registered via `track_external` that do not fit the above categories.
60
+ """
61
+
62
+ PARAM = "Parameter"
63
+ BUFFER = "Buffer"
64
+ GRAD = "Gradient"
65
+ ACT = "Activation"
66
+ TEMP = "Temp"
67
+ OPT = "Optstate"
68
+ OTH = "Other"
69
+
70
+
71
+ class _ModState(_State):
72
+ """
73
+ An enum to define the state of a module.
74
+
75
+ - PRE_FW: The module is about to run the forward pass.
76
+ - POST_FW: The module has finished running the forward pass.
77
+ - PEAK_FW: The module has reached the peak memory usage during the forward pass.
78
+ - PRE_BW: The module is about to run the backward pass.
79
+ - PRE_FW_AC: The module is about to run the forward pass with activation checkpointing.
80
+ - POST_FW_AC: The module has finished running the forward pass with activation checkpointing.
81
+ - POST_BW: The module has finished running the backward pass.
82
+ - PEAK_BW: The module has reached the peak memory usage during the backward pass.
83
+ """
84
+
85
+ PRE_FW = "Pre-Forward"
86
+ POST_FW = "Post-Forward"
87
+ PEAK_FW = "Peak-Forward"
88
+ PRE_BW = "Pre-Backward"
89
+ PRE_FW_AC = "Pre-Forward-AC"
90
+ POST_FW_AC = "Post-Forward-AC"
91
+ POST_BW = "Post-Backward"
92
+ PEAK_BW = "Peak-Backward"
93
+
94
+
95
+ class _ModMemStats:
96
+ """
97
+ A class to store the memory statistics of a module.
98
+
99
+ Args:
100
+ mod_fqn (str): The fully qualified name of the module.
101
+ Attributes:
102
+ mod_fqn (str): The fully qualified name of the module.
103
+ parameter_mem (int): The memory usage of the parameters of the module.
104
+ buffer_mem (int): The memory usage of the buffers of the module.
105
+ input_mem (int): The memory usage of the inputs to the module.
106
+ output_mem (int): The memory usage of the outputs from the module.
107
+ snapshots (Dict[_ModState, Dict[torch.device, Dict[str, int]]]): A dictionary of memory snapshots
108
+ of the module at different states defined by ``_ModState``.
109
+ Note:
110
+ The memory snapshot is stored as a dictionary - Dict[torch.device, Dict[str, int]], where each key is a device,
111
+ and each value is another dictionary with keys as memory reference types defined by `_MemRefType` and
112
+ values as the memory consumed in bytes.
113
+ """
114
+
115
+ def __init__(self, mod_fqn: str):
116
+ self.mod_fqn = mod_fqn
117
+ self.parameter_mem: int
118
+ self.buffer_mem: int
119
+ self.input_mem: int
120
+ self.output_mem: int
121
+ self.local_peak: dict[torch.device, int] = {}
122
+ self.snapshots: dict[_ModState, list[dict[torch.device, dict[str, int]]]] = {}
123
+
124
+
125
+ class _WeakRefInfo:
126
+ """
127
+ Manages memory statistics and device attributes for tensor storages.
128
+ """
129
+
130
+ def __init__(
131
+ self, size: int, element_size: int, device: torch.device, reftype: _RefType
132
+ ) -> None:
133
+ """
134
+ Initializes the ``_WeakRefInfo`` object with tensor storage properties.
135
+
136
+ Args:
137
+ size (int): The number of elements in the tensor storage.
138
+ element_size (int): The size of each element in the tensor storage.
139
+ device (torch.device): The device on which the tensor is allocated.
140
+ reftype (_RefType): The reference type of the tensor.
141
+ """
142
+ self.size = size
143
+ self.element_size = element_size
144
+ self.reftype = reftype
145
+ self.device = device
146
+ self.mem_consumed = self._calculate_mem_consumed()
147
+
148
+ def _calculate_mem_consumed(self) -> int:
149
+ """
150
+ Calculates the memory consumed by the tensor storage, considering device-specific allocation rules.
151
+
152
+ Returns:
153
+ int: The memory consumed in bytes.
154
+ """
155
+ mem = self.size * self.element_size
156
+ if self.device.type == "cuda":
157
+ return math.ceil((mem) / _PYTORCH_MIN_ALLOCATE) * _PYTORCH_MIN_ALLOCATE
158
+ return mem
159
+
160
+ def update_mem_consumed(self, st: torch.UntypedStorage) -> int:
161
+ """
162
+ Updates and returns the memory consumed if the storage size has changed.
163
+
164
+ Args:
165
+ st (torch.UntypedStorage): The tensor storage to check for size updates.
166
+
167
+ Returns:
168
+ int: The updated memory consumed in bytes.
169
+ """
170
+ if st.size() != self.size:
171
+ self.size = st.size()
172
+ self.mem_consumed = self._calculate_mem_consumed()
173
+ return self.mem_consumed
174
+
175
+ @classmethod
176
+ def create_winfo(
177
+ cls,
178
+ st: torch.UntypedStorage,
179
+ device: torch.device,
180
+ reftype: _RefType,
181
+ callback: Callable[[Self, weakref.ref], Any] | None = None,
182
+ ) -> tuple[Self, weakref.ref]:
183
+ """
184
+ Creates a new ``_WeakRefInfo`` instance and a weak reference to a ``torch.UntypedStorage`` object,
185
+ optionally attaching a callback to the weak reference.
186
+
187
+ Args:
188
+ st (torch.UntypedStorage): The storage object for which to create the weak reference info.
189
+ device (torch.device): The device associated with the storage object.
190
+ reftype (_RefType): The type of reference, used to categorize the storage.
191
+ callback (Optional[Callable[[Self, weakref.ref]]]): A callback function that is called when
192
+ the storage object is about to be finalized (garbage collected). The callback function
193
+ should accept two arguments: the ``_WeakRefInfo`` instance and the weak reference to the storage.
194
+ Returns:
195
+ Tuple[Self, weakref.ref]: A tuple containing the newly created ``_WeakRefInfo`` instance and the
196
+ weak reference to the storage object. The weak reference may have an attached callback if provided.
197
+ """
198
+
199
+ winfo = cls(st.size(), st.element_size(), device, reftype)
200
+ w_st = weakref.ref(st, partial(callback, winfo) if callback else None)
201
+ return winfo, w_st
202
+
203
+
204
+ def _get_mem_divisor(units: str) -> int:
205
+ unit_dict = {"B": 1, "KiB": 2**10, "MiB": 2**20, "GiB": 2**30}
206
+ if units in unit_dict:
207
+ return unit_dict[units]
208
+ else:
209
+ raise ValueError(
210
+ f"Unsupported unit: {units}. Supported units are: {', '.join(unit_dict.keys())}"
211
+ )
212
+
213
+
214
+ def _rounding_fn(value: int, divisor: int, precision: int) -> float | int:
215
+ return value if divisor == 1 else round(value / divisor, precision)
216
+
217
+
218
+ def _print_snapshot(snapshot: dict[torch.device, dict[str, int]], units: str) -> None:
219
+ if len(snapshot) == 0:
220
+ print("No memory tracked.")
221
+ return
222
+ divisor = _get_mem_divisor(units)
223
+ for dev, dev_snap in snapshot.items():
224
+ if _rounding_fn(dev_snap[_TOTAL_KEY], divisor, 2) <= 0:
225
+ continue
226
+ print(
227
+ f"Device: {dev}",
228
+ *(
229
+ f"\t{k.value}: {_rounding_fn(v, divisor, 2)} {units}"
230
+ if isinstance(k, _RefType)
231
+ else f"\t{k}: {_rounding_fn(v, divisor, 2)} {units}"
232
+ for k, v in dev_snap.items()
233
+ ),
234
+ sep="\n",
235
+ )
236
+
237
+
238
+ def _print_snapshot_tabular(
239
+ snapshot: dict[torch.device, dict[str, int]], units: str
240
+ ) -> None:
241
+ if len(snapshot) == 0:
242
+ print("No memory tracked.")
243
+ return
244
+ try:
245
+ from tabulate import tabulate
246
+ except ImportError as err:
247
+ raise ImportError(
248
+ "Please install tabulate to use the tabulate option."
249
+ ) from err
250
+ divisor = _get_mem_divisor(units)
251
+ table_data = []
252
+ key_list = list(next(iter(snapshot.values())).keys())
253
+ headers = ["Device"] + [
254
+ f"{key.value}" if isinstance(key, _RefType) else f"{key}" for key in key_list
255
+ ]
256
+
257
+ for dev, dev_snap in snapshot.items():
258
+ if _rounding_fn(dev_snap[_TOTAL_KEY], divisor, 2) <= 0:
259
+ continue
260
+ row = [str(dev)]
261
+ row.extend(f"{_rounding_fn(v, divisor, 2)} {units}" for v in dev_snap.values())
262
+ table_data.append(row)
263
+ print(tabulate(table_data, headers=headers, tablefmt="rst"))
264
+
265
+
266
+ def _print_state_snapshots(
267
+ snapshots: dict[_State, list[dict[torch.device, dict[str, int]]]], units: str
268
+ ) -> None:
269
+ for state, snapshot_list in snapshots.items():
270
+ print(f"{state.value}")
271
+ for i, snapshot in enumerate(snapshot_list):
272
+ print(f"# {i + 1}:")
273
+ _print_snapshot(snapshot, units)
274
+ print()
275
+
276
+
277
+ def _print_state_snapshots_tabular(
278
+ snapshots: dict[_State, list[dict[torch.device, dict[str, int]]]], units: str
279
+ ) -> None:
280
+ try:
281
+ from tabulate import tabulate
282
+ except ImportError as err:
283
+ raise ImportError(
284
+ "Please install tabulate to use the tabulate option."
285
+ ) from err
286
+
287
+ table_data = []
288
+ last_state_call = None
289
+ divisor = _get_mem_divisor(units)
290
+ for state, snapshot_list in snapshots.items():
291
+ for i, snapshot in enumerate(snapshot_list):
292
+ state_call = f"{state.value} # {i + 1}"
293
+ for dev, dev_snap in snapshot.items():
294
+ if _rounding_fn(dev_snap[_TOTAL_KEY], divisor, 2) <= 0:
295
+ continue
296
+ row = {
297
+ "State & Call": (
298
+ state_call if state_call != last_state_call else ""
299
+ ),
300
+ "Device": str(dev),
301
+ }
302
+ last_state_call = state_call
303
+ for k, v in dev_snap.items():
304
+ row[f"{k.value}" if isinstance(k, _RefType) else f"{k}"] = (
305
+ f"{_rounding_fn(v, divisor, 2)} {units}"
306
+ )
307
+ table_data.append(row)
308
+ print(tabulate(table_data, headers="keys", tablefmt="rst"))
309
+
310
+
311
+ class _UpdateType(Enum):
312
+ # These are used for tracking updates to the continuouly maintained memory snapshot.
313
+ # ADD - When a new tensor storage is tracked
314
+ # DEL - When a tensor storage is about to be finalized (garbage collected).
315
+ # REF - When a tensor reference is updated, for instance, the gradients are marked as
316
+ # generic backward reference types until the grad_hook categorizes them as gradients.
317
+ # SIZE - When a tensor's storage is resized.
318
+ ADD = auto()
319
+ DEL = auto()
320
+ REF = auto()
321
+ SIZE = auto()
322
+
323
+
324
+ class MemTracker(TorchDispatchMode):
325
+ """
326
+ A TorchDispatchMode to track, categorize and attribute the tensor memory created or accessed within its context.
327
+
328
+ It categorizes the tracked tensors as parameters, buffers, activations, gradients, temporary memory and optimizer states
329
+ as defined by ``_MemRefType`` within its context. It captures memory `snapshots` for the modules, called within its context,
330
+ at various states defined by ``_ModState``.
331
+
332
+ Attributes:
333
+ memory_tracking: A weakref key dictionary to store the memory statistics of each module. Each key
334
+ is a reference to a module, and each value is a ``_ModMemStats`` object that stores the memory
335
+ statistics of the module.
336
+
337
+ Note:
338
+ The MemTracker should be used as a context manager. The modules, optimizers, and any other tensors created within
339
+ the context of MemTracker will be tracked by default. Any tensors or stateful objects such as modules, optimizers etc.
340
+ that need to be tracked but are created outside the MemTracker should be registered using the `track_external` method.
341
+ The `track_external` method should be called before the MemTracker is used. Any tensors created outside the ``MemTracker``
342
+ and not supplied to the `track_external` method will not be tracked by the ``MemTracker``.
343
+
344
+ Example usage:
345
+
346
+ .. code-block:: python
347
+
348
+ module = ...
349
+ optimizer = ...
350
+ inp = ...
351
+ mem_tracker = MemTracker()
352
+ mem_tracker.track_external(module, optimizer, inp)
353
+ with mem_tracker as mt:
354
+ loss = module(inp)
355
+ print("After Forward:")
356
+ mt.display_snapshot("current")
357
+ loss.backward()
358
+ optimizer.step()
359
+ optimizer.zero_grad()
360
+ mt.display_snapshot("peak")
361
+ mt.display_modulewise_snapshots(depth=3, units="MiB")
362
+
363
+ Known Limitations:
364
+ - The ``MemTracker`` does not track memory for tensors that bypass the ``TorchDispatchMode`` ex. under ``no_dispatch``.
365
+ - Resizing tensor storages directly by using non-Tensor methods other than using ``torch.Untyped_Storage.resize_``
366
+ is not tracked. File a Github issue if you have use-cases for this.
367
+ - If the tensors are not traceable or wrappable subclasses of ``torch.Tensor``, then the tracker does not know how to
368
+ track their storages. File a Github issue if you have use-cases for this.
369
+ - During AC in the backward pass there might be misattribution between activation and temp memory, but the peak memory
370
+ will be tracked accurately. This will be fixed in the next update by hooking intricately with ``torch.uitls.checkpoint``.
371
+ """
372
+
373
+ def __init__(self) -> None:
374
+ self.memory_tracking = WeakIdKeyDictionary()
375
+ self._curr_mem_snap: dict[torch.device, dict[str, int]] = {}
376
+ self._peak_mem: dict[torch.device, int] = {}
377
+ self._peak_mem_snap: dict[torch.device, dict[str, int]] = {}
378
+ self._param_to_grad_hook_handles = WeakIdKeyDictionary()
379
+ self._optimizer_hook_handles: tuple[RemovableHandle, RemovableHandle] | None = (
380
+ None
381
+ )
382
+ # Dictionary to store the ``_WeakRefInfo`` instances corresponding to each tensor's storage.
383
+ self._WINFO = WeakIdKeyDictionary()
384
+ self._mod_tracker = ModTracker()
385
+ # This is a general memory tracker which can be used with any ``_RefType`` subclass
386
+ self._ref_class: type[_RefType] = _MemRefType
387
+ # Flags to track if we are in the AC region or optimizer step region
388
+ self._in_opt: bool = False
389
+ self._in_ac: bool = False
390
+ # Weak references to the topmost AC module currently active
391
+ self._ac_mod: weakref.ref | None = None
392
+ self._orig_resize = torch.UntypedStorage.resize_
393
+ self._depth = 0
394
+
395
+ def _update_snap(
396
+ self,
397
+ u_type: _UpdateType,
398
+ winfo: _WeakRefInfo,
399
+ old_mem_consumed: int | None = None,
400
+ old_reftype: _RefType | None = None,
401
+ ) -> None:
402
+ # Initialize a flag to track if the total memory might drop to zero after updates.
403
+ maybe_zero = False
404
+ # Ensure the device entry exists in the current memory snapshot, initializing if necessary.
405
+ # pyrefly: ignore [no-matching-overload]
406
+ dev_snap = self._curr_mem_snap.setdefault(
407
+ winfo.device, dict.fromkeys(self._ref_class, 0)
408
+ )
409
+ dev_snap.setdefault(_TOTAL_KEY, 0)
410
+ # Handle different types of updates based on the update type (`u_type`).
411
+ if u_type == _UpdateType.ADD:
412
+ # Increase the memory consumed for the specific reference type and update the total.
413
+ dev_snap[winfo.reftype] += winfo.mem_consumed
414
+ dev_snap[_TOTAL_KEY] += winfo.mem_consumed
415
+ elif u_type == _UpdateType.DEL:
416
+ # Decrease the memory consumed for the specific reference type and reduce the total.
417
+ dev_snap[winfo.reftype] -= winfo.mem_consumed
418
+ dev_snap[_TOTAL_KEY] -= winfo.mem_consumed
419
+ maybe_zero = True
420
+ elif u_type == _UpdateType.REF:
421
+ if old_reftype is None:
422
+ raise AssertionError
423
+ # Adjust memory consumption between two reference types within the same device.
424
+ dev_snap[old_reftype] -= winfo.mem_consumed
425
+ dev_snap[winfo.reftype] += winfo.mem_consumed
426
+ elif u_type == _UpdateType.SIZE:
427
+ if old_mem_consumed is None:
428
+ raise AssertionError
429
+ # Adjust the memory consumed for a reference type due to a change in size.
430
+ change = winfo.mem_consumed - old_mem_consumed
431
+ dev_snap[winfo.reftype] += change
432
+ dev_snap[_TOTAL_KEY] += change
433
+ maybe_zero = True
434
+ else:
435
+ raise ValueError(f"Invalid update type: {u_type}")
436
+ # Check if the total memory for the device has dropped to zero.
437
+ if maybe_zero:
438
+ if self._curr_mem_snap[winfo.device][_TOTAL_KEY] == 0:
439
+ # Remove the device entry from the memory snapshot if the total memory is zero.
440
+ del self._curr_mem_snap[winfo.device]
441
+
442
+ def _update_and_maybe_create_winfos(
443
+ self,
444
+ t: torch.Tensor,
445
+ reftype: _RefType,
446
+ update_existing: bool = False,
447
+ ) -> set[_WeakRefInfo]:
448
+ sts = get_untyped_storages(t)
449
+ winfos = set()
450
+ for st in sts:
451
+ # Attempt to retrieve existing ``_WeakRefInfo`` and its weak reference from the tracking dictionary.
452
+ winfo, _ = self._WINFO.get(st, (None, None))
453
+ if winfo is not None:
454
+ # If ``_WeakRefInfo`` exists, check if the reference type needs to be updated.
455
+ old_reftype = winfo.reftype
456
+ if old_reftype != reftype:
457
+ # Update the reference type and apply changes via ``_update_snap``.
458
+ winfo.reftype = reftype
459
+ self._update_snap(_UpdateType.REF, winfo, old_reftype=old_reftype)
460
+ winfos.add(winfo)
461
+ elif update_existing:
462
+ # If no existing ``_WeakRefInfo`` is found and update_existing is True, raise an error.
463
+ raise KeyError("No existing winfo found")
464
+ else:
465
+ # If no existing _WeakRefInfo is found and update_existing is False, create a new ``_WeakRefInfo``.
466
+ winfo, w_st = _WeakRefInfo.create_winfo(
467
+ st, t.device, reftype, self._delete_callback
468
+ )
469
+ # Store the new ``_WeakRefInfo`` and its weak reference in the tracking dictionary.
470
+ self._WINFO[st] = (winfo, w_st)
471
+ # Update the snapshot for the newly added ``_WeakRefInfo``.
472
+ if winfo.mem_consumed > 0:
473
+ self._update_snap(_UpdateType.ADD, winfo)
474
+ winfos.add(winfo)
475
+ return winfos
476
+
477
+ def _delete_callback(self, winfo: _WeakRefInfo, w_st: weakref.ref) -> None:
478
+ # Callback to be called when the storage object corresponding to the ``_WeakRefInfo``
479
+ # instance is about to be finalized.
480
+ if winfo.mem_consumed > 0:
481
+ self._update_snap(_UpdateType.DEL, winfo)
482
+
483
+ def _track_resize(self) -> None:
484
+ # Need to monkey-patch this because ``torch.UntypedStorage.resize_`` is not captured
485
+ # by ``TorchDispatchMode``.
486
+ @wraps(self._orig_resize)
487
+ def resize_(st: torch.UntypedStorage, size: int) -> None:
488
+ self._orig_resize(st, size)
489
+ winfo, _ = self._WINFO.get(st, (None, None))
490
+ if winfo is not None and winfo.size != st.size():
491
+ old_mem_consumed = winfo.mem_consumed
492
+ winfo.update_mem_consumed(st)
493
+ self._update_snap(
494
+ _UpdateType.SIZE, winfo, old_mem_consumed=old_mem_consumed
495
+ )
496
+
497
+ torch.UntypedStorage.resize_ = resize_ # type: ignore[method-assign, assignment]
498
+
499
+ def _restore_resize(self) -> None:
500
+ torch.UntypedStorage.resize_ = self._orig_resize # type: ignore[method-assign]
501
+
502
+ def _update_peak_stats(self, peak_state: _State) -> None:
503
+ # We first capture the current memory snapshot of the current tracker state then,
504
+ # We step through each of the modules we have tracked so far in ``memory_tracking``
505
+ # and check if it is currently active by querying ``_mod_tracker.parents``
506
+ # If it is active, we update the per device peak memory usage for the module
507
+ # corresponding to the ``_State`` which can be ``PEAK_FW`` or ``PEAK_BW``.
508
+ curr_snap = self._curr_mem_snap
509
+
510
+ for mod_stats in self.memory_tracking.values():
511
+ if mod_stats.mod_fqn in self._mod_tracker.parents:
512
+ if peak_state in mod_stats.snapshots:
513
+ for dev, dev_snap in curr_snap.items():
514
+ if mod_stats.local_peak.get(dev, 0) < dev_snap[_TOTAL_KEY]:
515
+ mod_stats.local_peak[dev] = dev_snap[_TOTAL_KEY]
516
+ mod_stats.snapshots[peak_state][-1][dev] = deepcopy(
517
+ dev_snap
518
+ )
519
+
520
+ for dev, dev_snap in curr_snap.items():
521
+ if self._peak_mem.get(dev, 0) < dev_snap[_TOTAL_KEY]:
522
+ self._peak_mem[dev] = dev_snap[_TOTAL_KEY]
523
+ self._peak_mem_snap[dev] = deepcopy(dev_snap)
524
+
525
+ def _track(self, reftype: _RefType, t: torch.Tensor) -> None:
526
+ # Get the storages of the tensor and check if we have already tracked them.
527
+ # If yes, then check if the storage size has changed and update the current snapshot.
528
+ # Else create a new ``_WeakRefInfo`` instance and add it to the dictionary.
529
+ sts = get_untyped_storages(t)
530
+ for st in sts:
531
+ winfo, _ = self._WINFO.get(st, (None, None))
532
+ if winfo is not None:
533
+ if winfo.size != st.size():
534
+ old_mem_consumed = winfo.mem_consumed
535
+ winfo.update_mem_consumed(st)
536
+ self._update_snap(
537
+ _UpdateType.SIZE, winfo, old_mem_consumed=old_mem_consumed
538
+ )
539
+ return
540
+ else:
541
+ winfo, w_st = _WeakRefInfo.create_winfo(
542
+ st, t.device, reftype, self._delete_callback
543
+ )
544
+ self._WINFO[st] = (winfo, w_st)
545
+ # Update the current snapshot for the newly added ``_WeakRefInfo``.
546
+ if winfo.mem_consumed > 0:
547
+ self._update_snap(_UpdateType.ADD, winfo)
548
+
549
+ def get_tracker_snapshot(
550
+ self, type: str = "current"
551
+ ) -> dict[torch.device, dict[str, int]]:
552
+ """
553
+ Capture a snapshot of the memory usage breakdown per device, based on the specified type.
554
+
555
+ Args:
556
+ type (str): The type of snapshot to capture. Can be "current" for the current memory usage or "peak" for the
557
+ peak memory usage. Defaults to "current".
558
+ Returns:
559
+ Dict[torch.device, Dict[str, int]]: A dictionary where each key is a torch.device, and each value is another
560
+ dictionary. This inner dictionary has keys representing memory reference
561
+ types as defined in ``_MemRefType`` and values representing the amount of
562
+ memory consumed in bytes.
563
+ Raises:
564
+ ValueError: If an invalid type is specified.
565
+ """
566
+ if type == "current":
567
+ return deepcopy(self._curr_mem_snap)
568
+ elif type == "peak":
569
+ return deepcopy(self._peak_mem_snap)
570
+ else:
571
+ raise ValueError(f"Invalid type {type}")
572
+
573
+ def _track_module_params_and_buffers(
574
+ self, module: nn.Module, install_grad_hooks: bool = True
575
+ ) -> tuple[int, int]:
576
+ # Track the parameters and buffers of the module if not already tracked.
577
+ # If the parameters have gradients, track the gradients as well.
578
+ # If install_grad_hooks is True, install a gradient hook on the parameters
579
+ # to track the gradients, if it has not already been installed.
580
+ # Return the total memory consumed by the parameters and buffers.
581
+ def _grad_hook(grad: torch.Tensor) -> None:
582
+ self._update_and_maybe_create_winfos(
583
+ grad,
584
+ _MemRefType.GRAD,
585
+ )
586
+
587
+ param_memory = 0
588
+ for param in module.parameters():
589
+ winfos = self._update_and_maybe_create_winfos(
590
+ param,
591
+ _MemRefType.PARAM,
592
+ )
593
+ param_memory += sum(winfo.mem_consumed for winfo in winfos)
594
+ if param.grad is not None:
595
+ self._update_and_maybe_create_winfos(
596
+ param.grad,
597
+ _MemRefType.GRAD,
598
+ )
599
+ if (
600
+ self._param_to_grad_hook_handles.get(param, None) is None
601
+ and install_grad_hooks
602
+ ):
603
+ grad_hook_handle = param.register_hook(_grad_hook)
604
+ post_acc_grad_hook_handle = param.register_post_accumulate_grad_hook(
605
+ lambda p: (_grad_hook(p.grad))
606
+ )
607
+ self._param_to_grad_hook_handles[param] = (
608
+ grad_hook_handle,
609
+ post_acc_grad_hook_handle,
610
+ )
611
+ buffer_memory = 0
612
+ for buffer in module.buffers():
613
+ winfos = self._update_and_maybe_create_winfos(
614
+ buffer,
615
+ _MemRefType.BUFFER,
616
+ )
617
+ buffer_memory += sum(winfo.mem_consumed for winfo in winfos)
618
+ return (param_memory, buffer_memory)
619
+
620
+ def _track_inputs_or_outputs(self, args: Any) -> int:
621
+ # Calculate the memory consumed by the inputs or outputs of the module.
622
+ input_or_output_memory = 0
623
+
624
+ def add_inps_or_outs(t: torch.Tensor) -> None:
625
+ nonlocal input_or_output_memory
626
+ sts = get_untyped_storages(t)
627
+ for st in sts:
628
+ winfo, _ = self._WINFO.get(st, (None, None))
629
+ if winfo is not None:
630
+ input_or_output_memory += winfo.mem_consumed
631
+
632
+ tree_map_only(torch.Tensor, add_inps_or_outs, args)
633
+ return input_or_output_memory
634
+
635
+ def _pre_fw_hook(self, module: nn.Module, inputs: Any) -> None:
636
+ # This is installed as a pre-fwd user hook with ``ModTracker.`` Based on the following cases we
637
+ # set the state and capture the memory snapshot for the module.
638
+ # Case 1: If the module is not in the ``memory_tracking`` dictionary, we track the parameters, buffers,
639
+ # input and output memory of the module. Create a new ``_ModMemStats`` instance for the module
640
+ # and add it to the ``memory_tracking`` dictionary.
641
+ # Case 2: If the module is already in the ``memory_tracking`` dictionary and we are in backward, this means
642
+ # we are in the AC region. We check if this is the top most module in the AC region. If it is,
643
+ # we store a weak reference and set the flag ``_in_ac`` to True.
644
+ # Case 3: If the module is already in the ``memory_tracking`` dictionary and we are in forward, this means
645
+ # this module is called for the second time. If it is a root module, that means we are in the next
646
+ # iteration and we error out. If it is not a root module, that means it's a submodule that is being
647
+ # used multiple times in the same iteration, which we allow and track.
648
+ # For Case 1 and 3, we also initialize the ``local_peak`` and ``PEAK_FW`` snapshot for the module.
649
+ mod_name = self._mod_tracker.get_known_fqn(module)
650
+ if mod_name is None:
651
+ raise AssertionError
652
+ if module not in self.memory_tracking:
653
+ mod_stats = _ModMemStats(mod_name)
654
+ param_mem, buffer_mem = self._track_module_params_and_buffers(
655
+ module, install_grad_hooks=True
656
+ )
657
+ input_mem = self._track_inputs_or_outputs(inputs)
658
+ mod_stats.parameter_mem = param_mem
659
+ mod_stats.buffer_mem = buffer_mem
660
+ mod_stats.input_mem = input_mem
661
+ self.memory_tracking[module] = mod_stats
662
+ state = _ModState.PRE_FW
663
+
664
+ elif self._mod_tracker.is_bw:
665
+ mod_stats = self.memory_tracking[module]
666
+ state = _ModState.PRE_FW_AC
667
+ if self._ac_mod is None:
668
+ self._ac_mod = weakref.ref(module)
669
+ self._in_ac = True
670
+ else:
671
+ parents = set(self._mod_tracker.parents) - {mod_name}
672
+ if len(parents) == 1 and "Global" in parents:
673
+ raise NotImplementedError(
674
+ "MemTracker does not support memory tracking for multiple iterative calls."
675
+ " Either use ``reset_mod_stats`` to clear module memory stats for the previous iteration"
676
+ " or file a github issue if you need this feature."
677
+ )
678
+ mod_stats = self.memory_tracking[module]
679
+ state = _ModState.PRE_FW
680
+ input_mem = self._track_inputs_or_outputs(inputs)
681
+ mod_stats.mod_fqn = mod_name
682
+ mod_stats.input_mem = input_mem
683
+
684
+ mem_snapshot = self.get_tracker_snapshot()
685
+ if state == _ModState.PRE_FW:
686
+ mod_stats.local_peak = {
687
+ dev: dev_snap[_TOTAL_KEY] for dev, dev_snap in mem_snapshot.items()
688
+ }
689
+ mod_stats.snapshots.setdefault(_ModState.PEAK_FW, []).append(mem_snapshot)
690
+ mod_stats.snapshots.setdefault(state, []).append(deepcopy(mem_snapshot))
691
+
692
+ def _post_fw_hook(self, module: nn.Module, inputs: Any, outputs: Any) -> None:
693
+ # This is installed as a post-fwd user hook with ``ModTracker``. Based on the following cases we
694
+ # set the state and capture the memory snapshot for the module.
695
+ # Case 1: This is called in backward, which means we are in the AC region. If this is the top most module
696
+ # in the AC region, we set the flag ``_in_ac`` to False.
697
+ # Case 2: This is called in forward so we calculate the output memory
698
+ # of the module and update its mod_stats.
699
+ mod_stats = self.memory_tracking[module]
700
+ if self._mod_tracker.is_bw:
701
+ state = _ModState.POST_FW_AC
702
+ if self._ac_mod is not None and self._ac_mod() is module:
703
+ self._ac_mod = None
704
+ self._in_ac = False
705
+ else:
706
+ state = _ModState.POST_FW
707
+ output_mem = self._track_inputs_or_outputs(outputs)
708
+ mod_stats.output_mem = output_mem
709
+ mod_stats.snapshots.setdefault(state, []).append(self.get_tracker_snapshot())
710
+
711
+ def _pre_bw_hook(self, module: nn.Module, args: Any) -> None:
712
+ # This is installed as a pre-bwd user hook with ``ModTracker``. We set the state and capture the
713
+ # snapshot for the module. We also initialize the ``local_peak`` and ``PEAK_BW`` snapshot for it.
714
+ # If the module is None, we skip the hook.
715
+ # This can happen since this installed inside a multi-grad hook on the module's output tensors
716
+ # and the module itself may not be alive during backward.
717
+ if module is None:
718
+ warnings.warn("Module is None. Skipping PRE_BW hook.", stacklevel=2)
719
+ return
720
+ mod_stats = self.memory_tracking[module]
721
+ mem_snapshot = self.get_tracker_snapshot()
722
+ mod_stats.local_peak = {
723
+ dev: dev_snap[_TOTAL_KEY] for dev, dev_snap in mem_snapshot.items()
724
+ }
725
+ mod_stats.snapshots.setdefault(_ModState.PEAK_BW, []).append(mem_snapshot)
726
+ mod_stats.snapshots.setdefault(_ModState.PRE_BW, []).append(
727
+ deepcopy(mem_snapshot)
728
+ )
729
+
730
+ def _post_bw_hook(self, module: nn.Module, args: Any) -> None:
731
+ # This is installed as a post-bwd user hook with ``ModTracker``. We set the state and capture the
732
+ # snapshot for the module if it is not None.
733
+ # This can happen since this installed inside a multi-grad hook on the module's input tensors
734
+ # and the module itself may not be alive during backward.
735
+ if module is None:
736
+ warnings.warn("Module is None. Skipping POST_BW hook.", stacklevel=2)
737
+ return
738
+ mod_stats = self.memory_tracking[module]
739
+ mod_stats.snapshots.setdefault(_ModState.POST_BW, []).append(
740
+ self.get_tracker_snapshot()
741
+ )
742
+
743
+ def _track_optimizer_states(
744
+ self, reftype: _RefType, optimizer: optim.Optimizer
745
+ ) -> None:
746
+ for states in optimizer.state.values():
747
+ for val in states.values():
748
+ if isinstance(val, torch.Tensor):
749
+ self._update_and_maybe_create_winfos(
750
+ val,
751
+ reftype,
752
+ )
753
+
754
+ def _register_global_optimizer_hook(self) -> None:
755
+ # Register a hook on the optimizer step to track the optimizer states.
756
+ # The pre-hook is to set the flag ``_in_opt`` to True. The post-hook unsets the flag,
757
+ # and also tracks any optimizer states that are created during the optimizer step.
758
+ def _opt_step_pre_hook(
759
+ optimizer: optim.Optimizer, args: Any, kwargs: Any
760
+ ) -> None:
761
+ self._in_opt = True
762
+
763
+ def _opt_step_post_hook(
764
+ optimizer: optim.Optimizer, args: Any, kwargs: Any
765
+ ) -> None:
766
+ self._track_optimizer_states(_MemRefType.OPT, optimizer)
767
+ self._in_opt = False
768
+
769
+ self._optimizer_hook_handles = (
770
+ register_optimizer_step_pre_hook(_opt_step_pre_hook),
771
+ register_optimizer_step_post_hook(_opt_step_post_hook),
772
+ )
773
+
774
+ def _deregister_param_and_optimizer_hooks(self) -> None:
775
+ for (
776
+ grad_hook_handle,
777
+ post_acc_grad_hook_handle,
778
+ ) in self._param_to_grad_hook_handles.values():
779
+ grad_hook_handle.remove()
780
+ post_acc_grad_hook_handle.remove()
781
+ self._param_to_grad_hook_handles.clear()
782
+
783
+ if self._optimizer_hook_handles is not None:
784
+ for handle in self._optimizer_hook_handles:
785
+ handle.remove()
786
+ self._optimizer_hook_handles = None
787
+
788
+ def track_external(
789
+ self, *external: nn.Module | optim.Optimizer | torch.Tensor
790
+ ) -> None:
791
+ """
792
+ Track tensors and stateful objects like modules, optimizers etc. that are created outside the MemTracker.
793
+
794
+ This method should be called before the ``MemTracker`` is used. Any tensors that are not module parameters, buffers,
795
+ gradients activations, or optimizer states will be categorized as ``Other``. If you want them categorized with a
796
+ custom name, please file a GitHub issue. Any tensors created outside the MemTracker and not supplied to this
797
+ method will not be tracked by ``MemTracker``.
798
+
799
+ Args:
800
+ *external (Union[nn.Module, optim.Optimizer, torch.Tensor]): The external modules, optimizers, and
801
+ tensors to be tracked.
802
+ """
803
+ flat_external, _ = tree_flatten(external)
804
+ for obj in flat_external:
805
+ if isinstance(obj, torch.Tensor):
806
+ self._update_and_maybe_create_winfos(
807
+ obj,
808
+ _MemRefType.OTH,
809
+ )
810
+ elif isinstance(obj, torch.nn.Module):
811
+ self._track_module_params_and_buffers(obj, install_grad_hooks=False)
812
+ elif isinstance(obj, optim.Optimizer):
813
+ self._track_optimizer_states(_MemRefType.OPT, obj)
814
+ elif obj is None:
815
+ continue
816
+ else:
817
+ raise TypeError(
818
+ f"Object of type {type(obj)} is not supported for tracking. "
819
+ f"Only stateful objects like modules, optimizers, and tensors are supported."
820
+ )
821
+
822
+ def display_snapshot(
823
+ self, type: str = "current", units: str = "B", tabulate: bool = False
824
+ ) -> None:
825
+ """
826
+ Display the memory usage breakdown snapshot of the tracker based on the specified type and units.
827
+
828
+ Keyword args:
829
+ type (str): The type of snapshot to display. Can be "current" for the current memory usage or "peak" for the
830
+ peak memory usage. Defaults to "current".
831
+ units (str): The units to use for displaying memory usage. Defaults to "B". Supports ["B", "KiB", "MiB", "GiB"].
832
+ tabulate (bool): Whether to display the snapshot in a tabular format. Defaults to False.
833
+ """
834
+ snapshot = self.get_tracker_snapshot(type)
835
+ if tabulate:
836
+ _print_snapshot_tabular(snapshot, units)
837
+ else:
838
+ _print_snapshot(snapshot, units)
839
+
840
+ def display_modulewise_snapshots(
841
+ self, depth: int = 2, units: str = "B", tabulate: bool = False
842
+ ) -> None:
843
+ """
844
+ Print per device memory breakdown snapshot for each module called within MemTracker.
845
+
846
+ Snapshots are displayed for the states defined by ``_ModState``.
847
+ The module hierarchy is displayed up to the specified depth.
848
+
849
+ Keyword Args:
850
+ depth (int, optional): The depth of the module hierarchy to display. Defaults to 2.
851
+ units (str, optional): The units to use for memory tracking. Defaults to "B". Supports ["B", "KiB", "MiB", "GiB"].
852
+ tabulate (bool, optional): Whether to display the snapshot in a tabular format. Defaults to False.
853
+ """
854
+
855
+ def natural_sort_key(s: str) -> list[int | str]:
856
+ return [
857
+ int(text) if text.isdigit() else text.lower()
858
+ for text in re.split("([0-9]+)", s)
859
+ ]
860
+
861
+ for mod_stats in sorted(
862
+ self.memory_tracking.values(),
863
+ key=lambda m_stats: natural_sort_key(m_stats.mod_fqn),
864
+ ):
865
+ mod_fqn = mod_stats.mod_fqn
866
+ mod_depth = mod_fqn.count(".") + 1
867
+ if mod_depth > depth:
868
+ continue
869
+ print(f"Module: {mod_fqn}")
870
+ if tabulate:
871
+ _print_state_snapshots_tabular(mod_stats.snapshots, units)
872
+ else:
873
+ _print_state_snapshots(mod_stats.snapshots, units)
874
+
875
+ def reset_mod_stats(self) -> None:
876
+ """
877
+ Reset all the module memory stats. Clears ``memory_tracking`` dictionary.
878
+ """
879
+ self.memory_tracking.clear()
880
+
881
+ def __enter__(self) -> "MemTracker":
882
+ if self._depth == 0:
883
+ # None in eager, a FakeTensorMode instance in SAC. Used in
884
+ # __torch_dispatch__ to skip DTensor propagation ops.
885
+ self._fake_mode_on_entry = active_fake_mode()
886
+ self._register_global_optimizer_hook()
887
+ self._mod_tracker.register_user_hooks(
888
+ self._pre_fw_hook,
889
+ self._post_fw_hook,
890
+ self._pre_bw_hook,
891
+ self._post_bw_hook,
892
+ )
893
+ self._track_resize()
894
+ self._peak_mem_snap = self.get_tracker_snapshot()
895
+ self._peak_mem = {
896
+ dev: dev_snap[_TOTAL_KEY]
897
+ for dev, dev_snap in self._peak_mem_snap.items()
898
+ }
899
+ self._mod_tracker.__enter__()
900
+ super().__enter__()
901
+ self._depth += 1
902
+ return self
903
+
904
+ # pyrefly: ignore [bad-override]
905
+ def __exit__(self, *args: Any) -> None:
906
+ self._depth -= 1
907
+ if self._depth == 0:
908
+ self._deregister_param_and_optimizer_hooks()
909
+ self._mod_tracker.clear_user_hooks()
910
+ self._restore_resize()
911
+ self._mod_tracker.__exit__(*args)
912
+ super().__exit__(*args)
913
+
914
+ def __torch_dispatch__(self, func, types, args=(), kwargs=None): # type: ignore[no-untyped-def]
915
+ # When running this mode with DTensor, ordinarily all modes will
916
+ # run **before** subclasses get a chance to run.
917
+ # Returning NotImplemented here gives us a chance to let DTensor
918
+ # run and desugar into local tensor ops, before `MemTracker` sees them.
919
+ if any(t == DTensor for t in types):
920
+ return NotImplemented
921
+ if (
922
+ func is torch.ops._c10d_functional.wait_tensor.default
923
+ and active_fake_mode()
924
+ ):
925
+ # N.B: This is a hacky way to override the Meta IMPL of wait_tensor. The original impl returns
926
+ # a new tensor which does not happen in eager mode, when a wait_tensor is called.
927
+ # pyrefly: ignore [bad-index]
928
+ res = args[0]
929
+ else:
930
+ res = func(*args, **kwargs or {})
931
+ # DTensor sharding propagation may use a nested FakeTensorMode to
932
+ # compute output metadata (shapes, placements). The real memory
933
+ # usage comes from the local op on shard data, which
934
+ # runs outside DTensor's fake mode. We use identity comparison
935
+ # against the fake mode at tracker entry to skip propagation ops
936
+ # while still tracking local ops in both eager and SAC contexts.
937
+ # - Eager (entry mode is None): real ops have no fake mode
938
+ # (None is None), DTensor propagation ops are skipped (B is not None).
939
+ # - SAC (entry mode is A): SAC ops run under mode A (A is A),
940
+ # DTensor propagation ops are skipped (B is not A).
941
+ if active_fake_mode() is self._fake_mode_on_entry:
942
+ # If we are tracking an optimizer state, we use the optimizer reference type.
943
+ # If we are in backward region and not in AC region, we use the backward reference type.
944
+ # Else we use the forward reference type.
945
+ if self._in_opt:
946
+ reftype = _MemRefType.OPT
947
+ elif self._mod_tracker.is_bw and not self._in_ac:
948
+ reftype = _MemRefType.TEMP
949
+ else:
950
+ reftype = _MemRefType.ACT
951
+ tree_map_only(torch.Tensor, partial(self._track, reftype), res)
952
+ peak_state = (
953
+ _ModState.PEAK_BW if self._mod_tracker.is_bw else _ModState.PEAK_FW
954
+ )
955
+ self._update_peak_stats(peak_state)
956
+ return res
western_ai/usa/pytorch/torch/distributed/_tools/memory_tracker.py ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import operator
3
+ import pickle
4
+ from collections import defaultdict
5
+ from collections.abc import Callable, Sequence
6
+ from itertools import chain
7
+ from typing import Any, no_type_check, TYPE_CHECKING
8
+
9
+ import torch
10
+ import torch.nn as nn
11
+ from torch.utils._python_dispatch import TorchDispatchMode
12
+
13
+
14
+ if TYPE_CHECKING:
15
+ from torch.utils.hooks import RemovableHandle
16
+
17
+
18
+ BYTES_PER_MB = 1024 * 1024.0
19
+
20
+
21
+ class MemoryProfileDispatchMode(TorchDispatchMode):
22
+ """Run in ``TorchDispatchMode`` to get memory stats at operator level."""
23
+
24
+ def __init__(self, memory_tracker) -> None:
25
+ self.memory_tracker = memory_tracker
26
+
27
+ def __torch_dispatch__(self, func, types, args=..., kwargs=None):
28
+ rs = func(*args, **kwargs)
29
+ if func is torch.ops.aten.detach.default:
30
+ return rs
31
+ func_name: str = (
32
+ self.memory_tracker._cur_module_name
33
+ + "."
34
+ + func.__name__
35
+ + "_"
36
+ + str(self.memory_tracker._operator_names[func.__name__])
37
+ )
38
+ self.memory_tracker._operator_names[func.__name__] = (
39
+ self.memory_tracker._operator_names[func.__name__] + 1
40
+ )
41
+ self.memory_tracker._record_memory_stats(func_name)
42
+
43
+ return rs
44
+
45
+
46
+ class MemoryTracker:
47
+ """
48
+ Collect and plot the memory stats at operator level.
49
+
50
+ Includes ``memories_allocated``, ``memories_active`` and ``memories_reserved``.
51
+ It also prints a summary for the top 20 operators that generate the most memories.
52
+
53
+ Example usage:
54
+
55
+ >>> # xdoctest: +SKIP(failing)
56
+ >>> net.cuda()
57
+ >>> input = input.cuda()
58
+
59
+ >>> mem_tracker = MemoryTracker()
60
+ >>> mem_tracker.start_monitor(net)
61
+
62
+ >>> net.zero_grad(True)
63
+ >>> loss = net(input)
64
+ >>> if isinstance(loss, dict):
65
+ >>> loss = loss['out']
66
+ >>> loss.sum().backward()
67
+ >>> net.zero_grad(set_to_none=True)
68
+
69
+ >>> mem_tracker.stop()
70
+ >>> mem_tracker.summary()
71
+ >>> mem_tracker.show_traces()
72
+ """
73
+
74
+ def __init__(self) -> None:
75
+ torch._C._log_api_usage_once("torch.distributed.memory_tracker")
76
+ self._hooks: list[RemovableHandle] = []
77
+ self._operator_names: dict[str, int] = defaultdict(int)
78
+ self.memories_allocated: dict[int, dict[str, float]] = defaultdict()
79
+ self.memories_active: dict[int, dict[str, float]] = defaultdict()
80
+ self.memories_reserved: dict[int, dict[str, float]] = defaultdict()
81
+ self._markers: dict[str, int] = defaultdict(int)
82
+ self._cur_module_name: str = ""
83
+ self._op_index: int = 0
84
+ self._num_alloc_retries: int = 0
85
+ self._device_module = torch.get_device_module()
86
+
87
+ @no_type_check
88
+ def start_monitor(self, root_module: nn.Module) -> None:
89
+ """
90
+ Register module hooks and entering ``MemoryProfileDispatchMode``.
91
+
92
+ This enables operator level memory stats can be tracked during module runtime.
93
+ """
94
+ self._clear_state()
95
+ root_module.__setattr__("_memory_tracker_is_root", True)
96
+ for name, m in root_module.named_modules():
97
+ if m is not root_module:
98
+ m.__setattr__("_memory_tracker_is_root", False)
99
+ # fused_proxy_group does not support hooks
100
+ if ".fused_proxy_grouped_embedding_bag" in name:
101
+ continue
102
+ # hook ordering with other hooks added by users is not managed, so
103
+ # the memory stats tracked here may not completely accurate.
104
+ h1 = m.register_forward_pre_hook(self._create_pre_forward_hook(name))
105
+ h2 = m.register_forward_hook(self._create_post_forward_hook(name))
106
+ # it does not work well with jagged tensor somehow, the root cause is not
107
+ # clear and remove it for now as it does not really capture important info.
108
+ # h3 = m.register_backward_hook(self._create_backward_hook(name))
109
+ self._hooks.extend([h1, h2])
110
+ self._device_module.empty_cache()
111
+ if getattr(self, "profile_mode", None) is not None:
112
+ raise AssertionError
113
+ self.profile_mode = MemoryProfileDispatchMode(self)
114
+ self.profile_mode.__enter__()
115
+
116
+ @no_type_check
117
+ def stop(self) -> None:
118
+ """
119
+ Remove module hooks and exit ``MemoryProfileDispatchMode`` to stop tracking memory stats at operator level.
120
+
121
+ Get some aggregated stats when the memory_tracker() is enabled, like ``num_alloc_retries``.
122
+ """
123
+ self._num_alloc_retries = self._device_module.memory_stats().get(
124
+ "num_alloc_retries", 0
125
+ )
126
+
127
+ for h in self._hooks:
128
+ h.remove()
129
+ self._hooks.clear()
130
+ if getattr(self, "profile_mode", None) is None:
131
+ raise AssertionError
132
+ self.profile_mode.__exit__(None, None, None)
133
+ self.profile_mode = None
134
+
135
+ @no_type_check
136
+ def summary(self, top: int = 20) -> None:
137
+ """
138
+ Print out the top operators that generate the most memories.
139
+
140
+ The number of the top operators can be configured.
141
+ """
142
+ op_diff: dict[str, float] = defaultdict(float)
143
+ op_name, previous_allocated_memory = self.memories_allocated[0]
144
+ for i in range(1, self._op_index):
145
+ op_name, current_allocated_memory = self.memories_allocated[i]
146
+ op_diff[op_name] = current_allocated_memory - previous_allocated_memory
147
+ previous_allocated_memory = current_allocated_memory
148
+
149
+ print("------------------------------------------------")
150
+ print(f"The number of alloc retries are: {self._num_alloc_retries}")
151
+ print(f"Top {top} ops that generates memory are:")
152
+ for k, v in sorted(op_diff.items(), key=operator.itemgetter(1), reverse=True)[
153
+ :top
154
+ ]:
155
+ print(f"{k}: {v}MB")
156
+ print("------------------------------------------------")
157
+
158
+ @no_type_check
159
+ def show_traces(self, path: str = "") -> None:
160
+ import matplotlib.pyplot as plt
161
+
162
+ def _plot_figure(x, y_values, labels):
163
+ min_val = min(chain.from_iterable(y_values)) * 0.999
164
+ max_val = max(chain.from_iterable(y_values)) * 1.001
165
+ plt.figure()
166
+ for y, label in zip(y_values, labels):
167
+ plt.plot(x, y, label=label)
168
+ plt.xlabel("# Operator Calls")
169
+ plt.ylabel("Memory (MB)")
170
+ plt.legend()
171
+ for marker_name, marker in self._markers.items():
172
+ if marker_name == "fw_bw_boundary":
173
+ plt.plot(
174
+ [marker, marker],
175
+ [min_val, max_val],
176
+ "r",
177
+ lw=2,
178
+ label=marker_name,
179
+ )
180
+ else:
181
+ plt.plot(
182
+ [marker, marker],
183
+ [min_val, max_val],
184
+ "k-",
185
+ lw=2,
186
+ label=marker_name,
187
+ )
188
+
189
+ if path != "":
190
+ self.load(path)
191
+
192
+ y_1 = [gb for (name, gb) in self.memories_allocated.values()]
193
+ y_2 = [gb for (name, gb) in self.memories_active.values()]
194
+ y_3 = [gb for (name, gb) in self.memories_reserved.values()]
195
+ x = list(range(len(y_1)))
196
+ # Split figures when there is big difference between
197
+ # "reserved_memory" and "allocated_memory" or "active_memory".
198
+ _plot_figure(
199
+ x,
200
+ [list(y_1), list(y_2), list(y_3)],
201
+ ["allocated_memory", "active_memory", "reserved_memory"],
202
+ )
203
+ _plot_figure(x, [list(y_1)], ["allocated_memory"])
204
+ _plot_figure(x, [list(y_2)], ["active_memory"])
205
+ _plot_figure(x, [list(y_3)], ["reserved_memory"])
206
+
207
+ def save_stats(self, path: str) -> None:
208
+ """Save the stats using pickle during runtime if users want to plot the traces in other places like notebook."""
209
+ stats = {
210
+ "memories_allocated": self.memories_allocated,
211
+ "memories_active": self.memories_active,
212
+ "memories_reserved": self.memories_reserved,
213
+ "markers": self._markers,
214
+ "num_alloc_retries": self._num_alloc_retries,
215
+ }
216
+
217
+ with open(path, "wb") as f:
218
+ pickle.dump(stats, f, pickle.HIGHEST_PROTOCOL)
219
+
220
+ def load(self, path: str) -> None:
221
+ """Load the pickled memory stats to plot the traces or print the summary."""
222
+ with open(path, "rb") as f:
223
+ stats = pickle.load(f)
224
+
225
+ self.memories_allocated = stats["memories_allocated"]
226
+ self.memories_active = stats["memories_active"]
227
+ self.memories_reserved = stats["memories_reserved"]
228
+ self._markers = stats["markers"]
229
+ self._num_alloc_retries = stats["num_alloc_retries"]
230
+
231
+ def _create_pre_forward_hook(self, name: str) -> Callable:
232
+ """Prefix operator name with current module and 'forward', and insert 'fw_start' marker at forward pass start."""
233
+
234
+ def _pre_forward_hook(module: nn.Module, inputs: Any) -> None:
235
+ self._cur_module_name = f"{name}.forward"
236
+ if (
237
+ hasattr(module, "_memory_tracker_is_root")
238
+ and module._memory_tracker_is_root
239
+ ):
240
+ self._add_marker("fw_start")
241
+
242
+ return _pre_forward_hook
243
+
244
+ def _create_post_forward_hook(self, name: str) -> Callable:
245
+ """Insert the marker 'fw_bw_boundary' at the boundary of forward and backward pass."""
246
+
247
+ def _post_forward_hook(
248
+ module: nn.Module,
249
+ inputs: Sequence[torch.Tensor],
250
+ outputs: Sequence[torch.Tensor],
251
+ ) -> None:
252
+ if (
253
+ hasattr(module, "_memory_tracker_is_root")
254
+ and module._memory_tracker_is_root
255
+ ):
256
+ self._add_marker("fw_bw_boundary")
257
+
258
+ return _post_forward_hook
259
+
260
+ def _create_backward_hook(self, name: str) -> Callable:
261
+ """Insert the current module name with backward prefix for the operator name."""
262
+
263
+ def _backward_hook(
264
+ module: nn.Module, grad_input: torch.Tensor, grad_output: torch.Tensor
265
+ ) -> None:
266
+ self._cur_module_name = f"{name}.backward"
267
+
268
+ return _backward_hook
269
+
270
+ @no_type_check
271
+ def _record_memory_stats(self, fn_name: str) -> None:
272
+ """
273
+ Record current memory allocated, current memory active and current memory reserved.
274
+
275
+ The memory stats dict is indexed with ``self._op_index``.
276
+ """
277
+ memory_allocated: float = self._device_module.memory_allocated() / BYTES_PER_MB
278
+ memory_reserved: float = self._device_module.memory_reserved() / BYTES_PER_MB
279
+ memory_active: float = (
280
+ self._device_module.memory_stats().get("active_bytes.all.current", 0)
281
+ / BYTES_PER_MB
282
+ )
283
+ self.memories_allocated[self._op_index] = (fn_name, memory_allocated)
284
+ self.memories_reserved[self._op_index] = (fn_name, memory_reserved)
285
+ self.memories_active[self._op_index] = (fn_name, memory_active)
286
+ self._op_index += 1
287
+
288
+ def _add_marker(self, marker_name: str) -> None:
289
+ """Set the marker's x-axis value."""
290
+ marker_val = len(self.memories_allocated.values())
291
+ self._markers[marker_name] = marker_val
292
+
293
+ def _clear_state(self) -> None:
294
+ """Clear states when start_monitor() is called."""
295
+ self._operator_names.clear()
296
+ self.memories_allocated.clear()
297
+ self.memories_active.clear()
298
+ self.memories_reserved.clear()
299
+ self._markers.clear()
300
+ self._cur_module_name = ""
301
+ self._op_index = 0
302
+ self._num_alloc_retries = 0
western_ai/usa/pytorch/torch/distributed/_tools/mod_tracker.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import warnings
3
+ import weakref
4
+ from collections.abc import Callable
5
+
6
+ import torch
7
+ from torch.autograd.graph import register_multi_grad_hook
8
+ from torch.nn.modules.module import (
9
+ register_module_forward_hook,
10
+ register_module_forward_pre_hook,
11
+ )
12
+ from torch.utils._pytree import tree_flatten
13
+
14
+
15
+ __all__ = ["ModTracker"]
16
+
17
+
18
+ class ModTracker:
19
+ """
20
+ ``ModTracker`` is a context manager that tracks the nn.Module hierarchy during execution
21
+ so that other system can query which Module is currently being executed (or its backward is being
22
+ executed).
23
+
24
+ You can access the ``parents`` attribute on this context manager to get the set of all the
25
+ Modules currently being executed via their fqn (fully qualified name, also used as the key within
26
+ the state_dict).
27
+ You can access the ``is_bw`` attribute to know if you are currently running in backward or not.
28
+
29
+ Note that ``parents`` is never empty and always contains the "Global" key. The ``is_bw`` flag
30
+ will remain ``True`` after the forward until another Module is executed. If you need it to be
31
+ more accurate, please submit an issue requesting this. Adding a map from fqn to the module instance
32
+ is possible but not done yet, please submit an issue requesting this if you need it.
33
+
34
+ Example usage
35
+
36
+ .. code-block:: python
37
+
38
+ mod = torch.nn.Linear(2, 2)
39
+
40
+ with ModTracker() as tracker:
41
+ # Access anything during the forward pass
42
+ def my_linear(m1, m2, bias):
43
+ print(f"Current modules: {tracker.parents}")
44
+ return torch.mm(m1, m2.t()) + bias
45
+
46
+ torch.nn.functional.linear = my_linear
47
+
48
+ mod(torch.rand(2, 2))
49
+
50
+ """
51
+
52
+ parents: set[str]
53
+ """
54
+ A Set containing the fqn for each module currently running their forward
55
+ """
56
+
57
+ def __init__(self):
58
+ self.parents = {"Global"}
59
+ self._active_module_cnt = {}
60
+ self._known_modules: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()
61
+ self._seen_modules: weakref.WeakSet = weakref.WeakSet()
62
+ self._has_callback = False
63
+ self._post_bw_callbacks_to_enqueue: list[Callable] = []
64
+ self._user_pre_fw_hook = None
65
+ self._user_post_fw_hook = None
66
+ self._user_pre_bw_hook = None
67
+ self._user_post_bw_hook = None
68
+
69
+ def _maybe_set_engine_callback(self):
70
+ # This assumes no concurrent calls to backward
71
+ if self._has_callback:
72
+ return
73
+
74
+ for post_bw_callback in reversed(self._post_bw_callbacks_to_enqueue):
75
+ torch.autograd.Variable._execution_engine.queue_callback(post_bw_callback)
76
+ self._post_bw_callbacks_to_enqueue.clear()
77
+
78
+ def callback():
79
+ self.parents = {"Global"}
80
+ self._has_callback = False
81
+
82
+ torch.autograd.Variable._execution_engine.queue_callback(callback)
83
+ self._has_callback = True
84
+
85
+ @property
86
+ def is_bw(self):
87
+ """
88
+ A boolean marking if this is currently running during the backward pass or not
89
+ """
90
+ return torch._C._current_graph_task_id() != -1
91
+
92
+ def get_known_fqn(self, mod):
93
+ """
94
+ Return the fqn for the given module if it is known to the ``ModTracker``, otherwise ``None``.
95
+ """
96
+ return self._known_modules.get(mod, None)
97
+
98
+ def register_user_hooks(
99
+ self,
100
+ pre_fw_hook: Callable | None = None,
101
+ post_fw_hook: Callable | None = None,
102
+ pre_bw_hook: Callable | None = None,
103
+ post_bw_hook: Callable | None = None,
104
+ ):
105
+ """
106
+ Registers user-specified hooks to be called before/after the forward/backward pass for each
107
+ module tracked by the ``ModTracker``. One or more can be ``None``.
108
+ Args:
109
+ pre_fw_hook (Callable, optional): A hook to be called before the forward pass for the
110
+ module. It should have the following signature:
111
+ pre_fw_hook (module, input) -> None
112
+ post_fw_hook (Callable, optional): A hook to be called after the forward pass for the
113
+ module. It should have the following signature:
114
+ post_fw_hook (module, input, output) -> None
115
+ pre_bw_hook (Callable, optional): A multi-grad hook to be called on all the outputs of
116
+ the module that require gradients. It should have the following signature:
117
+ pre_bw_hook (module, grad_output) -> None
118
+ post_bw_hook (Callable, optional): A multi-grad hook to be called on all the inputs of
119
+ the module that require gradients. It should have the following signature:
120
+ post_bw_hook (module, grad_input) -> None
121
+ Raises:
122
+ AssertionError: If a new hook is provided when one is already registered.
123
+ Note:
124
+ If the module is not alive during the backward pass, the pre_bw_hook and post_bw_hook will
125
+ will receive None as the module argument.
126
+ The module fqn will be present in the ``parents`` attribute when each of the hooks is called.
127
+ Hooks are intended to be used as markers only not to modify the inputs/outputs.
128
+ """
129
+
130
+ def set_hook(hook, user_hook, hook_name):
131
+ if hook is not None and user_hook is not None:
132
+ raise AssertionError(
133
+ f"Only one {hook_name} can be registered at a time"
134
+ f" Clear the existing hook by calling ``clear_user_hooks`` before registering a new one"
135
+ )
136
+ return hook
137
+
138
+ self._user_pre_fw_hook = set_hook(
139
+ pre_fw_hook, self._user_pre_fw_hook, "pre_fw_hook"
140
+ )
141
+ self._user_post_fw_hook = set_hook(
142
+ post_fw_hook, self._user_post_fw_hook, "post_fw_hook"
143
+ )
144
+ self._user_pre_bw_hook = set_hook(
145
+ pre_bw_hook, self._user_pre_bw_hook, "pre_bw_hook"
146
+ )
147
+ self._user_post_bw_hook = set_hook(
148
+ post_bw_hook, self._user_post_bw_hook, "post_bw_hook"
149
+ )
150
+
151
+ def clear_user_hooks(self):
152
+ """
153
+ Clears the user specified hooks registered with ``register_user_hooks``
154
+ """
155
+ self._user_pre_fw_hook = None
156
+ self._user_post_fw_hook = None
157
+ self._user_pre_bw_hook = None
158
+ self._user_post_bw_hook = None
159
+
160
+ def _get_mod_name(self, mod):
161
+ if mod not in self._known_modules:
162
+ self._known_modules[mod] = type(mod).__name__
163
+ mod_name = self._known_modules[mod]
164
+ if mod not in self._seen_modules:
165
+ for name, submod in mod.named_children():
166
+ self._known_modules[submod] = f"{mod_name}.{name}"
167
+ self._get_mod_name(submod)
168
+ self._seen_modules.add(mod)
169
+ return mod_name
170
+
171
+ def _get_append_fn(self, w_mod, name, is_bw):
172
+ def fn(*args):
173
+ if is_bw:
174
+ self._maybe_set_engine_callback()
175
+ if name in self.parents and not self.is_bw:
176
+
177
+ def custom_formatwarning(msg, category, filename, lineno, line=None):
178
+ return f"{filename}:{lineno}: {category.__name__}: {msg} \n"
179
+
180
+ # pyrefly: ignore [bad-assignment]
181
+ warnings.formatwarning = custom_formatwarning
182
+ warnings.warn(
183
+ "The module hierarchy tracking maybe be messed up."
184
+ " Please file a bug to PyTorch, if it is the case.",
185
+ stacklevel=2,
186
+ )
187
+ if name not in self.parents:
188
+ self._active_module_cnt[name] = 1
189
+ self.parents.add(name)
190
+ else:
191
+ self._active_module_cnt[name] += 1
192
+
193
+ if self._user_pre_bw_hook is not None and is_bw:
194
+ self._user_pre_bw_hook(w_mod(), args)
195
+
196
+ return fn
197
+
198
+ def _get_pop_fn(self, w_mod, name, is_bw):
199
+ def fn(*args):
200
+ if self._user_post_bw_hook is not None and is_bw:
201
+ self._user_post_bw_hook(w_mod(), args)
202
+ if name in self.parents:
203
+ self._active_module_cnt[name] -= 1
204
+ if self._active_module_cnt[name] == 0:
205
+ self.parents.remove(name)
206
+ elif not self.is_bw:
207
+ # Due to some input/output not requiring gradients, we cannot enforce
208
+ # proper nesting in backward
209
+ raise RuntimeError(
210
+ "The Module hierarchy tracking is wrong. Report a bug to PyTorch"
211
+ )
212
+
213
+ return fn
214
+
215
+ def _fw_pre_hook(self, mod, input):
216
+ if torch._dynamo.eval_frame._is_in_optimized_module():
217
+ return
218
+
219
+ name = self._get_mod_name(mod)
220
+ w_mod = weakref.ref(mod)
221
+ self._get_append_fn(w_mod, name, False)()
222
+ if self._user_pre_fw_hook is not None:
223
+ self._user_pre_fw_hook(mod, input)
224
+ args, _ = tree_flatten(input)
225
+ tensors = [a for a in args if isinstance(a, torch.Tensor) and a.requires_grad]
226
+ if not self.is_bw:
227
+ if tensors:
228
+ register_multi_grad_hook(tensors, self._get_pop_fn(w_mod, name, True))
229
+ else:
230
+ self._post_bw_callbacks_to_enqueue.append(
231
+ self._get_pop_fn(w_mod, name, True)
232
+ )
233
+
234
+ def _fw_post_hook(self, mod, input, output):
235
+ if torch._dynamo.eval_frame._is_in_optimized_module():
236
+ return
237
+
238
+ name = self._get_mod_name(mod)
239
+ w_mod = weakref.ref(mod)
240
+ if self._user_post_fw_hook is not None:
241
+ self._user_post_fw_hook(mod, input, output)
242
+ self._get_pop_fn(w_mod, name, False)()
243
+ args, _ = tree_flatten(output)
244
+ tensors = [a for a in args if isinstance(a, torch.Tensor) and a.requires_grad]
245
+ if not self.is_bw and tensors:
246
+ register_multi_grad_hook(
247
+ tensors, self._get_append_fn(w_mod, name, True), mode="any"
248
+ )
249
+
250
+ def __enter__(self):
251
+ self._fw_pre_handle = register_module_forward_pre_hook(self._fw_pre_hook)
252
+ self._fw_post_handle = register_module_forward_hook(
253
+ self._fw_post_hook, always_call=True
254
+ )
255
+ return self
256
+
257
+ def __exit__(self, *args):
258
+ self._fw_pre_handle.remove()
259
+ self._fw_post_handle.remove()
western_ai/usa/pytorch/torch/distributed/_tools/runtime_estimator.py ADDED
@@ -0,0 +1,401 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Owner(s): ["module: unknown"]
2
+ from collections import defaultdict
3
+ from typing import Any, TYPE_CHECKING
4
+ from typing_extensions import Self
5
+
6
+ import torch
7
+ import torch.utils._pytree as pytree
8
+ from torch._guards import active_fake_mode
9
+ from torch._subclasses.fake_tensor import FakeTensorMode
10
+ from torch.distributed._tools.mod_tracker import ModTracker
11
+ from torch.utils._mode_utils import no_dispatch
12
+ from torch.utils._python_dispatch import TorchDispatchMode
13
+ from torch.utils._runtime_estimation import (
14
+ _FLOAT_TYPES,
15
+ _IGNORE_OPS,
16
+ _VIEW_OPS,
17
+ get_compute_time,
18
+ get_transfer_time,
19
+ )
20
+
21
+
22
+ if TYPE_CHECKING:
23
+ from collections.abc import Callable
24
+
25
+ __all__ = ["RuntimeEstimator"]
26
+
27
+
28
+ class RuntimeEstimator(TorchDispatchMode):
29
+ """
30
+ Estimates the GPU runtime in milliseconds using various estimation methods under the ``FakeTensorMode``.
31
+
32
+ This class provides a ``TorchDispatchMode`` based context manager that can be used to estimate the eager
33
+ runtime of PyTorch functions. It supports two estimation modes, benchmarking (`operator-level-benchmark`) and
34
+ roofline cost modeling (`operator-level-cost-model`).
35
+ For modules executed under this context manager, it aggregates the forward and backward operation runtimes
36
+ and also records their execution orders.
37
+
38
+ Attributes:
39
+ mod_runtimes (Dict[str, Dict[str, float]]): A dictionary of module runtimes. The key to the outer dictionary
40
+ is the fully qualified name (FQN) of the module. For each module the forward and backward runtimes of the
41
+ operations are aggregated in the inner dictionary keyed by 'fw' and 'bw'.
42
+ mod_fw_pre_order (List[str]): List of module FQNs in pre-forward execution order.
43
+ mod_bw_pre_order (List[str]): List of module FQNs in pre-backward execution order.
44
+ mod_fw_post_order (List[str]): List of module FQNs in post-forward execution order.
45
+ mod_bw_post_order (List[str]): List of module FQNs in post-backward execution order.
46
+ total_runtime (float): The total estimated runtime in milliseconds.
47
+
48
+ Note:
49
+ 1) The benchmarking estimate mode will execute kernels on GPU and assumes that every operation can run in
50
+ isolation without causing an OOM error. It is also designed to be used only under ``FakeTensorMode``.
51
+ 2) Currently wrapper tensor sub-classes such as ``DTensor`` won't produce correct estimates. We plan to support
52
+ them in future PRs.
53
+ 3) We only estimate the compute time, if your code has communication, it will not be considered. Again, we will
54
+ support this in future PRs.
55
+
56
+ Example usage:
57
+
58
+ .. code-block:: python
59
+
60
+ runtime_estimator = RuntimeEstimator()
61
+ with FakeTensorMode():
62
+ module = ...
63
+ optimizer = ...
64
+ inp = ...
65
+ with runtime_estimator(estimate_mode_type="operator-level-cost-model"):
66
+ loss = module(inp)
67
+ loss.backward()
68
+ optimizer.step()
69
+ optimizer.zero_grad()
70
+ runtime_estimator.display_modulewise_stats()
71
+ """
72
+
73
+ _no_fallback_kernel: set[torch._ops._OpNamespace] = set()
74
+ fake_mode: FakeTensorMode
75
+
76
+ def __init__(self) -> None:
77
+ super().__init__()
78
+ self._estimate: Callable
79
+ self._estimate_mode_type: str
80
+ self._mod_tracker = ModTracker()
81
+ self.mod_runtimes: dict[str, dict[str, float]] = defaultdict(
82
+ lambda: defaultdict(lambda: 0.0)
83
+ )
84
+ self.mod_fw_pre_order: list[str] = []
85
+ self.mod_bw_pre_order: list[str] = []
86
+ self.mod_fw_post_order: list[str] = []
87
+ self.mod_bw_post_order: list[str] = []
88
+ self.total_runtime: float = 0.0
89
+
90
+ # Adapted from: https://github.com/pytorch/pytorch/blob/9b902b3ee3bd608a19543362b66bf06c373dd374/torch/_subclasses/fake_tensor.py#L1969
91
+ # NB: returns fake tensors
92
+ @classmethod
93
+ def _maybe_run_and_benchmark_fallback_kernel( # type: ignore[no-untyped-def]
94
+ cls,
95
+ func,
96
+ args,
97
+ kwargs,
98
+ orig_not_implemented_exception,
99
+ ):
100
+ """
101
+ Runs and benchmarks a fallback kernel for a given function.
102
+
103
+ Args:
104
+ func (Callable): The function to benchmark.
105
+ args (Tuple): The arguments to pass to the function.
106
+ kwargs (Dict[str, Any]): The keyword arguments to pass to the function.
107
+ orig_not_implemented_exception (Exception): The original exception to raise if the fallback kernel
108
+ is not implemented.
109
+
110
+ Returns:
111
+ Tuple[Any, float]: A tuple containing the result of the function and
112
+ the mean operation time in milliseconds.
113
+ """
114
+ # these should all be supported, just to be safe
115
+ # avoid fallback for operators which inplace modify metadata
116
+ # because the input fake tensors would be umodified
117
+ if torch.Tag.inplace_view in func.tags: # type: ignore[attr-defined]
118
+ raise orig_not_implemented_exception
119
+
120
+ inp_impls = {}
121
+ flat_args, args_spec = pytree.tree_flatten((args, kwargs))
122
+ # Don't use in_kernel_invocation_manager(fake_mode) as we want to do
123
+ # REAL compute (not with meta device)
124
+ with no_dispatch():
125
+
126
+ def to_real_tensor(e): # type: ignore[no-untyped-def]
127
+ if cls.fake_mode.is_our_fake(e):
128
+ if e.dtype in _FLOAT_TYPES:
129
+ out = torch.rand_like(e, device=e.fake_device)
130
+ else:
131
+ out = torch.ones_like(e, device=e.fake_device)
132
+ if e.is_sparse:
133
+ out._coalesced_(e.is_coalesced())
134
+ inp_impls[id(out)] = e
135
+ return out
136
+ return e
137
+
138
+ flat_args = [to_real_tensor(a) for a in flat_args]
139
+ args, kwargs = pytree.tree_unflatten(flat_args, args_spec)
140
+ r = func(*args, **kwargs)
141
+ warmup_iters, actual_iters = 2, 3
142
+ for _ in range(warmup_iters):
143
+ func(*args, **kwargs)
144
+ start_event = torch.Event(enable_timing=True)
145
+ end_event = torch.Event(enable_timing=True)
146
+ start_event.record(torch.accelerator.current_stream())
147
+ for _ in range(actual_iters):
148
+ func(*args, **kwargs)
149
+ end_event.record(torch.accelerator.current_stream())
150
+ torch.accelerator.synchronize()
151
+ cuda_time = start_event.elapsed_time(end_event)
152
+ mean_op_time = cuda_time / actual_iters
153
+
154
+ storages = set()
155
+
156
+ for e in flat_args:
157
+ if isinstance(e, torch.Tensor):
158
+ if not e.is_sparse:
159
+ storages.add(e._typed_storage()._cdata)
160
+
161
+ # TODO: also check metadata change on inputs
162
+ # proper aliasing/metadata relationship between outputs and inputs will
163
+ # not be set up, bc of conversion to device, unless we can reuse an
164
+ # input impl
165
+
166
+ def map_out(e): # type: ignore[no-untyped-def]
167
+ if id(e) not in inp_impls and (
168
+ isinstance(e, torch.Tensor)
169
+ and not e.is_sparse
170
+ and e._typed_storage()._cdata in storages
171
+ ):
172
+ raise orig_not_implemented_exception
173
+
174
+ if isinstance(e, torch.Tensor):
175
+ if id(e) in inp_impls:
176
+ return inp_impls[id(e)]
177
+ else:
178
+ return cls.fake_mode.fake_tensor_converter.from_real_tensor(
179
+ cls.fake_mode, e
180
+ )
181
+ else:
182
+ return e
183
+
184
+ return (pytree.tree_map(map_out, r), mean_op_time)
185
+
186
+ @classmethod
187
+ def _benchmark_estimate(cls, func, args, kwargs) -> tuple[Any, float]: # type: ignore[no-untyped-def]
188
+ """
189
+ Estimates the runtime of a function using benchmarking.
190
+
191
+ Args:
192
+ func: The function to estimate.
193
+ args: The arguments to pass to the function.
194
+ kwargs: The keyword arguments to pass to the function.
195
+ res: The result of the function.
196
+
197
+ Returns:
198
+ Tuple[Any, float]: A tuple containing the result of the function and
199
+ the mean operation time in milliseconds.
200
+ """
201
+ if not isinstance(cls.fake_mode, FakeTensorMode):
202
+ raise AssertionError(
203
+ "Initialize/Assign FakeTensorMode before using this function"
204
+ )
205
+ mean_op_time = 0.0
206
+ if func._overloadpacket not in _VIEW_OPS:
207
+ try:
208
+ res, mean_op_time = cls._maybe_run_and_benchmark_fallback_kernel(
209
+ func,
210
+ args,
211
+ kwargs,
212
+ NotImplementedError,
213
+ )
214
+ return (res, mean_op_time)
215
+ except NotImplementedError:
216
+ cls._no_fallback_kernel.add(func._overloadpacket)
217
+ res = func(*args, **kwargs or {})
218
+ return (res, mean_op_time)
219
+
220
+ # Adapted from: https://github.com/pytorch/pytorch/blob/9b902b3ee3bd608a19543362b66bf06c373dd374/torch/_inductor/scheduler.py#L589
221
+ @classmethod
222
+ def _roofline_estimate(cls, func, args, kwargs) -> tuple[Any, float]: # type: ignore[no-untyped-def]
223
+ """
224
+ Estimates the runtime of a function using a roofline cost model.
225
+
226
+ Args:
227
+ func: The function to estimate.
228
+ args: The arguments to pass to the function.
229
+ kwargs: The keyword arguments to pass to the function.
230
+ out: The output of the function.
231
+
232
+ Returns:
233
+ Tuple[Any, float]: A tuple containing the result of the function and
234
+ the mean operation time in milliseconds.
235
+ """
236
+ if not torch.accelerator.is_available():
237
+ raise AssertionError(
238
+ "Roofline estimation needs to access CUDA capabilities to make estimations"
239
+ )
240
+
241
+ # Roofline Cost Model Explanation
242
+
243
+ # The roofline cost model estimates the execution time of an operator based on
244
+ # the device's empirical maximum FLOPs/sec (pi) and device DRAM bandwidth (beta).
245
+
246
+ # Variables:
247
+ # - pi: Maximum empirical FLOPs/sec of the device
248
+ # - beta: Maximum empirical device DRAM bandwidth (bytes/sec) of the device
249
+ # - I: Arithmetic intensity of the operator (FLOPs/bytes)
250
+ # - op_flops: FLOPs required by the operator
251
+ # - op_bytes: Bytes transferred to and from DRAM for the operator
252
+
253
+ # Calculation Steps:
254
+ # 1. Calculate arithmetic intensity: I = op_flops / op_bytes
255
+ # 2. Calculate estimated FLOPs/sec: est_flops_sec = min(pi, beta * I)
256
+ # 3. Calculate estimated operator time: estimated_op_time = op_flops / est_flops_sec
257
+ # This simplifies to: estimated_op_time = max(op_flops / pi, op_flops / (beta * I))
258
+ # Further simplifying: estimated_op_time = max(op_flops / pi, op_bytes / beta)
259
+
260
+ # Simplified Formulas:
261
+ # - compute_time = op_flops / pi
262
+ # - transfer_time = op_bytes / beta
263
+ # - estimated_op_time = max(compute_time, transfer_time)
264
+
265
+ kwargs = kwargs if kwargs else {}
266
+ out = func(*args, **kwargs)
267
+ op_time = 0.0
268
+ func_packet = func._overloadpacket
269
+ if func_packet not in _IGNORE_OPS:
270
+ flat_args_kwargs, args_spec = pytree.tree_flatten((args, kwargs))
271
+ flat_outs, out_spec = pytree.tree_flatten(out)
272
+ transfer_time = get_transfer_time(flat_args_kwargs, flat_outs)
273
+
274
+ out_dtypes = {
275
+ t.dtype
276
+ for t in flat_outs
277
+ if isinstance(t, torch.Tensor) and t.dtype in _FLOAT_TYPES
278
+ }
279
+
280
+ args, kwargs = pytree.tree_unflatten(flat_args_kwargs, args_spec)
281
+ out = pytree.tree_unflatten(flat_outs, out_spec)
282
+
283
+ compute_time = get_compute_time(func_packet, args, kwargs, out, out_dtypes)
284
+ # We get the estimated time as the max of the transfer time and
285
+ # compute time. We divide by 1e6 to get the time in ms
286
+ op_time = max(transfer_time, compute_time) / 1e6
287
+
288
+ return (out, op_time)
289
+
290
+ def display_modulewise_stats(self, depth: int = 2) -> None:
291
+ """
292
+ Displays module-wise statistics collected by ``RuntimeEstimator``.
293
+
294
+ Prints the pre-forward and pre-backward execution orders.
295
+ Displays the module-wise forward and backward runtimes in milliseconds.
296
+
297
+ Args:
298
+ depth (int): The maximum depth of module hierarchy to display (default to 2).
299
+ """
300
+ print("Pre-Forward Execution Order: ")
301
+ for mod_fqn in self.mod_fw_pre_order:
302
+ mod_depth = mod_fqn.count(".") + 1
303
+ if mod_depth > depth:
304
+ continue
305
+ print(mod_fqn)
306
+ print("Pre-Backward Execution Order: ")
307
+ for mod_fqn in self.mod_bw_pre_order:
308
+ mod_depth = mod_fqn.count(".") + 1
309
+ if mod_depth > depth:
310
+ continue
311
+ print(mod_fqn)
312
+ for mod_fqn, runtimes in self.mod_runtimes.items():
313
+ mod_depth = mod_fqn.count(".") + 1
314
+ if mod_depth > depth:
315
+ continue
316
+ print(
317
+ f"{mod_fqn} fw: {runtimes.get('fw', 0.0):.3f}ms bw: {runtimes.get('bw', 0.0):.3f}ms"
318
+ )
319
+
320
+ def __torch_dispatch__(self, func, types, args=..., kwargs=None): # type: ignore[no-untyped-def]
321
+ # TODO: @sanketpurandare: Flatten tensors by desugaring the tensor subclasses
322
+ # TODO: @sanketpurandare: Add logic for incorporating communication time
323
+ res, op_time = self._estimate(func, args, kwargs)
324
+ for par in self._mod_tracker.parents:
325
+ if self._mod_tracker.is_bw:
326
+ self.mod_runtimes[par]["bw"] += op_time
327
+ else:
328
+ self.mod_runtimes[par]["fw"] += op_time
329
+ self.total_runtime += op_time
330
+ return res
331
+
332
+ def __call__(self, estimate_mode_type: str) -> Self:
333
+ """
334
+ Sets the estimate mode type.
335
+
336
+ Currently supported modes:
337
+ - "operator-level-benchmark": Estimates runtime using operator benchmarking.
338
+ - "operator-level-cost-model": Estimates runtime using roofline cost model.
339
+
340
+ Args:
341
+ estimate_mode_type (str): The type of estimate mode to use.
342
+
343
+ Returns:
344
+ RuntimeEstimator: The runtime estimator instance.
345
+
346
+ Raises:
347
+ NotImplementedError: If the estimate mode type is not supported.
348
+ """
349
+ if estimate_mode_type == "operator-level-benchmark":
350
+ self._estimate = RuntimeEstimator._benchmark_estimate
351
+ elif estimate_mode_type == "operator-level-cost-model":
352
+ self._estimate = RuntimeEstimator._roofline_estimate
353
+ else:
354
+ raise NotImplementedError(
355
+ f"estimate_mode_type {estimate_mode_type} not supported"
356
+ )
357
+ self._estimate_mode_type = estimate_mode_type
358
+ return self
359
+
360
+ def __enter__(self) -> Self:
361
+ fake_mode = active_fake_mode()
362
+ if not isinstance(fake_mode, FakeTensorMode):
363
+ raise AssertionError(
364
+ "No FakeTensorMode found, designed to be used under FakeTensorMode"
365
+ )
366
+ RuntimeEstimator.fake_mode = fake_mode
367
+ self.total_runtime = 0.0
368
+ self.mod_runtimes = defaultdict(lambda: defaultdict(lambda: 0.0))
369
+ self.mod_fw_pre_order.clear()
370
+ self.mod_bw_pre_order.clear()
371
+ self.mod_fw_post_order.clear()
372
+ self.mod_bw_post_order.clear()
373
+ self._mod_tracker.register_user_hooks(
374
+ pre_fw_hook=lambda mod, inp: self.mod_fw_pre_order.append(
375
+ self._mod_tracker.get_known_fqn(mod)
376
+ ),
377
+ pre_bw_hook=lambda mod, g_out: self.mod_bw_pre_order.append(
378
+ self._mod_tracker.get_known_fqn(mod)
379
+ ),
380
+ post_fw_hook=lambda mod, inp, out: self.mod_fw_post_order.append(
381
+ self._mod_tracker.get_known_fqn(mod)
382
+ ),
383
+ post_bw_hook=lambda mod, g_inp: self.mod_bw_post_order.append(
384
+ self._mod_tracker.get_known_fqn(mod)
385
+ ),
386
+ )
387
+ self._mod_tracker.__enter__()
388
+ super().__enter__()
389
+ return self
390
+
391
+ # pyrefly: ignore [bad-override]
392
+ def __exit__(self, *args: Any) -> None:
393
+ print(
394
+ f"Estimated ({self._estimate_mode_type})"
395
+ f"total_time: {self.total_runtime:.3f} ms"
396
+ )
397
+ if len(self._no_fallback_kernel) > 0:
398
+ print("no_fallback_kernel: ", list(self._no_fallback_kernel))
399
+ super().__exit__(*args)
400
+ self._mod_tracker.clear_user_hooks()
401
+ self._mod_tracker.__exit__()
western_ai/usa/pytorch/torch/distributed/_tools/sac_estimator.py ADDED
@@ -0,0 +1,965 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os
3
+ import sys
4
+ from collections import OrderedDict
5
+ from dataclasses import astuple, dataclass
6
+ from typing import Any, NamedTuple
7
+ from typing_extensions import Self
8
+
9
+ import torch
10
+ from torch import nan, nn, UntypedStorage
11
+ from torch._guards import active_fake_mode
12
+ from torch._subclasses.fake_tensor import FakeTensorMode
13
+ from torch.distributed._tools.common_utils import get_untyped_storages
14
+ from torch.distributed._tools.mod_tracker import ModTracker
15
+ from torch.distributed._tools.runtime_estimator import RuntimeEstimator
16
+ from torch.testing._internal.composite_compliance import (
17
+ is_inplace,
18
+ is_inplace_view_fn,
19
+ is_view_fn,
20
+ )
21
+ from torch.utils._python_dispatch import TorchDispatchMode
22
+ from torch.utils._pytree import tree_flatten
23
+ from torch.utils.checkpoint import SAC_IGNORED_OPS
24
+
25
+
26
+ __all__ = ["SACEstimator", "SACStats", "MSPS", "SACTradeOffStats", "SACGreedyOrderMeta"]
27
+ aten = torch.ops.aten
28
+
29
+ _ADDITIONAL_IGNORED_OPS = {
30
+ aten.lift_fresh.default, # type: ignore[attr-defined]
31
+ torch.ops.profiler._record_function_exit._RecordFunction, # type: ignore[attr-defined]
32
+ aten.clone.default, # type: ignore[attr-defined] # seems needed for torch.compile
33
+ }
34
+ OPS_TO_ALWAYS_SKIP = SAC_IGNORED_OPS | _ADDITIONAL_IGNORED_OPS
35
+ # This value is hard-coded here:
36
+ # https://github.com/pytorch/pytorch/blob/5fba5d83f0703ff8077ab65448a998e9ad6598fd/c10/cuda/CUDACachingAllocator.cpp#L117
37
+ _PYTORCH_MIN_ALLOCATE = (
38
+ 2**9 if int(os.environ.get("PYTORCH_NO_CUDA_MEMORY_CACHING", 0)) == 0 else 1
39
+ )
40
+
41
+
42
+ def _display_stats_tabular(headers: list[str], table_data: list[list[Any]]) -> None:
43
+ try:
44
+ from tabulate import tabulate
45
+ except ImportError as err:
46
+ raise ImportError("Please install tabulate.") from err
47
+
48
+ # Use tabulate to print the table
49
+ print(tabulate(table_data, headers=headers, tablefmt="rst"))
50
+
51
+
52
+ # Based on:
53
+ # https://github.com/facebookresearch/xformers/blob/main/xformers/checkpoint.py#L71
54
+ @dataclass
55
+ class _SACMetadata:
56
+ """
57
+ Stores metadata for a single operator for SAC.
58
+
59
+ Attributes:
60
+ func (Any): The operator function.
61
+ time_taken (float): The time taken by the operator.
62
+ memory_used (float): The memory used by the operator.
63
+ curr_idx (int): The current operator index.
64
+ output_ids (Tuple[int, ...]): The storage IDs of the operator's outputs.
65
+ inplace_info (Tuple[int, ...]): Tuple of self and parent operator for in-place operator.
66
+ is_view_like (bool): Whether the operator is view-like.
67
+ is_rand_op (bool): Whether the operator is a random operator.
68
+ """
69
+
70
+ func: Any
71
+ time_taken: float
72
+ memory_used: float
73
+ curr_idx: int
74
+ output_ids: tuple[int, ...]
75
+ inplace_info: tuple[int, ...]
76
+ is_view_like: bool
77
+ is_rand_op: bool
78
+
79
+
80
+ @dataclass
81
+ class _SACModMetadata:
82
+ """
83
+ Stores metadata for a module for SAC.
84
+
85
+ Attributes:
86
+ start_idx (int): The starting index of the module's operators.
87
+ force_store_random (bool): Whether to force store random operators in the module.
88
+ sac_metadata (List[_SACMetadata]): List of metadata for each operator in the module.
89
+ """
90
+
91
+ start_idx: int
92
+ force_store_random: bool
93
+ sac_metadata: list[_SACMetadata]
94
+
95
+
96
+ @dataclass
97
+ class SACStats:
98
+ """
99
+ A class for storing Activation Checkpointing statistics corresponding to a module.
100
+
101
+ Attributes:
102
+ func_names (List[str]): List of operator names.
103
+ runtimes (List[float]): List of operator runtimes in millliseconds.
104
+ memory (List[int]): List of operator memory usage in bytes.
105
+ view_like_ops (List[int]): Indices of view-like operators.
106
+ rand_ops (List[int]): Indices of random operators.
107
+ saved_autograd_ops (List[int]): Indices of operator results saved by autograd engine.
108
+ inplace_ops (List[Tuple[int, int]]): Tuple of indices of op and its first parent for Inplace operators.
109
+ force_store_random (bool): Whether to force store random operator results.
110
+ """
111
+
112
+ func_names: list[str]
113
+ runtimes: list[float]
114
+ memory: list[int]
115
+ view_like_ops: list[int]
116
+ rand_ops: list[int]
117
+ saved_autograd_ops: list[int]
118
+ inplace_ops: list[tuple[int, int]]
119
+ force_store_random: bool
120
+
121
+
122
+ class MSPS(NamedTuple):
123
+ """
124
+ Represents Memory and Runtime Statistics for an operator/operator group.
125
+
126
+ Attributes:
127
+ func_names (set[str]): Set of operator/operator group names.
128
+ op_idx (int): Operator index (group head index in case of operator groups).
129
+ memory (int): Memory usage in bytes.
130
+ runtime (float): Runtime in milliseconds.
131
+ msps (float): Memory per second calculated as memory/runtime.
132
+ """
133
+
134
+ func_names: set[str]
135
+ op_idx: int
136
+ memory: int
137
+ runtime: float
138
+ msps: float
139
+
140
+
141
+ @dataclass
142
+ class SACTradeOffStats:
143
+ """
144
+ Stores statistics for activation-checkpointing trade-off.
145
+
146
+ Attributes:
147
+ n_segments (int): Number of piecewise linear segments fitted to the trade-off curve.
148
+ slopes (List[float]): Slopes of the pieces of linear segments fitted to the trade-off curve.
149
+ intercepts (List[float]): Intercepts of the of the pieces of linear segments fitted to the trade-off curve.
150
+ fit_breaks (List[float]): Breakpoints of the of the pieces of linear segments fitted to the trade-off curve.
151
+ tradeoff_curve (OrderedDict[float, float]): Trade-off curve data of memory discarded vs recomputation time.
152
+ sac_memory (int): Total memory of operations available for activation checkpointing in bytes.
153
+ sac_runtime (float): Total runtime of operations available for activation checkpointing in milliseconds.
154
+ """
155
+
156
+ n_segments: int
157
+ slopes: list[float]
158
+ intercepts: list[float]
159
+ fit_breaks: list[float]
160
+ tradeoff_curve: OrderedDict[float, float]
161
+ sac_memory: int
162
+ sac_runtime: float
163
+
164
+
165
+ @dataclass
166
+ class SACGreedyOrderMeta:
167
+ """
168
+ Stores metadata for Greedy-order SAC.
169
+
170
+ Attributes:
171
+ recomputed_ops (set[int]): Set of operator indices to be recomputed.
172
+ stored_ops (set[int]): Set of operator indices to be stored.
173
+ inplace_op_groups (dict[int, set[int]]): Dictionary of inplace operator groups from group-head to operators.
174
+ random_ops_group (dict[int, set[int]]): Dictionary of random op group head to random ops.
175
+ msps_meta (list[MSPS]): List of Memory and Runtime Statistics for operators.
176
+ """
177
+
178
+ recomputed_ops: set[int]
179
+ stored_ops: set[int]
180
+ inplace_op_groups: dict[int, set[int]]
181
+ random_ops_group: dict[int, set[int]]
182
+ msps_meta: list[MSPS]
183
+
184
+
185
+ class SACEstimator(TorchDispatchMode):
186
+ """
187
+ Estimates the memory and recomputation time trade-offs for applying Selective Activation Checkpointing (SAC).
188
+
189
+ This class provides a ``TorchDispatchMode`` based context manager that can be used to estimate the memory and
190
+ runtime trade-offs of functions or ``torch.nn.Module``s for Selective Activation Checkpointing (SAC). It provides
191
+ detailed statistics and metadata information for operators of each module and provides a greedy order for selecting
192
+ the operators to be recomputed/checkpointed. It also constructs the per-module trade-off graph of discarded memory
193
+ vs recomputation time for the obtained greedy order. Using ``RuntimeEstimator`` under the hood, it supports two
194
+ estimation modes, `operator-level-benchmark` and (`operator-level-cost-model` (roofline model).
195
+
196
+ Attributes:
197
+ sac_mod_stats (Dict[str, SACStats]): Dictionary from module FQN (fully qualified name) to ``SACStats``.
198
+ sac_mod_tradeoff_stats (Dict[str, SACTradeOffStats]): Dictionary from module FQN to ``SACTradeOffStats``.
199
+ sac_mod_greedy_order_meta (Dict[str, SACGreedyOrderMeta]): Dictionary from module FQN to ``SACGreedyOrderMeta``.
200
+
201
+ Note:
202
+ 1) This class is designed to be used under ``FakeTensorMode``.
203
+ 2) Currently, it only supports estimation of compute time and memory usage, and does not consider communication.
204
+
205
+ Example usage:
206
+
207
+ .. code-block:: python
208
+
209
+ sac_estimator = SACEstimator()
210
+ with FakeTensorMode():
211
+ module = ...
212
+ inp = ...
213
+ with sac_estimator("operator-level-cost-model"):
214
+ output = module(inp)
215
+ sac_estimator.display_modulewise_sac_stats(depth=4, print_tabular=True)
216
+ """
217
+
218
+ def __init__(self) -> None:
219
+ self.sac_mod_stats: dict[str, SACStats] = {}
220
+ self.sac_mod_tradeoff_stats: dict[str, SACTradeOffStats] = {}
221
+ self.sac_mod_greedy_order_meta: dict[str, SACGreedyOrderMeta] = {}
222
+ self._mod_tracker = ModTracker()
223
+ self._sac_metadata: list[_SACMetadata] = []
224
+ self._sac_mod_metadata: dict[str, _SACModMetadata] = {}
225
+ self._leaf_modules: set[str] = set()
226
+ self._saved_tensor_hook_ctx = torch.autograd.graph.saved_tensors_hooks(
227
+ self._pack_hook, lambda x: x
228
+ )
229
+ self._saved_tensor_ids: set[int] = set()
230
+ self._estimate_runtime = RuntimeEstimator._roofline_estimate
231
+
232
+ def _pack_hook(self, x: torch.Tensor) -> torch.Tensor:
233
+ # Hook function to track underlying storage IDs of tensors
234
+ # Updates the _saved_tensor_ids set with the IDs of the tensor's storages
235
+ # Used in conjunction with torch.autograd.graph.saved_tensors_hooks
236
+ untyped_storages = get_untyped_storages(x)
237
+ storage_ids = (hash(st) for st in untyped_storages)
238
+ self._saved_tensor_ids.update(storage_ids)
239
+ return x
240
+
241
+ def _pre_fw_hook(self, mod: nn.Module, inputs: Any) -> None:
242
+ # Pre-forward hook function to prepare module metadata
243
+ # Tracks module FQN, force store random flag, and ``SACModMetadata``
244
+ # Initializes metadata for non-leaf modules, marks leaf modules
245
+ mod_fqn = self._mod_tracker.get_known_fqn(mod)
246
+ if mod_fqn is None:
247
+ raise AssertionError
248
+ num_children = sum(1 for _ in mod.children())
249
+ if num_children > 0:
250
+ force_store_random = self._get_force_store_random(inputs)
251
+ self._sac_mod_metadata[mod_fqn] = _SACModMetadata(
252
+ start_idx=len(self._sac_metadata),
253
+ force_store_random=force_store_random,
254
+ sac_metadata=[],
255
+ )
256
+ else:
257
+ self._leaf_modules.add(mod_fqn)
258
+
259
+ def _post_fw_hook(self, mod: nn.Module, inputs: Any, outputs: Any) -> None:
260
+ # 1. Retrieves the module's FQN and checks if it's a leaf module
261
+ # 2. If not a leaf module, computes:
262
+ # - ``SACStats`` using the module's metadata and force store random flag
263
+ # - ``SACGreedyOrderMeta`` using the computed SAC statistics
264
+ mod_fqn = self._mod_tracker.get_known_fqn(mod)
265
+ if mod_fqn is None:
266
+ raise AssertionError
267
+ if mod_fqn in self._leaf_modules:
268
+ return
269
+ else:
270
+ self.sac_mod_stats[mod_fqn] = self._get_sac_stats(
271
+ data=self._sac_mod_metadata[mod_fqn].sac_metadata,
272
+ force_store_random=self._sac_mod_metadata[mod_fqn].force_store_random,
273
+ )
274
+ self.sac_mod_greedy_order_meta[mod_fqn] = self._get_greedy_order_meta(
275
+ self.sac_mod_stats[mod_fqn]
276
+ )
277
+
278
+ def _get_force_store_random(self, inputs: Any) -> bool:
279
+ flat_inputs, _ = tree_flatten(inputs)
280
+ return all(not isinstance(x, torch.Tensor) for x in flat_inputs)
281
+
282
+ def _get_sac_stats(
283
+ self, data: list[_SACMetadata], force_store_random: bool
284
+ ) -> SACStats:
285
+ # 1. Ignore the operations that should be skipped by SAC such as aten.detach.default because autograd
286
+ # inserts those during backward and it breaks the fwd-bwd alignment
287
+ filtered_data = [x for x in data if x.func not in OPS_TO_ALWAYS_SKIP]
288
+
289
+ (
290
+ ops,
291
+ runtimes_,
292
+ memory_,
293
+ new_ids,
294
+ output_ids,
295
+ inplace_ops_,
296
+ view_like_ops_,
297
+ rand_ops_,
298
+ ) = zip(*[astuple(x) for x in filtered_data], strict=True)
299
+
300
+ # 2. Extract the metadata information
301
+ runtimes = list(runtimes_)
302
+ memory = list(memory_)
303
+ func_names = [op._overloadpacket.__name__ for op in ops]
304
+ view_like_ops = [i for i, x in enumerate(view_like_ops_) if x]
305
+ rand_ops = [i for i, x in enumerate(rand_ops_) if x]
306
+ saved_autograd_ops = [
307
+ i
308
+ for i, out_ids in enumerate(output_ids)
309
+ if set(out_ids).issubset(self._saved_tensor_ids)
310
+ ]
311
+
312
+ # 3. Remap the inplace indices as we have removed OPS_TO_ALWAYS_SKIP
313
+ # FIXME @sanketpurandare: Fix this by changing the parent of the inplace-op
314
+ # to itself if the original parent is in OPS_TO_ALWAYS_SKIP.
315
+ try:
316
+ inplace_ops = [tuple(map(new_ids.index, x)) for x in inplace_ops_ if x]
317
+ except ValueError as err:
318
+ raise ValueError(
319
+ f"The remapping of inplace ops failed since one of the inplace op parents"
320
+ f" must have been present in {OPS_TO_ALWAYS_SKIP}"
321
+ ) from err
322
+
323
+ # 4. The last operation is always stored as the output of the checkpoint
324
+ # block, so we can avoid recomputing it. We set the memory to zero
325
+ # instead of adding a new constraint because we want both the 0 and 1
326
+ # endpoints for memory_budget to be valid
327
+ # FIXME @sanketpurandare: this heuristic for finding the last non-view non-inplace op
328
+ # might not always be correct, which would yield suboptimal policies
329
+ last_op = len(ops) - 1
330
+ skip_ops_ = set(view_like_ops) | set({x[0] for x in inplace_ops})
331
+ reversed_skip_ops = sorted(skip_ops_, reverse=True)
332
+ for op in reversed_skip_ops:
333
+ if op == last_op:
334
+ last_op -= 1
335
+
336
+ memory[last_op] = 0
337
+
338
+ # 5. Create a single ``SACStats`` object for the entire block of ``_SACMetadata``.
339
+ return SACStats(
340
+ func_names=func_names,
341
+ runtimes=runtimes,
342
+ memory=memory,
343
+ view_like_ops=view_like_ops,
344
+ rand_ops=rand_ops,
345
+ saved_autograd_ops=saved_autograd_ops,
346
+ inplace_ops=inplace_ops, # type: ignore[arg-type]
347
+ force_store_random=force_store_random,
348
+ )
349
+
350
+ def _get_inplace_metadata(
351
+ self, func: Any, out_storages: set[UntypedStorage]
352
+ ) -> tuple[int, tuple[int, ...], dict[str, tuple[int, ...]]]:
353
+ # 1. Get the current index of the metadata obtained so far
354
+ curr_idx = len(self._sac_metadata)
355
+ # 2. Get the set of active modules that are not leaf
356
+ active_mod_fqns: set[str] = {
357
+ par for par in self._mod_tracker.parents if par not in self._leaf_modules
358
+ }
359
+ # 3. Output ids are the identifies of the storage objects corresponding to the tensors
360
+ output_ids = tuple(hash(st) for st in out_storages)
361
+ # 4. If the function is not inplace, return
362
+ if not is_inplace(func):
363
+ return curr_idx, output_ids, dict.fromkeys(active_mod_fqns, ())
364
+
365
+ op_idx = curr_idx
366
+ # 5. Initialize the parent op ids of the inplace op for each of the active modules
367
+ mod_op_parent_idxs: dict[str, int] = dict.fromkeys(active_mod_fqns, -1)
368
+ for i, d in enumerate(self._sac_metadata):
369
+ # 6. Find the first occurrence of a tensor corresponding to each module that
370
+ # shares the same storage as the current tensor
371
+ past_output_ids = d.output_ids
372
+ if set(output_ids).issubset(set(past_output_ids)):
373
+ for mod_fqn, op_parent_idx in mod_op_parent_idxs.items():
374
+ if op_parent_idx == -1:
375
+ if acm_stats := self._sac_mod_metadata.get(mod_fqn, None):
376
+ if i >= acm_stats.start_idx:
377
+ mod_op_parent_idxs[mod_fqn] = i
378
+ else:
379
+ if mod_fqn != "Global":
380
+ raise AssertionError
381
+ mod_op_parent_idxs[mod_fqn] = i
382
+ # 7. If no parent tensor is found, then it's probably an inplace op on the arguments
383
+ # so one can just store the current-op idx as parent idx
384
+ for mod_fqn, op_parent_idx in mod_op_parent_idxs.items():
385
+ if op_parent_idx < 0:
386
+ mod_op_parent_idxs[mod_fqn] = op_idx
387
+ mod_inplace_info = {
388
+ mod_fqn: (op_idx, mod_op_parent_idxs[mod_fqn])
389
+ for mod_fqn in active_mod_fqns
390
+ }
391
+ return curr_idx, output_ids, mod_inplace_info # type: ignore[return-value]
392
+
393
+ def __torch_dispatch__( # type: ignore[no-untyped-def]
394
+ self, func, types, args=..., kwargs=None
395
+ ):
396
+ # 1. Get the runtime estimate
397
+ out, op_time = self._estimate_runtime(func, args, kwargs)
398
+ flat_outs, _ = tree_flatten(out)
399
+ out_storages_cuda: set[UntypedStorage] = set()
400
+ out_storages_cpu: set[UntypedStorage] = set()
401
+ cuda_devices: set[torch.device] = set()
402
+ for o in flat_outs:
403
+ if isinstance(o, torch.Tensor):
404
+ if o.device.type == "cuda":
405
+ out_storages_cuda.update(get_untyped_storages(o))
406
+ cuda_devices.add(o.device)
407
+ else:
408
+ out_storages_cpu.update(get_untyped_storages(o))
409
+
410
+ # Check if there's more than 1 CUDA device
411
+ if len(cuda_devices) > 1:
412
+ raise AssertionError(
413
+ f"{func.__name__}'s output has more than 1 CUDA devices {cuda_devices}"
414
+ )
415
+
416
+ # 2. Get the memory consumed by output
417
+ nbytes_cuda = sum(
418
+ math.ceil(st.nbytes() / _PYTORCH_MIN_ALLOCATE) * _PYTORCH_MIN_ALLOCATE
419
+ for st in out_storages_cuda
420
+ )
421
+ nbytes_cpu = sum(st.nbytes() for st in out_storages_cpu)
422
+ nbytes = nbytes_cuda + nbytes_cpu
423
+ # 3. Get the current operator index, output storage identifiers and inplace metadata
424
+ out_storages = out_storages_cuda | out_storages_cpu
425
+ curr_idx, output_ids, mod_inplace_info = self._get_inplace_metadata(
426
+ func, out_storages
427
+ )
428
+ # 4. Determine if the function is in-place, random-op or a view-like
429
+ is_view_like = is_view_fn(func) or is_inplace_view_fn(func)
430
+ is_rand_op = torch.Tag.nondeterministic_seeded in func.tags
431
+ if is_view_like:
432
+ nbytes = 0
433
+ # sdpa has non-deterministic seed, but might be deterministic
434
+ # if no dropout is applied
435
+ if func.overloadpacket.__name__ == "_scaled_dot_product_flash_attention":
436
+ # pyrefly: ignore [missing-attribute]
437
+ is_rand_op = kwargs.get("dropout_p", 0) != 0
438
+ # 5. Create metadata information per active non-leaf module
439
+ for mod_fqn in self._mod_tracker.parents:
440
+ if mod_fqn in self._leaf_modules:
441
+ continue
442
+ acm = _SACMetadata(
443
+ func=func,
444
+ time_taken=op_time,
445
+ memory_used=nbytes,
446
+ curr_idx=curr_idx,
447
+ output_ids=output_ids,
448
+ inplace_info=mod_inplace_info[mod_fqn],
449
+ is_view_like=is_view_like,
450
+ is_rand_op=is_rand_op,
451
+ )
452
+ if acm_stats := self._sac_mod_metadata.get(mod_fqn, None):
453
+ acm_stats.sac_metadata.append(acm)
454
+ else:
455
+ if mod_fqn != "Global":
456
+ raise AssertionError(f"Module {mod_fqn} not found in AC Mod Stats")
457
+ self._sac_metadata.append(acm)
458
+
459
+ return out
460
+
461
+ def _get_greedy_order_meta(self, sac_stats: SACStats) -> SACGreedyOrderMeta:
462
+ # An inplace-op group is a set of inplace-ops that operate on the same underlying tensor storage.
463
+ # 1. inplace_op_groups: A dictionary from the top-most parent of inplace-ops to the inplace-ops in the group
464
+ # The top-most op can itself be an inplace-op or can be a non-inplace op.
465
+ # 2. inplace_op_to_group_head: A dictionary that maps all the inplace-ops to their respective group heads.
466
+ inplace_op_groups: dict[int, set[int]] = {}
467
+ inplace_op_to_group_head: dict[int, int] = dict(sac_stats.inplace_ops)
468
+
469
+ # Initialize inplace_op_groups using inplace_op_to_group_head
470
+ for op_idx, group_head_idx in inplace_op_to_group_head.items():
471
+ op_group = inplace_op_groups.setdefault(group_head_idx, {group_head_idx})
472
+ op_group.add(op_idx)
473
+
474
+ # Like inplace ops, all of the random ops in the function/module should all be either recomputed or saved
475
+ # as a group. This is because, they affect the random seed generator. If force_store_random is set True,
476
+ # all of the random ops will be stored by default. For ease of manageability, we store the top-most random op
477
+ # as the leader of the random_ops_group.
478
+ random_ops_group: dict[int, set[int]] = {}
479
+ random_group_head_idx = min(sac_stats.rand_ops, default=-1)
480
+ has_rand_ops = bool(sac_stats.rand_ops)
481
+ if has_rand_ops:
482
+ random_ops_group[random_group_head_idx] = set(sac_stats.rand_ops)
483
+
484
+ # 1. Random ops are stored if force_store_random is set
485
+ # 2. View-like ops are recomputed by default
486
+ # 3. For inplace_op_groups:
487
+ # a) If the head of this group is an inplace op, then we have to store the entire group.
488
+ # b) If any op in the group is random and force_store_random is set, then entire group will be stored.
489
+ # c) If none of ops in the group are random and the head of the group is not an in-place op, then
490
+ # this group can be considered for recomputation in its entirety
491
+ stored_ops: set[int] = set()
492
+ recomputed_ops: set[int] = set()
493
+ # Case 1:
494
+ if has_rand_ops and sac_stats.force_store_random:
495
+ stored_ops.add(random_group_head_idx)
496
+ # Case 2:
497
+ recomputed_ops.update(set(sac_stats.view_like_ops))
498
+
499
+ for group_head_idx, op_group in inplace_op_groups.items():
500
+ # Case 3a:
501
+ if group_head_idx in inplace_op_to_group_head:
502
+ stored_ops.add(group_head_idx)
503
+ # Case 3b:
504
+ if (
505
+ sac_stats.force_store_random & len(op_group & set(sac_stats.rand_ops))
506
+ > 0
507
+ ):
508
+ stored_ops.add(group_head_idx)
509
+
510
+ # The potential recompute candidates are populated as:
511
+ recompute_candidates: set[int] = set()
512
+ # 1) The random group head if it is not stored
513
+ if has_rand_ops and random_group_head_idx not in stored_ops:
514
+ recompute_candidates.add(random_group_head_idx)
515
+ # 2) The in-place op group heads that are not stored
516
+ recompute_candidates.update(set(inplace_op_groups.keys()) - stored_ops)
517
+ # 3) The non-inplace and non-random ops that are neither stored nor recomputed by default
518
+ recompute_candidates.update(
519
+ set(range(len(sac_stats.memory)))
520
+ - recomputed_ops
521
+ - stored_ops
522
+ - set(inplace_op_to_group_head.keys())
523
+ - set(sac_stats.rand_ops)
524
+ )
525
+
526
+ # We define msps for a recomp candidate as the ratio of memory/runtime aka memory savings per second
527
+ msps_meta: list[MSPS] = []
528
+ for cand_idx in recompute_candidates:
529
+ op_indices = {cand_idx}
530
+ if cand_idx in inplace_op_groups:
531
+ op_indices.update(inplace_op_groups[cand_idx])
532
+ if has_rand_ops and cand_idx == random_group_head_idx:
533
+ op_indices.update(sac_stats.rand_ops)
534
+
535
+ mem = sum(sac_stats.memory[op_idx] for op_idx in op_indices)
536
+ runtime = sum(sac_stats.runtimes[op_idx] for op_idx in op_indices)
537
+ func_names = {sac_stats.func_names[op_idx] for op_idx in op_indices}
538
+ msps = (mem / runtime) if runtime > 0 else sys.float_info.max
539
+ msps_meta.append(MSPS(func_names, cand_idx, mem, runtime, msps))
540
+ # We choose candidates to be recomputed based on increasing msps
541
+ msps_meta.sort(key=lambda x: x.msps, reverse=True)
542
+ return SACGreedyOrderMeta(
543
+ recomputed_ops, stored_ops, inplace_op_groups, random_ops_group, msps_meta
544
+ )
545
+
546
+ def _get_sac_tradeoff_pwlf_stats(
547
+ self,
548
+ sac_stats: SACStats,
549
+ greedy_order_meta: SACGreedyOrderMeta,
550
+ n_segments: int = 2,
551
+ save_tradeoff_graph: bool = False,
552
+ filename: str = "ac_tradeoff",
553
+ ) -> SACTradeOffStats:
554
+ try:
555
+ import numpy as np # type: ignore[import-not-found]
556
+ import pwlf # type: ignore[import-untyped, import-not-found]
557
+ except ImportError as err:
558
+ raise ImportError("Please install pwlf and numpy package.") from err
559
+
560
+ stored_ops, recomputed_ops, inplace_op_groups, random_ops_group, msps_meta = (
561
+ greedy_order_meta.stored_ops,
562
+ greedy_order_meta.recomputed_ops,
563
+ greedy_order_meta.inplace_op_groups,
564
+ greedy_order_meta.random_ops_group,
565
+ greedy_order_meta.msps_meta,
566
+ )
567
+ # 1. Initialize the discarded memory and recomputation runtime to sum of already chosen recomputed_ops
568
+ recomp_indices: set[int] = set()
569
+ for r_idx in recomputed_ops:
570
+ recomp_indices.add(r_idx)
571
+ if r_idx in inplace_op_groups:
572
+ recomp_indices.update(inplace_op_groups[r_idx])
573
+ if r_idx in random_ops_group:
574
+ recomp_indices.update(random_ops_group[r_idx])
575
+
576
+ discarded_mem = sum(sac_stats.memory[op_idx] for op_idx in recomp_indices)
577
+ recomp_runtime = sum(sac_stats.runtimes[op_idx] for op_idx in recomp_indices)
578
+ # 2. Initialize the max recomputation time and total recomputation memory
579
+ sac_runtime = sum(sac_stats.runtimes)
580
+ sac_memory = sum(sac_stats.memory)
581
+ # 3. Tradeoff curve stores the KV pair of the discarded memory to total memory and,
582
+ # recomputation time to total runtime incurred.
583
+ delta = 1e-2
584
+ tradeoff_curve = OrderedDict()
585
+ # 4. Initialize the trade-off curve with the stats of already chosen recomputed_ops
586
+ tradeoff_curve[(discarded_mem / sac_memory) + delta] = (
587
+ recomp_runtime / sac_runtime
588
+ )
589
+ # 5. Update the trade-off curve with memory and runtime stats of SAC candidates in the
590
+ # greedy order of their ``MSPS``.
591
+ for cand in msps_meta:
592
+ discarded_mem += cand.memory
593
+ recomp_runtime += cand.runtime
594
+ tradeoff_curve[(discarded_mem / sac_memory) + delta] = (
595
+ recomp_runtime / sac_runtime
596
+ )
597
+ # 6. Finally, we add the memory and recomputation time of the always stored ops.
598
+ stored_indices: set[int] = set()
599
+ for s_idx in stored_ops:
600
+ stored_indices.add(s_idx)
601
+ if s_idx in inplace_op_groups:
602
+ stored_indices.update(inplace_op_groups[s_idx])
603
+ if s_idx in random_ops_group:
604
+ stored_indices.update(random_ops_group[s_idx])
605
+ discarded_mem += sum(sac_stats.memory[op_idx] for op_idx in stored_indices)
606
+ recomp_runtime += sum(sac_stats.runtimes[op_idx] for op_idx in stored_indices)
607
+ tradeoff_curve[(discarded_mem / sac_memory) + delta] = (
608
+ recomp_runtime / sac_runtime
609
+ )
610
+ x_ = list(tradeoff_curve.keys())
611
+ y_ = list(tradeoff_curve.values())
612
+ # 7. We shift the y values to left and x values to right to upperbound the trade-off function
613
+ # TODO: Write a better explanation why this needs to be done
614
+ x = x_[: len(x_) - 1]
615
+ y = y_[1:]
616
+ tradeoff_pwlf = pwlf.PiecewiseLinFit(x, y)
617
+ # 8. Fit a piecewise linear function with the specified number of segments to the trade-off curve.
618
+ n_segments = max(min(len(x) - 2, n_segments), 1)
619
+ tradeoff_pwlf.fit(n_segments=n_segments)
620
+
621
+ # save prediction graph
622
+ def save_prediction_graph(
623
+ pwlf_: pwlf.PiecewiseLinFit, x: list[float], y: list[float], filename: str
624
+ ) -> None:
625
+ try:
626
+ import matplotlib.pyplot as plt # type: ignore[import-not-found]
627
+ import numpy as np # type: ignore[import-not-found]
628
+ except ImportError as err:
629
+ raise ImportError(
630
+ "Install matplotlib and numpy using pip: pip install matplotlib numpy"
631
+ ) from err
632
+ # predict for the determined points
633
+ xHat = np.linspace(min(x), max(x), num=10000)
634
+ yHat = pwlf_.predict(xHat)
635
+
636
+ # plot the results
637
+ plt.figure()
638
+ plt.plot(x, y, "o", label="Shifted")
639
+ plt.plot(xHat, yHat, "-", label="Predicted")
640
+ plt.plot(x_, y_, "x", label="Original")
641
+ plt.ylabel("Recomp time / Total recomp time")
642
+ plt.xlabel("Memory discarded / Total memory")
643
+ plt.legend()
644
+ plt.title(f"{filename}")
645
+ plt.suptitle(
646
+ f"Total Memory = {sac_memory} B Total Runtime = {sac_runtime:.4f} ms",
647
+ fontsize=10,
648
+ )
649
+ folder_name = "tradeoff_graphs"
650
+ if not os.path.exists(folder_name):
651
+ os.makedirs(folder_name)
652
+ # Save the plots in the folder
653
+ plt.savefig(os.path.join(folder_name, f"{filename}.png"))
654
+
655
+ if save_tradeoff_graph:
656
+ save_prediction_graph(tradeoff_pwlf, x, y, filename)
657
+ # 9. Obtain the slopes, intercepts and breakpoints of the fitted piecewise linear functions
658
+ slopes = tradeoff_pwlf.calc_slopes().tolist()
659
+ if not (
660
+ isinstance(tradeoff_pwlf.intercepts, np.ndarray)
661
+ and isinstance(tradeoff_pwlf.fit_breaks, np.ndarray)
662
+ ):
663
+ raise AssertionError
664
+ intercepts = tradeoff_pwlf.intercepts.tolist()
665
+ fit_breaks = tradeoff_pwlf.fit_breaks.tolist()
666
+ return SACTradeOffStats(
667
+ n_segments=n_segments,
668
+ slopes=slopes,
669
+ intercepts=intercepts, # type: ignore[arg-type]
670
+ fit_breaks=fit_breaks, # type: ignore[arg-type]
671
+ tradeoff_curve=tradeoff_curve,
672
+ sac_memory=sac_memory,
673
+ sac_runtime=sac_runtime,
674
+ )
675
+
676
+ def display_sac_stats(
677
+ self, sac_stats: SACStats, print_tabular: bool = False
678
+ ) -> None:
679
+ """
680
+ Displays the SAC statistics.
681
+
682
+ Args:
683
+ sac_stats (SACStats): The SAC statistics to display.
684
+ print_tabular (bool, optional): Whether to print the statistics in a tabular format. Defaults to False.
685
+
686
+ Prints:
687
+ 1. Total Memory: The total memory usage in bytes.
688
+ 2. Total Runtime: The total runtime in milliseconds.
689
+ 3. Store Random: A flag indicating whether to force store random operator results.
690
+
691
+ Followed by a table with the following columns:
692
+ 1. Op Idx: The operator index.
693
+ 2. Op Name: The operator name.
694
+ 3. Runtimes (ms): The operator runtime in milliseconds.
695
+ 4. Memory (B): The operator memory usage in bytes.
696
+ 5. View-like: A flag indicating whether the operator is view-like.
697
+ 6. Random: A flag indicating whether the operator is random.
698
+ 7. Saved Autograd: A flag indicating whether the operator's result is saved by autograd engine.
699
+ 8. In-place: The index of the operator's first parent, or None if not in-place.
700
+
701
+ If print_tabular is True, the table is printed in a tabular format.
702
+ Otherwise, the table is printed in a plain text format.
703
+ """
704
+ print(
705
+ f"Total Memory: {sum(sac_stats.memory)} B Total Runtime: {sum(sac_stats.runtimes)} ms"
706
+ f" Store Random: {sac_stats.force_store_random}"
707
+ )
708
+ table_data = []
709
+ op_parent = dict(sac_stats.inplace_ops)
710
+ for i, fn_name in enumerate(sac_stats.func_names):
711
+ row = [
712
+ str(i),
713
+ fn_name,
714
+ f"{sac_stats.runtimes[i]:.4f}",
715
+ str(sac_stats.memory[i]),
716
+ str(i in sac_stats.view_like_ops),
717
+ str(i in sac_stats.rand_ops),
718
+ str(i in sac_stats.saved_autograd_ops),
719
+ str(op_parent.get(i)),
720
+ ]
721
+ table_data.append(row)
722
+ # Define headers
723
+ headers = [
724
+ "Op Idx",
725
+ "Op Name",
726
+ "Runtimes(ms)",
727
+ "Memory (B)",
728
+ "View-like",
729
+ "Random",
730
+ "Saved Autograd",
731
+ "In-place",
732
+ ]
733
+ if print_tabular:
734
+ _display_stats_tabular(headers, table_data)
735
+ else:
736
+ max_widths = [0 for _ in range(len(headers))]
737
+ table_data.insert(0, headers)
738
+ for row in table_data:
739
+ for i, elem in enumerate(row):
740
+ max_widths[i] = max(max_widths[i], len(elem))
741
+ for row in table_data:
742
+ print(
743
+ "\t".join(
744
+ [f"{elem:<{max_widths[i]}}" for i, elem in enumerate(row)]
745
+ )
746
+ )
747
+
748
+ def display_sac_tradeoff_stats(
749
+ self,
750
+ greedy_order_meta: SACGreedyOrderMeta,
751
+ sac_stats: SACStats,
752
+ print_tabular: bool = False,
753
+ ) -> None:
754
+ """
755
+ Displays the SAC trade-off statistics.
756
+
757
+ Args:
758
+ greedy_order_meta (SACGreedyOrderMeta): The SAC greedy order metadata.
759
+ sac_stats (SACStats): The SAC statistics.
760
+ print_tabular (bool, optional): Whether to print the statistics in a tabular format. Defaults to False.
761
+
762
+ Prints:
763
+ A table with the following columns:
764
+ 1. Op Id(s): The operator index(es).
765
+ 2. Op Name(s): The operator name(s).
766
+ 3. Discarded Mem (%): The percentage of discarded memory.
767
+ 4. Discarded Mem (B): The discarded memory in bytes.
768
+ 5. Recomp time (%): The percentage of recomputed time.
769
+ 6. Recomp time (ms): The recomputed time in milliseconds.
770
+ 7. MSPS: The memory per second.
771
+ 8. Always Stored: A flag indicating whether the operator is always stored.
772
+ 9. Always Recomputed: A flag indicating whether the operator is always recomputed.
773
+
774
+ If print_tabular is True, the table is printed in a tabular format.
775
+ Otherwise, the table is printed in a plain text format.
776
+ """
777
+ table_data = []
778
+ total_memory, total_runtime = sum(sac_stats.memory), sum(sac_stats.runtimes)
779
+ discarded_mem: int = 0
780
+ recomp_runtime: float = 0.0
781
+
782
+ def append_row(
783
+ op_indices: set[int],
784
+ func_names: set[str],
785
+ msps: float | None = None,
786
+ stored: bool | None = False,
787
+ recomputed: bool | None = False,
788
+ ) -> None:
789
+ row = [
790
+ str(op_indices),
791
+ str(func_names),
792
+ f"{discarded_mem / total_memory:.4f}",
793
+ str(discarded_mem),
794
+ f"{recomp_runtime / total_runtime:.4f}",
795
+ str(recomp_runtime),
796
+ f"{msps:.2e}" if msps is not None else str(nan),
797
+ str(stored),
798
+ str(recomputed),
799
+ ]
800
+ table_data.append(row)
801
+
802
+ stored_ops, recomputed_ops, inplace_op_groups, random_ops_group, msps_meta = (
803
+ greedy_order_meta.stored_ops,
804
+ greedy_order_meta.recomputed_ops,
805
+ greedy_order_meta.inplace_op_groups,
806
+ greedy_order_meta.random_ops_group,
807
+ greedy_order_meta.msps_meta,
808
+ )
809
+
810
+ for op_idx in recomputed_ops:
811
+ op_indices: set[int] = {op_idx}
812
+ if op_idx in inplace_op_groups:
813
+ op_indices.update(inplace_op_groups[op_idx])
814
+ if op_idx in random_ops_group:
815
+ op_indices.update(random_ops_group[op_idx])
816
+ discarded_mem += sum(sac_stats.memory[i] for i in op_indices)
817
+ recomp_runtime += sum(sac_stats.runtimes[i] for i in op_indices)
818
+ func_names = {sac_stats.func_names[i] for i in op_indices}
819
+ append_row(op_indices, func_names, recomputed=True)
820
+
821
+ for cand in msps_meta:
822
+ discarded_mem += cand.memory
823
+ recomp_runtime += cand.runtime
824
+ op_indices = {cand.op_idx}
825
+ if cand.op_idx in inplace_op_groups:
826
+ op_indices.update(inplace_op_groups[cand.op_idx])
827
+ if cand.op_idx in random_ops_group:
828
+ op_indices.update(random_ops_group[cand.op_idx])
829
+ append_row(op_indices, cand.func_names, msps=cand.msps)
830
+
831
+ for op_idx in stored_ops:
832
+ op_indices = {op_idx}
833
+ if op_idx in inplace_op_groups:
834
+ op_indices.update(inplace_op_groups[op_idx])
835
+ if op_idx in random_ops_group:
836
+ op_indices.update(random_ops_group[op_idx])
837
+ discarded_mem += sum(sac_stats.memory[i] for i in op_indices)
838
+ recomp_runtime += sum(sac_stats.runtimes[i] for i in op_indices)
839
+ func_names = {sac_stats.func_names[i] for i in op_indices}
840
+ append_row(op_indices, func_names, stored=True)
841
+
842
+ headers = [
843
+ "Op Id(s)",
844
+ "Op Name(s)",
845
+ "Discarded Mem (%)",
846
+ "Discarded Mem (B)",
847
+ "Recomp time (%)",
848
+ "Recomp time (ms)",
849
+ "MSPS",
850
+ "Always Stored",
851
+ "Always Recomputed",
852
+ ]
853
+ if print_tabular:
854
+ _display_stats_tabular(headers, table_data)
855
+ else:
856
+ max_widths = [0 for _ in range(len(headers))]
857
+ table_data.insert(0, headers)
858
+ for row in table_data:
859
+ for i, elem in enumerate(row):
860
+ max_widths[i] = max(max_widths[i], len(elem))
861
+ for row in table_data:
862
+ print(
863
+ "\t".join(
864
+ [f"{elem:<{max_widths[i]}}" for i, elem in enumerate(row)]
865
+ )
866
+ )
867
+
868
+ def pwlf_sac_tradeoff_curve(
869
+ self,
870
+ n_segments: int = 2,
871
+ save_tradeoff_graphs: bool = False,
872
+ ) -> None:
873
+ """
874
+ Fits a piecewise linear function with the specified sumber of segments to the SAC trade-off curve of
875
+ discarded memory vs recomputation time.
876
+
877
+ Args:
878
+ n_segments (int, optional): The number of segments to be used for fitting the piecewise linear function to
879
+ the trade-off curve. Defaults to 2.
880
+ save_tradeoff_graphs (bool, optional): Whether to save the trade-off graphs to file. Defaults to False.
881
+
882
+ If save_tradeoff_graphs is True, the trade-off graphs are saved to file using the module FQN as the filename.
883
+ """
884
+ for mod_fqn, sac_stats in self.sac_mod_stats.items():
885
+ self.sac_mod_tradeoff_stats[mod_fqn] = self._get_sac_tradeoff_pwlf_stats(
886
+ sac_stats=sac_stats,
887
+ greedy_order_meta=self.sac_mod_greedy_order_meta[mod_fqn],
888
+ n_segments=n_segments,
889
+ save_tradeoff_graph=save_tradeoff_graphs,
890
+ filename=mod_fqn,
891
+ )
892
+
893
+ def display_modulewise_sac_stats(
894
+ self, depth: int = 2, print_tabular: bool = False
895
+ ) -> None:
896
+ """
897
+ Displays the SAC and trade-off statistics for each module.
898
+
899
+ Args:
900
+ depth (int, optional): The maximum depth of modules to display. Defaults to 2.
901
+ print_tabular (bool, optional): Whether to print the statistics in a tabular format. Defaults to False.
902
+
903
+ Prints:
904
+ For each module with depth less than or equal to the specified depth:
905
+ 1. The SAC statistics for the module (using display_sac_stats).
906
+ 2. The SAC trade-off statistics for the module (using display_sac_tradeoff_stats).
907
+
908
+ If print_tabular is True, the statistics are printed in a tabular format.
909
+ Otherwise, the statistics are printed in a plain text format.
910
+ """
911
+ for mod_fqn, sac_stats in self.sac_mod_stats.items():
912
+ mod_depth = mod_fqn.count(".") + 1
913
+ if mod_depth > depth:
914
+ continue
915
+ print(f"Module: {mod_fqn}")
916
+ self.display_sac_stats(sac_stats, print_tabular)
917
+ print(f"AC Trade-off for Module: {mod_fqn} MSPS = Memory/Runtime")
918
+ self.display_sac_tradeoff_stats(
919
+ self.sac_mod_greedy_order_meta[mod_fqn], sac_stats, print_tabular
920
+ )
921
+
922
+ def __call__(self, estimate_mode_type: str) -> Self:
923
+ """
924
+ Sets the estimate mode type.
925
+
926
+ Currently supported modes:
927
+ - "operator-level-benchmark": Estimates runtime using operator benchmarking.
928
+ - "operator-level-cost-model": Estimates runtime using roofline cost model.
929
+
930
+ Args:
931
+ estimate_mode_type (str): The type of estimate mode to use.
932
+
933
+ Returns:
934
+ SACEstimator: The SAC estimator instance.
935
+
936
+ Raises:
937
+ NotImplementedError: If the estimate mode type is not supported.
938
+ """
939
+ if estimate_mode_type == "operator-level-benchmark":
940
+ self._estimate_runtime = RuntimeEstimator._benchmark_estimate
941
+ elif estimate_mode_type == "operator-level-cost-model":
942
+ self._estimate_runtime = RuntimeEstimator._roofline_estimate
943
+ else:
944
+ raise NotImplementedError(
945
+ f"estimate_mode_type {estimate_mode_type} not supported"
946
+ )
947
+ return self
948
+
949
+ def __enter__(self) -> Self: # type: ignore[no-untyped-def]
950
+ fake_mode = active_fake_mode()
951
+ if not isinstance(fake_mode, FakeTensorMode):
952
+ raise AssertionError("SAC Estimator should be called in FakeTensorMode")
953
+ RuntimeEstimator.fake_mode = fake_mode
954
+ self._mod_tracker.register_user_hooks(
955
+ pre_fw_hook=self._pre_fw_hook,
956
+ post_fw_hook=self._post_fw_hook,
957
+ )
958
+ self._mod_tracker.__enter__()
959
+ self._saved_tensor_hook_ctx.__enter__()
960
+ return super().__enter__()
961
+
962
+ def __exit__(self, *args: Any) -> None: # type: ignore[no-untyped-def]
963
+ self._saved_tensor_hook_ctx.__exit__()
964
+ self._mod_tracker.__exit__(*args)
965
+ super().__exit__(*args)
western_ai/usa/pytorch/torch/distributed/_tools/sac_ilp.py ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import math
3
+ from enum import IntEnum
4
+
5
+ from torch.distributed._tools.ilp_utils import Graph, is_submodule
6
+ from torch.distributed._tools.sac_estimator import SACStats
7
+
8
+
9
+ try:
10
+ from pulp import ( # type: ignore[import-untyped,import-not-found]
11
+ lpDot,
12
+ LpInteger,
13
+ LpMaximize,
14
+ LpMinimize,
15
+ LpProblem,
16
+ LpStatus,
17
+ lpSum,
18
+ LpVariable,
19
+ PULP_CBC_CMD,
20
+ value,
21
+ )
22
+ except ImportError as err:
23
+ raise ImportError(
24
+ "Please install pulp package. See: https://github.com/coin-or/pulp."
25
+ ) from err
26
+
27
+ # Create a logger object
28
+ logger = logging.getLogger(__name__)
29
+
30
+ # Set the logging level to INFO
31
+ logger.setLevel(logging.INFO)
32
+
33
+
34
+ def sac_milp(
35
+ graph: Graph,
36
+ memory_budget: float,
37
+ world_size: int = 1,
38
+ ac_units: list[str] | None = None,
39
+ fsdp_units: list[str] | None = None,
40
+ ) -> tuple[dict[str, float], float, int]:
41
+ """
42
+ MILP to decide which modules to AC and how much memory to discard.
43
+ The objective is to minimize recomputation time.
44
+ The constraint is to ensure peak memory is under budget.
45
+
46
+ Args:
47
+ graph: graph representation of the model as a module submodule tree
48
+ where each node is a submodule with memory & runtime stats
49
+ memory_budget: memory budget in GiB
50
+ world_size: number of GPUs. In the case of FSDP, world_size will be
51
+ used to compute the amount of parameter and gradient memory on each rank
52
+ ac_units: a list of user-specified AC units.
53
+ fsdp_units: a list of FSDP units. AC units cannot be supermodules of FSDP units.
54
+
55
+ Returns:
56
+ Dict[str, float]: the optimal SAC solution, mapping from module fqn to
57
+ the percentage of activation memory to **discard**
58
+ float: the recomputation time of the optimal SAC solution
59
+ int: upper bound on the peak memory of the optimal SAC solution.
60
+ note that value of -1 means that the ILP solver failed to find a solution.
61
+
62
+ """
63
+ num_nodes = len(graph.nodes)
64
+ max_runtime = max(
65
+ (graph.nodes[i]["sac_runtime"] for i in range(num_nodes)), default=1
66
+ )
67
+ M = max(10**2, 2 * max_runtime)
68
+ MEM_MULTIPLIER = 2**30
69
+
70
+ # Create a MILP problem
71
+ prob = LpProblem("SAC", LpMinimize)
72
+
73
+ # Create decision variables
74
+ # y_i: indicator for if module i is AC'ed
75
+ y = LpVariable.matrix("y", list(range(num_nodes)), 0, 1, LpInteger)
76
+ # r_i: percentage of discarded activation memory
77
+ r = LpVariable.matrix("r", list(range(num_nodes)), 0, 1)
78
+ # d_i: discarded activation memory for module i
79
+ d = LpVariable.matrix("d", list(range(num_nodes)), 0)
80
+ # a_i: total activation memory at module i
81
+ a = LpVariable.matrix("a", list(range(num_nodes)), 0)
82
+ # m_i: memory at module i, combining parameters, gradients, and activations
83
+ m = LpVariable.matrix("m", list(range(num_nodes)), 0)
84
+ # rcp_i: percentage of recomputation time
85
+ rcp = LpVariable.matrix("rcp", list(range(num_nodes)), 0)
86
+ # rct_i: recomputation time for module i (in ms)
87
+ rct = LpVariable.matrix("rct", list(range(num_nodes)), 0)
88
+ # max_m: peak memory
89
+ max_m = LpVariable("max_m", 0)
90
+
91
+ # Add constraints
92
+ # [Constraint] User specified AC units
93
+ if ac_units:
94
+ ac_units_set = set(ac_units)
95
+ for i in range(num_nodes):
96
+ if graph.nodes[i]["fqn"] not in ac_units_set:
97
+ prob += y[i] == 0
98
+
99
+ # [Constraint] AC units cannot be supmodules of user specified FSDP units
100
+ if fsdp_units:
101
+ for i in range(num_nodes):
102
+ if any(
103
+ is_submodule(fsdp_unit, graph.nodes[i]["fqn"])
104
+ for fsdp_unit in fsdp_units
105
+ ):
106
+ prob += y[i] == 0
107
+
108
+ # [Constraint] No nested AC units
109
+ for i in range(num_nodes):
110
+ for j in range(i + 1, num_nodes):
111
+ if graph.ad_matrix[i][j] == 1:
112
+ prob += y[i] + y[j] <= 1
113
+
114
+ # [Constraint] Do not AC leaf modules
115
+ for i in range(num_nodes):
116
+ if graph.nodes[i]["is_leaf"]:
117
+ prob += y[i] == 0
118
+
119
+ # [Constraint] Express amount of discarded activation memory
120
+ for i in range(num_nodes):
121
+ # There are two measures for activation memory: ACM and IA
122
+ # 1. IA is the activation memory saved when not using AC
123
+ # 2. ACM is the total activation memory, including those
124
+ # that are not typically saved when not using AC
125
+ # Note: ACM >= IA
126
+ if (not graph.nodes[i]["is_leaf"]) and graph.nodes[i][
127
+ "sac_memory"
128
+ ] < graph.nodes[i]["act_fw_per_module"]:
129
+ logger.warning("For module {%s}: ", graph.nodes[i]["fqn"])
130
+ logger.warning(
131
+ "activation memory from memory tracker is {%d},",
132
+ graph.nodes[i]["act_fw_per_module"],
133
+ )
134
+ logger.warning(
135
+ "activation memory from SAC estimator is {%d}.",
136
+ graph.nodes[i]["sac_memory"],
137
+ )
138
+ logger.warning("Something is wrong. Please check!")
139
+ logger.warning("Overriding the latter with the former.")
140
+ graph.nodes[i]["sac_memory"] = graph.nodes[i]["act_fw_per_module"]
141
+ ACM_i = graph.nodes[i]["sac_memory"] / MEM_MULTIPLIER
142
+ IA_i = graph.nodes[i]["act_fw_per_module"] / MEM_MULTIPLIER
143
+ prob += d[i] == ACM_i * r[i] - (ACM_i - IA_i) * y[i]
144
+
145
+ # [Constraint] Ensure correctness of r_i
146
+ # There are two parts to its correctness
147
+ # 1. r_i > 0 only if y_i == 1 (discard only if it is an AC unit)
148
+ # 2. r_i needs to be large enough to cover the difference between
149
+ # ACM and IA. Otherwise, we are not saving any memory
150
+ for i in range(num_nodes):
151
+ prob += y[i] >= r[i]
152
+ if graph.nodes[i]["is_leaf"]:
153
+ continue
154
+ ACM_i = graph.nodes[i]["sac_memory"] / MEM_MULTIPLIER
155
+ IA_i = graph.nodes[i]["act_fw_per_module"] / MEM_MULTIPLIER
156
+ prob += r[i] >= (ACM_i - IA_i) / ACM_i * y[i]
157
+
158
+ # [Constraint] Express total activation memory in the backward pass
159
+ for i in range(num_nodes):
160
+ AG_i = graph.nodes[i]["act_grad_per_module"] / MEM_MULTIPLIER
161
+ TA_i = graph.nodes[i]["act_total"] / MEM_MULTIPLIER
162
+ # related to discarded amount of memory
163
+ pos = graph.nodes[i]["pos_fw_post_order"]
164
+ coeff = [0] * num_nodes
165
+ for p in range(pos):
166
+ j = graph.name2node[graph.fw_post_order[p]]["index"]
167
+ coeff[j] = 1
168
+ prob += a[i] == TA_i + AG_i - lpDot(coeff, d)
169
+
170
+ # [Constraint] Express the total amount of memory at each module
171
+ # Note that unsharded parameters and gradients are not included here
172
+ P_1 = graph.nodes[0]["param_per_module"] / MEM_MULTIPLIER
173
+ for i in range(num_nodes):
174
+ TG_i = graph.nodes[i]["grad_total"] / MEM_MULTIPLIER
175
+ prob += m[i] == a[i] + (P_1 + TG_i) / world_size
176
+
177
+ # [Constraint] Express peak memory
178
+ for i in range(num_nodes):
179
+ prob += max_m >= m[i]
180
+
181
+ # [Constraint] Express percentage of recomputation time
182
+ for i in range(num_nodes):
183
+ for s in range(graph.nodes[i]["n_segments"]):
184
+ slope = graph.nodes[i]["slopes"][s]
185
+ intercept = graph.nodes[i]["intercepts"][s]
186
+ prob += rcp[i] >= slope * r[i] + intercept
187
+
188
+ # [Constraint] Express recomputation time
189
+ # rct_i = (rcp_i * ACT_i) if y_i == 1 else 0
190
+ for i in range(num_nodes):
191
+ ACT_i = graph.nodes[i]["sac_runtime"]
192
+ prob += rct[i] <= M * y[i]
193
+ prob += rct[i] <= ACT_i * rcp[i]
194
+ prob += rct[i] >= ACT_i * rcp[i] - M * (1 - y[i])
195
+
196
+ # [Constraint] Peak memory should be below budget
197
+ prob += max_m <= memory_budget
198
+
199
+ # Set Objeictive
200
+ prob += lpSum(rct)
201
+
202
+ # Solve
203
+ solver = PULP_CBC_CMD(gapRel=0.05, timeLimit=180, msg=0)
204
+ status = prob.solve(solver)
205
+
206
+ # If solver fails, print status and return empty solution
207
+ if status != 1:
208
+ logger.error("Solver failed to find a solution: %s", LpStatus[status])
209
+ return {}, 0, -1
210
+
211
+ # Gather and return solution if optimal solution is found
212
+ ac_decisions = {}
213
+ for i in range(num_nodes):
214
+ if round(y[i].varValue) == 1:
215
+ ac_decisions[graph.nodes[i]["fqn"]] = round(r[i].varValue, 4)
216
+ recomputation_time = round(value(prob.objective), 2)
217
+ peak_mem = round(max_m.varValue * MEM_MULTIPLIER)
218
+
219
+ return ac_decisions, recomputation_time, peak_mem
220
+
221
+
222
+ class SACDecision(IntEnum):
223
+ RECOMPUTE = 0
224
+ SAVE = 1
225
+
226
+
227
+ def get_optimal_checkpointing_policy_per_module(
228
+ sac_stats: SACStats, memory_budget: float
229
+ ) -> list[int]:
230
+ """
231
+ This is adapted from --
232
+ https://github.com/facebookresearch/xformers/blob/c6c0ac31f1b08542a0bc27278c6ed10f825f6963/xformers/checkpoint.py#L375
233
+
234
+ Given the SACStats of a module, including list of operators, their memory, runtimes, and metadata,
235
+ decide via MILP an optimal set of operators to checkpoint under a given ``memory_budget``.
236
+
237
+ Args:
238
+ sac_stats: the SACStats object of the module
239
+ memory_budget: a float between zero and one
240
+
241
+ Returns:
242
+ List[int]: the decision whether each operator should be saved (1) or recomptued (0).
243
+ """
244
+ if not (0 <= memory_budget <= 1):
245
+ raise ValueError(
246
+ f"`memory_budget` must be a float between 0 and 1. Got {memory_budget}."
247
+ )
248
+ num_ops = len(sac_stats.func_names)
249
+
250
+ # Create a MILP problem
251
+ prob = LpProblem("SAC-per-module", LpMaximize)
252
+
253
+ # Create decision variables
254
+ # x[i] = 1 means the i-th operator should be saved, otherwise it should be recomputed
255
+ x = LpVariable.matrix("x", list(range(num_ops)), 0, 1, LpInteger)
256
+
257
+ # Add constraints
258
+ # [Constraint] random ops should be saved if ``force_store_random`` is True
259
+ # otherwise, random ops should either be all recomputed or all saved
260
+ if sac_stats.force_store_random:
261
+ for i in sac_stats.rand_ops:
262
+ prob += x[i] == SACDecision.SAVE.value
263
+ else:
264
+ for i1, i2 in zip(sac_stats.rand_ops[:-1], sac_stats.rand_ops[1:]):
265
+ prob += x[i1] == x[i2]
266
+
267
+ # [Constraint] view-like ops should always be recomputed
268
+ for i in sac_stats.view_like_ops:
269
+ prob += x[i] == SACDecision.RECOMPUTE.value
270
+
271
+ # [Constraint] inplace ops should always be done in conjunction with its parent op
272
+ for op, op_parent in sac_stats.inplace_ops:
273
+ if op != op_parent:
274
+ prob += x[op] == x[op_parent]
275
+ else:
276
+ prob += x[op] == SACDecision.SAVE.value
277
+
278
+ # [Constraint] saved memory should be under the ``memory_budget``
279
+ max_memory = math.ceil(memory_budget * sum(sac_stats.memory))
280
+ prob += lpDot(x, sac_stats.memory) <= max_memory
281
+
282
+ # [Objective] minimize recomputation time, note the ILP is a maximization problem
283
+ # because x[i] == 1 means the op is saved (not recomputed), and thus recomputation
284
+ # time is sum(sac_stats.runtimes) - lpDot(x, sac_stats.runtimes)
285
+ prob += lpDot(x, sac_stats.runtimes)
286
+
287
+ # Solve
288
+ solver = PULP_CBC_CMD(gapRel=0.05, timeLimit=10, msg=0)
289
+ status = prob.solve(solver)
290
+
291
+ # If solver fails, print status and return empty solution
292
+ if status != 1:
293
+ logger.error("Solver failed to find a solution: %s", LpStatus[status])
294
+ return []
295
+
296
+ # Gather and return solution if optimal solution is found
297
+ return [round(x[i].varValue) for i in range(num_ops)]
western_ai/usa/pytorch/torch/distributed/algorithms/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .join import Join, Joinable, JoinHook
western_ai/usa/pytorch/torch/distributed/algorithms/_checkpoint/__init__.py ADDED
File without changes
western_ai/usa/pytorch/torch/distributed/algorithms/_checkpoint/checkpoint_wrapper.py ADDED
@@ -0,0 +1,313 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from abc import ABC, abstractmethod
3
+ from collections.abc import Callable, Iterator
4
+ from enum import auto, Enum
5
+ from functools import partial
6
+ from typing import Any
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+ from torch.autograd.graph import save_on_cpu
11
+ from torch.distributed.utils import _pack_kwargs, _replace_by_prefix, _unpack_kwargs
12
+ from torch.utils._typing_utils import copy_method_params
13
+ from torch.utils.checkpoint import checkpoint as torch_utils_checkpoint
14
+
15
+
16
+ _CHECKPOINT_WRAPPED_MODULE = "_checkpoint_wrapped_module"
17
+ _CHECKPOINT_PREFIX = _CHECKPOINT_WRAPPED_MODULE + "."
18
+
19
+
20
+ class CheckpointImpl(Enum):
21
+ REENTRANT = auto()
22
+ NO_REENTRANT = auto()
23
+
24
+
25
+ class ActivationWrapper(torch.nn.Module, ABC):
26
+ """
27
+ Base class for Activation Checkpoint and Activation Offload.
28
+
29
+ Not meant to be instantiated directly.
30
+ """
31
+
32
+ def __init__(self, mod):
33
+ super().__init__()
34
+ self._checkpoint_wrapped_module = mod
35
+ # state_dict post hook to remove prefix to allow loading into a
36
+ # non-checkpoint wrapped module.
37
+ self._register_state_dict_hook(self._post_state_dict_hook)
38
+ # load_state_dict pre-hook to allow loading back into
39
+ # checkpoint-wrapped module.
40
+ self.register_load_state_dict_pre_hook(self._pre_load_state_dict_hook)
41
+
42
+ @abstractmethod
43
+ def forward(self, *args, **kwargs):
44
+ raise ValueError("Subclasses should implement forward().")
45
+
46
+ def __getattr__(self, name: str) -> Any:
47
+ """Forward missing attributes to wrapped module."""
48
+ try:
49
+ return super().__getattr__(name) # defer to nn.Module's logic
50
+ except AttributeError:
51
+ return getattr(self._checkpoint_wrapped_module, name)
52
+
53
+ def __getitem__(self, key: int) -> Any:
54
+ """Forward indexing calls in case the module is a nn.Sequential."""
55
+ return self._checkpoint_wrapped_module.__getitem__(key) # type: ignore[operator]
56
+
57
+ @copy_method_params(torch.nn.Module.named_parameters)
58
+ def named_parameters(
59
+ self,
60
+ *args,
61
+ **kwargs,
62
+ ) -> Iterator[tuple[str, torch.nn.Parameter]]:
63
+ """
64
+ Override :meth:`named_parameters()` to intercept parameter names.
65
+
66
+ remove all occurrences of ``_CHECKPOINT_PREFIX``.
67
+ """
68
+ for param_name, param in super().named_parameters(*args, **kwargs):
69
+ yield param_name.replace(_CHECKPOINT_PREFIX, ""), param
70
+
71
+ @staticmethod
72
+ def _post_state_dict_hook(
73
+ module: nn.Module,
74
+ state_dict: dict[str, Any],
75
+ prefix: str,
76
+ *args: Any,
77
+ ) -> dict[str, Any]:
78
+ """
79
+ _post_state_dict_hook() is called after the state_dict() of this FSDP module is executed.
80
+
81
+ For ``checkpoint_wrapper``, it will strip checkpoint-wrapped module prefix,
82
+ so that this module can be loaded into non-checkpointed modules.
83
+ It would still be able to be loaded into checkpoint-wrapped modules as this class,
84
+ adds the prefix back before loading the state_dict.
85
+ """
86
+ _replace_by_prefix(state_dict, f"{prefix}{_CHECKPOINT_PREFIX}", prefix)
87
+ return state_dict
88
+
89
+ @staticmethod
90
+ def _pre_load_state_dict_hook(
91
+ module: nn.Module,
92
+ state_dict: dict[str, Any],
93
+ prefix: str,
94
+ *args: Any,
95
+ ) -> None:
96
+ """
97
+ ``_pre_state_dict_hook` is called before ``self._load_from_state_dict()`` is called.
98
+
99
+ For ``checkpoint_wrapper``, it will add back the module
100
+ prefix so that non-checkpointed modules can be loaded into
101
+ checkpoint_wrapper modules properly.
102
+ """
103
+ _replace_by_prefix(state_dict, prefix, prefix + f"{_CHECKPOINT_PREFIX}")
104
+
105
+
106
+ class OffloadWrapper(ActivationWrapper):
107
+ def forward(self, *args, **kwargs):
108
+ with save_on_cpu(pin_memory=True):
109
+ return self._checkpoint_wrapped_module(*args, **kwargs)
110
+
111
+
112
+ class CheckpointWrapper(ActivationWrapper):
113
+ """
114
+ An ``nn.Module`` that wraps another ``nn.Module`` with checkpointing.
115
+
116
+ Note that this module is not meant to be used directly but instead,
117
+ it is to be used through the ``checkpoint_wrapper`` function.
118
+ """
119
+
120
+ def __init__(
121
+ self,
122
+ mod: torch.nn.Module,
123
+ checkpoint_impl: CheckpointImpl = CheckpointImpl.NO_REENTRANT,
124
+ checkpoint_fn=None,
125
+ **checkpoint_fn_kwargs,
126
+ ):
127
+ super().__init__(mod)
128
+ self.checkpoint_impl = checkpoint_impl
129
+ if checkpoint_fn is None:
130
+ # use torch.utils.checkpoint
131
+ self.checkpoint_fn = partial(
132
+ torch_utils_checkpoint,
133
+ use_reentrant=(self.checkpoint_impl == CheckpointImpl.REENTRANT),
134
+ **checkpoint_fn_kwargs,
135
+ )
136
+ else:
137
+ # Construct user-specified checkpoint function.
138
+ self.checkpoint_fn = partial(
139
+ checkpoint_fn,
140
+ **checkpoint_fn_kwargs,
141
+ )
142
+
143
+ def forward(self, *args, **kwargs):
144
+ # Support keyword arguments for reentrant checkpoint. Note that this
145
+ # only works if user has specified self.checkpoint_impl and is not
146
+ # using their own custom checkpoint_fn.
147
+ if self.checkpoint_impl == CheckpointImpl.REENTRANT and kwargs != {}:
148
+ # Pack the args and kwargs
149
+ flat_args, kwarg_keys = _pack_kwargs(*args, **kwargs)
150
+
151
+ # Function that only takes (packed) args, but can unpack them
152
+ # into the original args and kwargs for the checkpointed
153
+ # function, and runs that function.
154
+ def my_function(*inputs):
155
+ # unpack back into args and kwargs
156
+ unpacked_args, unpacked_kwargs = _unpack_kwargs(inputs, kwarg_keys)
157
+ # run original module
158
+ return self._checkpoint_wrapped_module(
159
+ *unpacked_args, **unpacked_kwargs
160
+ )
161
+
162
+ # Pass the function that only takes packed args into reentrant
163
+ # checkpoint API.
164
+ return self.checkpoint_fn( # type: ignore[misc]
165
+ my_function,
166
+ *flat_args,
167
+ )
168
+ else:
169
+ return self.checkpoint_fn( # type: ignore[misc]
170
+ self._checkpoint_wrapped_module, *args, **kwargs
171
+ )
172
+
173
+
174
+ def offload_wrapper(module: torch.nn.Module) -> torch.nn.Module:
175
+ """
176
+ Wrap a module for activation offloading to CPU.
177
+
178
+ Offloads intermediate activations to the CPU for modules wrapped with this function.
179
+ Wrappers with activation offload can be composed with ones that do recomputation-based
180
+ checkpoint to trade off increased compute versus increased CPU
181
+ memory usage and additional H2D transfers.
182
+
183
+ Usage::
184
+ offloaded_module = offload_wrapper(module)
185
+ outputs = checkpointed_module(inputs)
186
+ Args:
187
+ module (nn.Module):
188
+ The module to be wrapped
189
+ Returns:
190
+ (nn.Module):
191
+ Wrapped module
192
+ """
193
+ return OffloadWrapper(module)
194
+
195
+
196
+ def checkpoint_wrapper(
197
+ module: torch.nn.Module,
198
+ checkpoint_impl: CheckpointImpl = CheckpointImpl.NO_REENTRANT,
199
+ checkpoint_fn=None,
200
+ **checkpoint_fn_kwargs,
201
+ ) -> torch.nn.Module:
202
+ """
203
+ Wrap a module for activation checkpointing.
204
+
205
+ If the module is wrapped with this function, all subsequent calls to the module will,
206
+ automatically perform checkpointing without the user having to explicitly call ``checkpoint`` function.
207
+
208
+ Usage::
209
+ checkpointed_module = checkpoint_wrapper(module)
210
+ outputs = checkpointed_module(inputs)
211
+ Args:
212
+ module (nn.Module):
213
+ The module to be wrapped
214
+ checkpoint_impl (Optional[CheckpointImpl]):
215
+ The checkpointing implementation to use. Note that this will only
216
+ be passed into the ``torch.utils.checkpoint.checkpoint``
217
+ implementation, and is ignored if a custom ``checkpoint_fn`` is
218
+ specified. Note that for implementations using reentrant checkpoint
219
+ from ``torch.utils.checkpoint``, keyword arguments will only be
220
+ supported if ``checkpoint_impl`` is passed as ``CheckpointImpl.REENTRANT`.
221
+ checkpoint_fn (Optional[Callable]):
222
+ Functional checkpoint implementation to use. If this is specified,
223
+ it will be used over the default ``torch.utils.checkpoint.checkpoint``
224
+ implementation and the `checkpoint_impl` argument will be ignored.
225
+ **checkpoint_fn_kwargs: (Dict[str, Any]): Keyword arguments to pass into `checkpoint_fn`.
226
+
227
+ Returns:
228
+ (nn.Module):
229
+ Wrapped module
230
+ """
231
+ return CheckpointWrapper(
232
+ module,
233
+ checkpoint_impl,
234
+ checkpoint_fn,
235
+ **checkpoint_fn_kwargs,
236
+ )
237
+
238
+
239
+ def apply_activation_checkpointing(
240
+ model,
241
+ checkpoint_wrapper_fn=checkpoint_wrapper,
242
+ check_fn=lambda _: True,
243
+ auto_wrap_policy: Callable[[nn.Module, bool, int], bool] | None = None,
244
+ ):
245
+ """
246
+ Apply :func:`checkpoint_wrapper` to modules within `model` based on a user-defined configuration.
247
+
248
+ For each module within `model`, the `check_fn` is used to decide
249
+ whether `module` should be wrapped with :func:`checkpoint_wrapper` or not.
250
+
251
+ Note::
252
+ This function modifies `model` in place and replaces appropriate layers with
253
+ their checkpoint-wrapped modules.
254
+ Note::
255
+ This function will not wrap the overall root module. If this is needed, please directly use
256
+ :func:`checkpoint_wrapper` or :func:`offload_wrapper`.
257
+ Usage::
258
+ model = nn.Sequential(
259
+ nn.Linear(10, 10), nn.Linear(10, 10), nn.Linear(10, 10)
260
+ )
261
+ check_fn = lambda l: isinstance(l, nn.Linear)
262
+ # checkpoint activations
263
+ apply_activation_checkpointing(model, checkpoint_wrapper_fn=checkpoint_wrapper, check_fn=check_fn)
264
+ # Or offload activations to CPU
265
+ apply_activation_checkpointing(model, checkpoint_wrapper_fn=offload_wrapper, check_fn=check_fn)
266
+ Args:
267
+ model (nn.Module):
268
+ The model whose submodules should be wrapped with activation checkpointing.
269
+ checkpoint_wrapper_fn (Optional[Callable[nn.Module]])
270
+ A ``Callable`` which will wrap modules
271
+ check_fn (Optional[Callable[nn.Module, nn.Module]])
272
+ A lambda function which will be passed each child submodule of ``model`` and returns
273
+ ``True`` or ``False`` depending on whether the submodule should be wrapped.
274
+ auto_wrap_policy (Optional[Callable[[nn.Module, bool, int], bool]]): A policy to wrap model's
275
+ submodules with AC. Note that if this is specified, it takes precedence over ``check_fn``.
276
+ Returns: None (`model` is modified inplace)
277
+ """
278
+ # TODO: Importing inside function to avoid circular import issue between FSDP and
279
+ # checkpoint_wrapper. This can be resolved once wrap() APIs are decoupled from FSDP code.
280
+ from torch.distributed.fsdp._wrap_utils import _construct_wrap_fn, _post_order_apply
281
+ from torch.distributed.fsdp.wrap import (
282
+ _Policy,
283
+ _recursive_wrap,
284
+ lambda_auto_wrap_policy,
285
+ )
286
+
287
+ policy = (
288
+ auto_wrap_policy
289
+ if auto_wrap_policy is not None
290
+ else partial(lambda_auto_wrap_policy, lambda_fn=check_fn)
291
+ )
292
+ if not callable(policy):
293
+ if not isinstance(policy, _Policy):
294
+ raise ValueError(
295
+ f"Expected {policy} to be callable or be a pre-defined wrap policy"
296
+ )
297
+ target_module_to_kwargs = policy._run_policy(
298
+ model, ignored_modules=set(), root_kwargs={}
299
+ )
300
+ wrap_fn = _construct_wrap_fn(
301
+ model, target_module_to_kwargs, checkpoint_wrapper_fn
302
+ )
303
+ _post_order_apply(model, wrap_fn)
304
+ return
305
+
306
+ _recursive_wrap(
307
+ module=model,
308
+ auto_wrap_policy=policy, # type: ignore[arg-type]
309
+ wrapper_cls=checkpoint_wrapper_fn,
310
+ ignored_modules=set(),
311
+ ignored_params=set(),
312
+ only_wrap_children=True,
313
+ )
western_ai/usa/pytorch/torch/distributed/algorithms/_comm_hooks/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ from . import default_hooks as default
2
+
3
+
4
+ LOW_PRECISION_HOOKS = [
5
+ default.fp16_compress_hook,
6
+ default.bf16_compress_hook,
7
+ ]
western_ai/usa/pytorch/torch/distributed/algorithms/_comm_hooks/default_hooks.py ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import functools
3
+
4
+ import torch
5
+ import torch.distributed as dist
6
+
7
+
8
+ class DefaultState:
9
+ r"""
10
+ Stores state needed to perform the default communication algorithm within a communication hook.
11
+
12
+ Args:
13
+ process_group (ProcessGroup): The process group to be used.
14
+ """
15
+
16
+ __slots__ = [
17
+ "process_group",
18
+ "world_size",
19
+ "gradient_predivide_factor",
20
+ "gradient_postdivide_factor",
21
+ ]
22
+
23
+ def __init__(self, process_group: dist.ProcessGroup):
24
+ if process_group is None:
25
+ raise ValueError(f"Expected to pass in an explicit ProcessGroup to {self}.")
26
+ self.process_group = process_group
27
+ self.world_size = dist.get_world_size(process_group)
28
+ # Setting two factors `self.gradient_predivide_factor`
29
+ # and `self.gradient_postdivide_factor` to avoid underflow and overflow
30
+ self.gradient_predivide_factor = self._get_gradient_predivide_factor(
31
+ self.world_size
32
+ )
33
+ self.gradient_postdivide_factor = (
34
+ self.world_size / self.gradient_predivide_factor
35
+ )
36
+
37
+ @staticmethod
38
+ def _get_gradient_predivide_factor(world_size: int) -> float:
39
+ factor: int = 1
40
+ while world_size % factor == 0 and world_size / factor > factor:
41
+ factor *= 2
42
+ return float(factor)
43
+
44
+
45
+ class LowPrecisionState(DefaultState):
46
+ r"""
47
+ Stores state needed to perform gradient communication in a lower precision within a communication hook.
48
+
49
+ Communication hook will cast gradients back to the original
50
+ parameter precision specified by ``parameter_type`` (default: torch.float32).
51
+ Builds on top of the :class:`DefaultState`.
52
+
53
+ Args:
54
+ parameter_type (torch.dtype): The precision of model's parameters.
55
+ Required for a hook to cast gradients back to a parameter's precision.
56
+ """
57
+
58
+ __slots__ = [
59
+ "parameter_type",
60
+ ]
61
+
62
+ def __init__(
63
+ self,
64
+ process_group,
65
+ parameter_type=torch.float32,
66
+ ):
67
+ super().__init__(process_group)
68
+ self.parameter_type = parameter_type
69
+
70
+
71
+ def _decompress(state: LowPrecisionState, grad: torch.Tensor):
72
+ """
73
+ Casts gradients back to full parameter precision so that further computation happens in full precision.
74
+ """
75
+ orig_grad_data = grad.data
76
+ grad.data = grad.data.to(state.parameter_type)
77
+ device_type = ""
78
+ try:
79
+ if grad.device.type == "privateuse1":
80
+ device_type = torch._C._get_privateuse1_backend_name()
81
+ else:
82
+ device_type = grad.device.type
83
+ backend = getattr(torch, device_type)
84
+ except AttributeError as e:
85
+ raise AttributeError(
86
+ f"Device {grad.device} does not have a \
87
+ corresponding backend registered as 'torch.device_type'."
88
+ ) from e
89
+
90
+ # Don't let this memory get reused until after the transfer.
91
+ orig_grad_data.record_stream(backend.current_stream()) # type: ignore[arg-type]
92
+
93
+
94
+ def allreduce_hook(state: DefaultState, grad: torch.Tensor):
95
+ r"""
96
+ Implement the FSDP communication hook for ``all_reduce`` algorithm and a necessary pre- and post-division of gradients.
97
+
98
+ Args:
99
+ state (DefaultState): State information, configures pre- and post-division factors.
100
+ grad (torch.Tensor): A gradient for the local batch that needs to be communicated across ranks.
101
+ """
102
+ # Average grad by pre-division factor. Together pre- and post-division factors
103
+ # lead to an overall averaging by world_size, required for consistency with PyTorch DDP.
104
+ # This is a two-step process to avoid potential underflow and overflow.
105
+ if state.gradient_predivide_factor > 1:
106
+ grad.div_(state.gradient_predivide_factor)
107
+ dist.all_reduce(grad, group=state.process_group)
108
+ # Average grad by post-division factor.
109
+ if state.gradient_postdivide_factor > 1:
110
+ grad.div_(state.gradient_postdivide_factor)
111
+
112
+
113
+ def reduce_scatter_hook(state: DefaultState, grad: torch.Tensor, output: torch.Tensor):
114
+ r"""
115
+ Implement the FSDP communication hook for ``reduce_scatter`` algorithm.
116
+
117
+ For sharded FSDP strategies and a necessary pre- and post-division of gradients.
118
+
119
+ Args:
120
+ state (DefaultState): State information, configures pre- and post-division factors.
121
+ grad (torch.Tensor): An unsharded gradient for the local batch that needs to be
122
+ communicated across ranks.
123
+ output (torch.Tensor): Stores a single shard of the gradient after ``reduce_scatter``.
124
+ """
125
+ # Average grad by pre-division factor.
126
+ if state.gradient_predivide_factor > 1:
127
+ grad.div_(state.gradient_predivide_factor)
128
+ dist.reduce_scatter_single(output, grad, group=state.process_group)
129
+ # Average grad's shard by post-division factor.
130
+ if state.gradient_postdivide_factor > 1:
131
+ output.div_(state.gradient_postdivide_factor)
132
+
133
+
134
+ def _low_precision_hook(
135
+ prec: torch.dtype,
136
+ state: LowPrecisionState,
137
+ grad: torch.Tensor,
138
+ output: torch.Tensor | None,
139
+ ):
140
+ if grad.dtype != prec:
141
+ grad.data = grad.data.to(prec)
142
+ if output is not None:
143
+ if output.dtype != prec:
144
+ output.data = output.data.to(prec)
145
+ reduce_scatter_hook(state, grad, output)
146
+ _decompress(state, output)
147
+ else:
148
+ allreduce_hook(state, grad)
149
+ _decompress(state, grad)
150
+
151
+
152
+ def fp16_compress_hook(
153
+ state: LowPrecisionState, grad: torch.Tensor, output: torch.Tensor | None = None
154
+ ):
155
+ r"""
156
+ Implement FSDP communication hook for a simple gradient compression approach.
157
+ Casts ``grad`` to half-precision floating-point format (``torch.float16``).
158
+
159
+ It also averages gradients by ``world_size`` in two steps: first it pre-divides gradients by a
160
+ ``state.gradient_predivide_factor``, and after a communication step (``all_reduce`` or ``reduce_scatter``)
161
+ gradients are averaged by a ``state.gradient_postdivide_factor``.
162
+ Once post-division is done, compressed gradients are casted back to parameters' precision.
163
+
164
+ Args:
165
+ state (LowPrecisionState): State information, configures pre- and post-division factors, parameters' precision.
166
+ grad (torch.Tensor): A gradient for the local batch that needs to be communicated across ranks in a lower precision.
167
+ output (torch.Tensor): Stores a single shard of the gradient after ``reduce_scatter``.
168
+ """
169
+ fp16_hook = functools.partial(_low_precision_hook, torch.float16)
170
+ return fp16_hook(state, grad, output)
171
+
172
+
173
+ def bf16_compress_hook(
174
+ state: LowPrecisionState, grad: torch.Tensor, output: torch.Tensor | None = None
175
+ ):
176
+ r"""
177
+ Implement FSDP communication hook for a simple gradient compression approach .
178
+ Casts ``grad`` to half-precision floating-point format.
179
+
180
+ It also averages gradients by ``world_size`` in two steps: first it pre-divides gradients by a
181
+ ``state.gradient_predivide_factor``, and after a communication step (``all_reduce`` or ``reduce_scatter``)
182
+ gradients are averaged by a ``state.gradient_postdivide_factor``.
183
+ Once post-division is done, compressed gradients are casted back to parameters' precision.
184
+
185
+ Args:
186
+ state (LowPrecisionState): State information, configures pre- and post-division factors, parameters' precision.
187
+ grad (torch.Tensor): A gradient for the local batch that needs to be communicated across ranks in a lower precision.
188
+ output (torch.Tensor): Stores a single shard of the gradient after ``reduce_scatter``.
189
+ """
190
+ bf16_hook = functools.partial(_low_precision_hook, torch.bfloat16)
191
+ return bf16_hook(state, grad, output)
western_ai/usa/pytorch/torch/distributed/algorithms/_optimizer_overlap/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .optimizer_overlap import _as_overlapped_optim
western_ai/usa/pytorch/torch/distributed/algorithms/_optimizer_overlap/optimizer_overlap.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import inspect
3
+ from abc import ABC, abstractmethod
4
+
5
+ from torch.distributed.algorithms.ddp_comm_hooks.default_hooks import allreduce_hook
6
+ from torch.distributed.algorithms.ddp_comm_hooks.optimizer_overlap_hooks import (
7
+ _hook_then_optimizer,
8
+ _OptimizerHookState,
9
+ )
10
+ from torch.distributed.fsdp import FullyShardedDataParallel
11
+ from torch.distributed.optim import as_functional_optim
12
+ from torch.nn.parallel import DistributedDataParallel
13
+ from torch.optim import Optimizer
14
+
15
+
16
+ # Contains the mappings between the regular and overlapped optimizer types.
17
+ _registered_overlapped_optims: dict[type, type] = {}
18
+
19
+
20
+ def register_overlapped(optim_cls):
21
+ def decorator(target_overlapped_optim_cls):
22
+ if target_overlapped_optim_cls in _registered_overlapped_optims:
23
+ raise ValueError(
24
+ f"{target_overlapped_optim_cls} already registered with optim_cls "
25
+ f"{_registered_overlapped_optims[optim_cls]} {optim_cls}, trying to"
26
+ f"re-register it for {optim_cls} is not supported."
27
+ )
28
+ _registered_overlapped_optims[optim_cls] = target_overlapped_optim_cls
29
+ return target_overlapped_optim_cls
30
+
31
+ return decorator
32
+
33
+
34
+ class OverlappedOptimizer(ABC):
35
+ def __init__(self, optim_cls: type) -> None:
36
+ """
37
+ Initialize the OverlappedOptimizer.
38
+
39
+ Overlappedoptimizer is a base class that child classes can implement to
40
+ specify how different optimizers will register themselves with DDP.
41
+ """
42
+ self.optim_cls = optim_cls
43
+
44
+ @abstractmethod
45
+ def register_ddp(self, ddp: DistributedDataParallel) -> None:
46
+ """Registers the overlapped optimizer with DDP."""
47
+ raise NotImplementedError(
48
+ f"{self.__class__.__name__} does not support overlapped DDP."
49
+ )
50
+
51
+ @abstractmethod
52
+ def register_fsdp(self, fsdp: FullyShardedDataParallel) -> None:
53
+ """Registers the overlapped optimizer with FSDP."""
54
+ raise NotImplementedError(
55
+ f"{self.__class__.__name__} does not support overlapped FSDP."
56
+ )
57
+
58
+
59
+ @register_overlapped(Optimizer)
60
+ class _OverlappedStandardOptimizer(OverlappedOptimizer):
61
+ """Overlaps a regular ``Optimizer``."""
62
+
63
+ def __init__(self, optim_cls: type, params, *optim_args, **optim_kwargs) -> None:
64
+ super().__init__(optim_cls)
65
+ f_optim = as_functional_optim(self.optim_cls, *optim_args, **optim_kwargs)
66
+ self._opt_hook_state = _OptimizerHookState(f_optim, params)
67
+
68
+ def register_ddp(self, ddp_inst: DistributedDataParallel):
69
+ # NOTE: using a custom communication hook and fused optimizer is not
70
+ # yet supported.
71
+ ddp_inst.register_comm_hook( # type: ignore[operator]
72
+ None, # wrapped hook state
73
+ _hook_then_optimizer(allreduce_hook, self._opt_hook_state),
74
+ )
75
+
76
+ # TODO: register_fsdp once FSDP supports communication hook.
77
+ def register_fsdp(self, fsdp: FullyShardedDataParallel) -> None:
78
+ """Register the overlapped optimizer with FSDP."""
79
+ raise NotImplementedError(
80
+ f"{self.__class__.__name__} does not support overlapped FSDP."
81
+ )
82
+
83
+
84
+ def _as_overlapped_optim(optim_cls: type, params, *args, **kwargs):
85
+ """Return a new ``OverlappedOptimizer`` instance that supports ``optim_cls``."""
86
+ for clz in inspect.getmro(optim_cls):
87
+ try:
88
+ return _registered_overlapped_optims[clz](
89
+ optim_cls, params, *args, **kwargs
90
+ )
91
+ except KeyError:
92
+ pass
93
+
94
+ # Fallback to standard overlapped optimizer, which will raise errors if user
95
+ # is attempting to use an unsupported optimizer.
96
+ return _OverlappedStandardOptimizer(optim_cls, params, *args, **kwargs)
western_ai/usa/pytorch/torch/distributed/algorithms/_quantization/__init__.py ADDED
File without changes
western_ai/usa/pytorch/torch/distributed/algorithms/_quantization/quantization.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import functools
3
+ from enum import Enum
4
+
5
+ import torch
6
+ import torch.distributed as dist
7
+
8
+
9
+ TORCH_HALF_MIN = torch.finfo(torch.float16).min
10
+ TORCH_HALF_MAX = torch.finfo(torch.float16).max
11
+
12
+
13
+ class DQuantType(Enum):
14
+ """
15
+ Different quantization methods for auto_quantize API are identified here.
16
+
17
+ auto_quantize API currently supports fp16 and bfp16 methods.
18
+ """
19
+
20
+ FP16 = ("fp16",)
21
+ BFP16 = "bfp16"
22
+
23
+ def __str__(self) -> str:
24
+ return self.value
25
+
26
+
27
+ def _fp32_to_fp16_with_clamp(tensor: torch.Tensor) -> torch.Tensor:
28
+ return torch.clamp(tensor, TORCH_HALF_MIN, TORCH_HALF_MAX).half()
29
+
30
+
31
+ def _quantize_tensor(tensor, qtype):
32
+ if not isinstance(tensor, torch.Tensor):
33
+ raise RuntimeError(
34
+ f"_quantize_tensor expecting torch.Tensor as input but found {type(tensor)}"
35
+ )
36
+ if qtype == DQuantType.FP16:
37
+ return _fp32_to_fp16_with_clamp(tensor)
38
+ elif qtype == DQuantType.BFP16:
39
+ return torch.ops.quantization._FloatToBfloat16Quantized(tensor)
40
+ else:
41
+ raise RuntimeError(f"Quantization type {qtype} is not supported")
42
+
43
+
44
+ def _quantize_tensor_list(tensor_list, qtype):
45
+ if not isinstance(tensor_list, list) or not all(
46
+ isinstance(p, torch.Tensor) for p in tensor_list
47
+ ):
48
+ raise RuntimeError(
49
+ f"_quantize_tensor_list expecting list of torch.Tensor as input but found {type(tensor_list)}"
50
+ )
51
+ quantized_tensor_list = [_quantize_tensor(t, qtype) for t in tensor_list]
52
+ return quantized_tensor_list
53
+
54
+
55
+ def _dequantize_tensor(tensor, qtype, quant_loss=None):
56
+ if not isinstance(tensor, torch.Tensor):
57
+ raise RuntimeError(
58
+ f"_dequantize_tensor expecting torch.Tensor as input but found {type(tensor)}"
59
+ )
60
+ if qtype == DQuantType.FP16:
61
+ if tensor.dtype != torch.float16:
62
+ raise RuntimeError(
63
+ f"tensor dtype is {tensor.dtype} while expected to be FP16."
64
+ )
65
+ elif tensor.dtype == torch.float16 and quant_loss is None:
66
+ return tensor.float()
67
+ else:
68
+ # pyrefly: ignore [unsupported-operation]
69
+ return tensor.float() / quant_loss
70
+ elif qtype == DQuantType.BFP16:
71
+ if tensor.dtype != torch.float16:
72
+ raise RuntimeError(
73
+ f"tensor dtype is {tensor.dtype} while expected to be FP16."
74
+ )
75
+ else:
76
+ return torch.ops.quantization._Bfloat16QuantizedToFloat(tensor)
77
+ else:
78
+ raise RuntimeError(f"Quantization type {qtype} is not supported")
79
+
80
+
81
+ def _dequantize_tensor_list(tensor_list, qtype, quant_loss=None):
82
+ if not isinstance(tensor_list, list) or not all(
83
+ isinstance(p, torch.Tensor) for p in tensor_list
84
+ ):
85
+ raise RuntimeError(
86
+ f"_dequantize_tensor_list expecting list of torch.Tensor as input but found {type(tensor_list)}"
87
+ )
88
+ dequantized_tensor_list = [_dequantize_tensor(t, qtype) for t in tensor_list]
89
+ return dequantized_tensor_list
90
+
91
+
92
+ def auto_quantize(func, qtype, quant_loss=None):
93
+ """
94
+ Quantize the input tensors, choose the precision types, and pass other necessary arguments and then dequantizes the output.
95
+
96
+ Currently it only supports:
97
+ . FP16 and BFP16 quantization method supported for gloo and nccl backends
98
+ . all_gather, all_to_all collective ops
99
+ Note: BFP16 only supports 2D tensors.
100
+ Args:
101
+ func (Callable): A function representing collective operations.
102
+ qtype (QuantType): Quantization method
103
+ quant_loss (float, optional): This can be used to improve accuracy in the dequantization.
104
+ Returns:
105
+ (Callable): the same collective as func but enables automatic quantization/dequantization.
106
+ """
107
+
108
+ @functools.wraps(func)
109
+ def wrapper(*args, **kwargs):
110
+ group = kwargs.get("group")
111
+ async_op = kwargs.get("async_op", False)
112
+ if async_op is True:
113
+ raise RuntimeError("The async_op=True mode is not supported yet.")
114
+ if func is dist.all_gather:
115
+ tensors = args[0]
116
+ input_tensors = _quantize_tensor(args[1], qtype)
117
+ out_tensors = _quantize_tensor_list(tensors, qtype)
118
+ dist.all_gather(out_tensors, input_tensors, group=group, async_op=async_op)
119
+ for i, t in enumerate(
120
+ _dequantize_tensor_list(out_tensors, qtype, quant_loss=quant_loss)
121
+ ):
122
+ tensors[i] = t
123
+
124
+ elif func is dist.all_to_all:
125
+ tensors = args[0]
126
+ input_tensors = _quantize_tensor_list(args[1], qtype)
127
+ out_tensors = _quantize_tensor_list(tensors, qtype)
128
+ dist.all_to_all(out_tensors, input_tensors, group=group, async_op=async_op)
129
+ for i, t in enumerate(
130
+ _dequantize_tensor_list(out_tensors, qtype, quant_loss=quant_loss)
131
+ ):
132
+ tensors[i] = t
133
+
134
+ elif func is dist.all_to_all_single:
135
+ tensors = args[0]
136
+ out_splits = kwargs.get("out_splits")
137
+ in_splits = kwargs.get("in_splits")
138
+ # Quantizing the input/output tensor
139
+ input_tensors = _quantize_tensor(args[1], qtype)
140
+ out_tensors = _quantize_tensor(tensors, qtype)
141
+ dist.all_to_all_single(
142
+ out_tensors, input_tensors, out_splits, in_splits, group=group
143
+ )
144
+ for i, t in enumerate(
145
+ _dequantize_tensor(out_tensors, qtype, quant_loss=quant_loss)
146
+ ):
147
+ tensors[i] = t
148
+ else:
149
+ raise RuntimeError(f"The collective op {func} is not supported yet")
150
+
151
+ return wrapper
western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/__init__.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ from enum import Enum
4
+ from functools import partial
5
+
6
+
7
+ # To suppress FutureWarning from partial since 3.13
8
+ if sys.version_info >= (3, 11):
9
+ # member was introduced in Python 3.11
10
+ from enum import member
11
+
12
+ def _enum_member(x):
13
+ return member(x)
14
+ else:
15
+
16
+ def _enum_member(x):
17
+ return x
18
+
19
+
20
+ import torch.distributed as dist
21
+
22
+ from . import (
23
+ debugging_hooks as debugging,
24
+ default_hooks as default,
25
+ optimizer_overlap_hooks as optimizer_overlap,
26
+ powerSGD_hook as powerSGD,
27
+ quantization_hooks as quantization,
28
+ )
29
+
30
+
31
+ __all__ = ["DDPCommHookType", "register_ddp_comm_hook"]
32
+
33
+
34
+ def _ddp_comm_hook_wrapper(comm_hook, model, state):
35
+ model.register_comm_hook(state, comm_hook)
36
+
37
+
38
+ def _powerSGD_comm_hook_wrapper(
39
+ comm_hook,
40
+ model,
41
+ state,
42
+ matrix_approximation_rank,
43
+ start_powerSGD_iter=1_000,
44
+ ):
45
+ """
46
+ Wrap PowerSGD communication hook.
47
+
48
+ To be consistent with the wrappers of other DDP comm hooks, the input state only needs to be a process group,
49
+ which will be wrapped up with other state info.
50
+ """
51
+ powerSGD_state = powerSGD.PowerSGDState(
52
+ process_group=state,
53
+ matrix_approximation_rank=matrix_approximation_rank,
54
+ start_powerSGD_iter=start_powerSGD_iter,
55
+ )
56
+ model.register_comm_hook(powerSGD_state, comm_hook)
57
+
58
+
59
+ class DDPCommHookType(Enum):
60
+ """
61
+ Enumerate ``ddp_comm_hooks`` and ``ddp_comm_hook_wrapper`` communucation hook types.
62
+
63
+ DDPCommHookType enumerates the hooks of ``torch.distributed.algorithms.ddp_comm_hooks``
64
+ as names and ``ddp_comm_hook_wrapper`` partials with hook specified. As an example,
65
+ you can register allreduce hook by
66
+ ``DDPCommHookType.ALLREDUCE.value(model=model, state=process_group)``.
67
+ """
68
+
69
+ ALLREDUCE = _enum_member(
70
+ partial(_ddp_comm_hook_wrapper, comm_hook=default.allreduce_hook)
71
+ )
72
+ FP16_COMPRESS = _enum_member(
73
+ partial(_ddp_comm_hook_wrapper, comm_hook=default.fp16_compress_hook)
74
+ )
75
+ BF16_COMPRESS = _enum_member(
76
+ partial(_ddp_comm_hook_wrapper, comm_hook=default.bf16_compress_hook)
77
+ )
78
+ QUANTIZE_PER_TENSOR = _enum_member(
79
+ partial(
80
+ _ddp_comm_hook_wrapper, comm_hook=quantization.quantization_pertensor_hook
81
+ )
82
+ )
83
+ QUANTIZE_PER_CHANNEL = _enum_member(
84
+ partial(
85
+ _ddp_comm_hook_wrapper, comm_hook=quantization.quantization_perchannel_hook
86
+ )
87
+ )
88
+ POWER_SGD = _enum_member(
89
+ partial(
90
+ _powerSGD_comm_hook_wrapper,
91
+ comm_hook=powerSGD.powerSGD_hook,
92
+ matrix_approximation_rank=1,
93
+ )
94
+ )
95
+ # Rank-2 PowerSGD can give a higher accuracy than the default rank-1 version,
96
+ # but it runs slower and consumes more memory.
97
+ POWER_SGD_RANK2 = _enum_member(
98
+ partial(
99
+ _powerSGD_comm_hook_wrapper,
100
+ comm_hook=powerSGD.powerSGD_hook,
101
+ matrix_approximation_rank=2,
102
+ )
103
+ )
104
+ # Batching can lead to a faster training at the cost of accuracy.
105
+ BATCHED_POWER_SGD = _enum_member(
106
+ partial(
107
+ _powerSGD_comm_hook_wrapper,
108
+ comm_hook=powerSGD.batched_powerSGD_hook,
109
+ matrix_approximation_rank=1,
110
+ )
111
+ )
112
+ BATCHED_POWER_SGD_RANK2 = _enum_member(
113
+ partial(
114
+ _powerSGD_comm_hook_wrapper,
115
+ comm_hook=powerSGD.batched_powerSGD_hook,
116
+ matrix_approximation_rank=2,
117
+ )
118
+ )
119
+ NOOP = _enum_member(
120
+ partial(
121
+ _ddp_comm_hook_wrapper,
122
+ comm_hook=debugging.noop_hook,
123
+ )
124
+ )
125
+
126
+
127
+ def register_ddp_comm_hook(comm_hook_type: DDPCommHookType, model, state=None):
128
+ """
129
+ Register ``ddp_comm_hooks`` to DDP model.
130
+
131
+ Registers the hooks of ``torch.distributed.algorithms.ddp_comm_hooks``
132
+ to the DDP model. User can specify the type of hook as an enum
133
+ ``DDPCommHookType`` type using ``comm_hook_type`` input. State input will
134
+ be passed to the model.
135
+ Uses Python comm hook implementations.
136
+
137
+ Example::
138
+ >>> # xdoctest: +SKIP
139
+ >>> register_ddp_comm_hook(DDPCommHookType.FP16_COMPRESS, model, state)
140
+ """
141
+ comm_hook_type.value(model=model, state=state)
western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py ADDED
@@ -0,0 +1,464 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import weakref
3
+ from collections.abc import Callable
4
+ from typing import Any
5
+
6
+ import torch
7
+ import torch.distributed as dist
8
+ from torch.distributed.optim import ZeroRedundancyOptimizer
9
+ from torch.distributed.optim.zero_redundancy_optimizer import _OverlapStatus
10
+ from torch.nn.parallel.distributed import DistributedDataParallel
11
+
12
+
13
+ __all__ = ["hook_with_zero_step", "hook_with_zero_step_interleaved"]
14
+
15
+ # Functional optimizers require passing a list of gradients to their `step()`
16
+ # method, and ZeRO requires a functional optimizer to overlap with DDP
17
+ # Passing a `None` instead of an actual gradient indicates to the optimizer
18
+ # to not update the corresponding parameter
19
+ _NO_PARAM_UPDATE: None = None
20
+
21
+
22
+ def _perform_local_step(
23
+ bucket: dist.GradBucket,
24
+ zero: ZeroRedundancyOptimizer,
25
+ rank: int,
26
+ ):
27
+ r"""
28
+ Perform a local optimizer step using the gradients provided by ``bucket``.
29
+
30
+ Arguments:
31
+ bucket (dist.GradBucket): the bucket providing the gradients.
32
+ zero (ZeroRedundancyOptimizer): the :class:`ZeroRedundancyOptimizer`
33
+ instance to perform the :meth:`_local_step`.
34
+ rank (int): the calling process's rank.
35
+
36
+ .. warning::
37
+ This function assumes that appropriate synchronization has taken place
38
+ so that the bucket's gradients can be used.
39
+ """
40
+ overlap_info = zero._overlap_info
41
+ bucket_index = bucket.index()
42
+ if len(zero.optim.param_groups) != 1:
43
+ raise AssertionError(
44
+ "Overlapping DDP with ZeRO only supports a single parameter group"
45
+ )
46
+
47
+ # Construct the `gradients` input for the local optimizer step, which
48
+ # expects `None` in a list position to indicate that the corresponding
49
+ # parameter should not be updated
50
+ num_local_optim_params = len(zero.optim.param_groups[0]["params"])
51
+ gradients: list[torch.Tensor | None] = [
52
+ _NO_PARAM_UPDATE for _ in range(num_local_optim_params)
53
+ ]
54
+ if bucket_index not in overlap_info.offsets:
55
+ raise AssertionError(
56
+ f"Bucket index {bucket_index} was not assigned to rank {rank}"
57
+ )
58
+ gradients_offset = overlap_info.offsets[bucket_index]
59
+ bucket_assignment = zero._bucket_assignments_per_rank[rank][bucket_index]
60
+ bucket_offset = bucket_assignment.offset
61
+ length = len(bucket_assignment.parameters)
62
+ bucket_gradients = bucket.gradients()[bucket_offset : bucket_offset + length]
63
+ for i, grad in enumerate(bucket_gradients):
64
+ gradients[gradients_offset + i] = grad
65
+
66
+ zero._local_step(gradients)
67
+
68
+
69
+ def _broadcast_bucket(
70
+ bucket_index: int,
71
+ zero: ZeroRedundancyOptimizer,
72
+ ):
73
+ r"""
74
+ Broadcasts a bucket's parameters.
75
+
76
+ Arguments:
77
+ bucket_index (int): the index of the bucket corresponding to the
78
+ parameters to broadcast.
79
+ zero (ZeroRedundancyOptimizer): the calling process's
80
+ :class:`ZeroRedundancyOptimizer` instance.
81
+ """
82
+ overlap_info = zero._overlap_info
83
+ if len(overlap_info.assigned_ranks_per_bucket) <= bucket_index:
84
+ raise AssertionError("`assigned_ranks_per_bucket` is not fully constructed")
85
+ # Sort to ensure the same ordering across ranks
86
+ assigned_ranks = sorted(overlap_info.assigned_ranks_per_bucket[bucket_index])
87
+ if len(assigned_ranks) <= 0:
88
+ raise AssertionError(
89
+ f"Bucket {bucket_index} should be assigned to at least one rank"
90
+ )
91
+ for assigned_rank in assigned_ranks:
92
+ bucket_assignments = zero._bucket_assignments_per_rank[assigned_rank]
93
+ if bucket_index in bucket_assignments:
94
+ send_tensor = bucket_assignments[bucket_index].tensor
95
+ if send_tensor is None:
96
+ raise AssertionError
97
+ overlap_info.broadcast_handles.append(
98
+ dist.broadcast(
99
+ send_tensor,
100
+ src=dist.get_global_rank(zero.process_group, assigned_rank),
101
+ group=zero.process_group,
102
+ async_op=True,
103
+ )
104
+ )
105
+
106
+
107
+ def _save_ddp_bucket_info(
108
+ bucket: dist.GradBucket,
109
+ zero: ZeroRedundancyOptimizer,
110
+ ):
111
+ r"""
112
+ Save :class:`DistributedDataParallel` gradient bucket information for :class:`ZeroRedundancyOptimizer` instance ``zero``.
113
+
114
+ In particular, this function is meant to be called upon seeing each
115
+ gradient bucket to use when overlapping, meaning it does not save or compute any global
116
+ information.
117
+
118
+ Arguments:
119
+ bucket (dist.GradBucket): the current gradient bucket.
120
+ zero (ZeroRedundancyOptimizer): the calling process's
121
+ :class:`ZeroRedundancyOptimizer` instance.
122
+ """
123
+ overlap_info = zero._overlap_info
124
+ bucket_params = bucket.parameters()
125
+ if len(bucket_params) <= 0:
126
+ raise AssertionError("Empty bucket")
127
+
128
+ # Save the parameters in the bucket
129
+ overlap_info.params_per_bucket.append(bucket_params)
130
+ if overlap_info.shard_buckets:
131
+ # Additionally save the bucket size for the assignment heuristic to use
132
+ bucket_size = 0
133
+ for param in bucket_params:
134
+ bucket_size += param.numel()
135
+ if overlap_info.total_size is None:
136
+ raise AssertionError
137
+ overlap_info.total_size += bucket_size
138
+
139
+
140
+ def _hook_with_zero_step_setup(
141
+ ddp_ref: weakref.ReferenceType,
142
+ zero: ZeroRedundancyOptimizer,
143
+ bucket: dist.GradBucket,
144
+ ):
145
+ r"""
146
+ Encapsulate the setup logic for :func:`hook_with_zero_step` and :func:`hook_with_zero_step_interleaved`.
147
+
148
+ This means the logic to run in the
149
+ hook before the backward pass and optimizer step can actually be
150
+ overlapped. This is factored out since it is common to both
151
+ :func:`hook_with_zero_step` and :func:`hook_with_zero_step_interleaved`.
152
+
153
+ Arguments:
154
+ ddp_ref (weakref.ReferenceType): weak reference to the process's
155
+ :class:`DistributedDataParallel` instance.
156
+ zero (ZeroRedundancyOptimizer): the calling process's
157
+ :class:`ZeroRedundancyOptimizer` instance.
158
+ bucket (dist.GradBucket): the current gradient bucket.
159
+ """
160
+ # Proceed as normal until the DDP buckets have been rebuilt
161
+ if not ddp_ref()._has_rebuilt_buckets: # type: ignore[union-attr]
162
+ if zero._overlap_info.status != _OverlapStatus.UNINITIALIZED:
163
+ raise AssertionError
164
+ return
165
+
166
+ bucket_index = bucket.index()
167
+ overlap_info = zero._overlap_info
168
+ if overlap_info.status == _OverlapStatus.UNINITIALIZED:
169
+ overlap_info.status = _OverlapStatus.DDP_HAS_REBUILT_BUCKETS
170
+
171
+ if overlap_info.status == _OverlapStatus.DDP_HAS_REBUILT_BUCKETS:
172
+ if bucket_index == 0 and len(overlap_info.params_per_bucket) > 0:
173
+ # This corresponds to the first bucket of the backward pass
174
+ # immediately after all information has been saved, so we
175
+ # can perform the delayed ZeRO initialization
176
+ zero._init_zero_for_overlap()
177
+ else:
178
+ # Once DDP buckets have been rebuilt but ZeRO has not been
179
+ # properly initialized yet, save the information needed
180
+ _save_ddp_bucket_info(bucket, zero)
181
+
182
+
183
+ def hook_with_zero_step(
184
+ hook: Callable[[Any, dist.GradBucket], torch.futures.Future],
185
+ ddp: DistributedDataParallel,
186
+ zero: ZeroRedundancyOptimizer,
187
+ shard_buckets: bool = False,
188
+ ) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]:
189
+ r"""
190
+ Modify ``hook`` to overlap :class:`ZeroRedundancyOptimizer` optimizer step with :class:`DistributedDataParallel` backward pass.
191
+
192
+ This approach overlaps the optimizer computation and communication with the
193
+ backward communication. In particular, the backward computation proceeds
194
+ contiguously, and the optimizer computation follows, overlapping with
195
+ outstanding backward communication (i.e. all-reduces) and possibly other
196
+ optimizer communication (i.e. broadcasts).
197
+ The optimizer step computation begins after the last gradient bucket computation has finished.
198
+
199
+ This approach may be preferred over :meth:`hook_with_zero_step_interleaved`
200
+ if communication is relatively slow compared to computation.
201
+
202
+ Arguments:
203
+ hook (Callable[[Any, dist.GradBucket], torch.futures.Future]): the hook
204
+ to modify.
205
+ ddp (DistributedDataParallel): the :class:`DistributedDataParallel`
206
+ instance to use.
207
+ zero (ZeroRedundancyOptimizer): the :class:`ZeroRedundancyOptimizer`
208
+ instance to use.
209
+ shard_buckets (bool): if ``True``, then the assignment of each
210
+ :class:`DistributedDataParallel` bucket is partitioned across
211
+ possibly multiple :class:`ZeroRedundancyOptimizer` instances (i.e.
212
+ across possibly multiple ranks) to approximate uniformity; if
213
+ ``False``, then each bucket is wholly assigned to a single
214
+ :class:`ZeroRedundancyOptimizer` instance (i.e. to a single rank).
215
+
216
+ Returns:
217
+ The modified hook.
218
+
219
+ Raises:
220
+ ValueError: if ``zero`` was constructed with ``overlap_with_ddp=False``.
221
+ RuntimeError: if using any backend other than NCCL/HCCL since currently
222
+ Gloo may hang.
223
+
224
+ .. warning::
225
+ Given the way that overlapping :class:`DistributedDataParallel` with
226
+ :class:`ZeroRedundancyOptimizer` is currently implemented, the first
227
+ two or three training iterations do not perform parameter updates in
228
+ the optimizer step, depending on if ``static_graph=False`` or
229
+ ``static_graph=True``, respectively. This is because it needs
230
+ information about the gradient bucketing strategy used by
231
+ :class:`DistributedDataParallel`, which is not finalized until the
232
+ second forward pass if ``static_graph=False`` or until the third
233
+ forward pass if ``static_graph=True``.
234
+ """
235
+ if not zero._overlap_with_ddp:
236
+ raise ValueError(
237
+ "ZeroRedundancyOptimizer must be constructed with "
238
+ "`overlap_with_ddp=True` to use this hook properly"
239
+ )
240
+ ddp_ref = weakref.ref(ddp)
241
+
242
+ # NOTE: Gloo may hang with this overlapping approach; see https://github.com/pytorch/pytorch/issues/62300
243
+ pg = dist.get_backend(ddp_ref().process_group) # type: ignore[union-attr]
244
+ if pg == dist.Backend.GLOO:
245
+ raise RuntimeError(
246
+ "Gloo backend using Overlapping DDP with ZeRO may meet hangs"
247
+ )
248
+
249
+ if shard_buckets:
250
+ zero._overlap_info.shard_buckets = True
251
+ zero._overlap_info.total_size = 0
252
+
253
+ def hook_with_zero_fn(
254
+ state: Any,
255
+ bucket: dist.GradBucket,
256
+ ) -> torch.futures.Future[torch.Tensor]:
257
+ r"""
258
+ Return :class:`Future` that runs the optimizer step if this corresponds to the last gradient bucket.
259
+
260
+ Perform equivalent of :class:`ZeroRedundancyOptimizer` :meth:`step` if ``bucket`` is last gradient bucket.
261
+ The function gives a gradient bucket tensor and
262
+ performs additional computation on the iteration that
263
+ the :class:`DistributedDataParallel` buckets are rebuilt to collect
264
+ information used to implement the modified hook.
265
+
266
+ Arguments:
267
+ state (Any): any state for the hook.
268
+ bucket (dist.GradBucket): the :class:`DistributedDataParallel`
269
+ gradient bucket.
270
+ """
271
+ fut = hook(state, bucket)
272
+ _hook_with_zero_step_setup(ddp_ref, zero, bucket)
273
+ if zero._overlap_info.status != _OverlapStatus.INITIALIZED:
274
+ return fut
275
+
276
+ overlap_info = zero._overlap_info
277
+ bucket_index = bucket.index()
278
+ rank = zero.global_rank
279
+
280
+ if overlap_info.status != _OverlapStatus.INITIALIZED:
281
+ raise AssertionError
282
+ if len(overlap_info.assigned_ranks_per_bucket) <= bucket_index:
283
+ raise AssertionError("`assigned_ranks_per_bucket` is not fully constructed")
284
+ assigned_to_bucket = (
285
+ rank in overlap_info.assigned_ranks_per_bucket[bucket_index]
286
+ )
287
+
288
+ # Save the bucket reference and all-reduce future for the final bucket
289
+ if assigned_to_bucket:
290
+ overlap_info.bucket_index_to_bucket[bucket_index] = bucket
291
+ overlap_info.bucket_index_to_future[bucket_index] = fut
292
+
293
+ # Check that buckets are indexed incrementally starting from 0 in the
294
+ # order of their autograd hooks firing
295
+ if len(overlap_info.bucket_indices_seen) > 0:
296
+ if overlap_info.bucket_indices_seen[-1] != bucket_index - 1:
297
+ raise AssertionError("Bucket indices are not in incremental order")
298
+ else:
299
+ if bucket_index != 0:
300
+ raise AssertionError("Bucket indices do not start from 0")
301
+ overlap_info.bucket_indices_seen.append(bucket_index)
302
+
303
+ # Directly return the future without any optimizer computation if this
304
+ # is not the last bucket
305
+ num_buckets = len(overlap_info.params_per_bucket)
306
+ is_last_bucket = bucket_index == num_buckets - 1
307
+ if not is_last_bucket:
308
+ return fut
309
+
310
+ # Perform partial optimizer step on all buckets after the final
311
+ # bucket has been computed
312
+ # NOTE: This should not be chained as a callback to the last bucket's
313
+ # all-reduce future since that would add synchronization that delays
314
+ # all optimizer computation to wait for that last all-reduce
315
+ for bucket_index in range(num_buckets):
316
+ assigned_ranks = overlap_info.assigned_ranks_per_bucket[bucket_index]
317
+ if rank in assigned_ranks:
318
+ # Wait on the bucket's all-reduce future to ensure correct
319
+ # gradients
320
+ if bucket_index not in overlap_info.bucket_index_to_future:
321
+ raise AssertionError(
322
+ f"All-reduce future for bucket {bucket_index} not saved "
323
+ f"on rank {rank}"
324
+ )
325
+ allreduce_future = overlap_info.bucket_index_to_future[bucket_index]
326
+ allreduce_future.wait()
327
+
328
+ # Perform the partial optimizer step
329
+ curr_bucket = overlap_info.bucket_index_to_bucket[bucket_index]
330
+ _perform_local_step(curr_bucket, zero, rank)
331
+
332
+ _broadcast_bucket(bucket_index, zero)
333
+
334
+ # Ensure that all parameter updates are finished before the
335
+ # next forward pass
336
+ overlap_info.wait_for_broadcasts()
337
+ overlap_info.clear_per_iter_info()
338
+
339
+ return fut
340
+
341
+ return hook_with_zero_fn
342
+
343
+
344
+ def hook_with_zero_step_interleaved(
345
+ hook: Callable[[Any, dist.GradBucket], torch.futures.Future],
346
+ ddp: DistributedDataParallel,
347
+ zero: ZeroRedundancyOptimizer,
348
+ shard_buckets: bool = False,
349
+ ) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]:
350
+ r"""
351
+ Modify ``hook`` to overlap :class:`ZeroRedundancyOptimizer` optimizer step with :class:`DistributedDataParallel` backward pass
352
+
353
+ This approach overlaps the optimizer computation and communication with the
354
+ backward computation and communication. In particular, once a bucket's
355
+ gradients have been computed, the optimizer computation using those
356
+ gradients is launched (though the actual computation must wait for the
357
+ bucket's all-reduce to complete). This yields an interleaving of all-
358
+ reduces and broadcasts in the communication stream.
359
+
360
+ This approach may be preferred over :meth:`hook_with_zero_step` if
361
+ communication is relatively fast compared to computation.
362
+
363
+ Arguments:
364
+ hook (Any * dist.GradBucket -> torch.futures.Future): the hook to
365
+ modify.
366
+ ddp (DistributedDataParallel): the :class:`DistributedDataParallel`
367
+ instance to use.
368
+ zero (ZeroRedundancyOptimizer): the :class:`ZeroRedundancyOptimizer`
369
+ instance to use.
370
+ shard_buckets (bool): if ``True``, then the assignment of each
371
+ :class:`DistributedDataParallel` bucket is partitioned across
372
+ possibly multiple :class:`ZeroRedundancyOptimizer` instances (i.e.
373
+ across possibly multiple ranks) to approximate uniformity; if
374
+ ``False``, then each bucket is wholly assigned to a single
375
+ :class:`ZeroRedundancyOptimizer` instance (i.e. to a single rank).
376
+
377
+ Returns:
378
+ The modified hook.
379
+
380
+ Raises:
381
+ ValueError: if ``zero`` was constructed with ``overlap_with_ddp=False``.
382
+ RuntimeError: if using any backend other than NCCL since currently
383
+ Gloo may hang.
384
+
385
+ .. warning::
386
+ Given the way that overlapping :class:`DistributedDataParallel` with
387
+ :class:`ZeroRedundancyOptimizer` is currently implemented, the first
388
+ two or three training iterations do not perform parameter updates in
389
+ the optimizer step, depending on if ``static_graph=False`` or
390
+ ``static_graph=True``, respectively. This is because it needs
391
+ information about the gradient bucketing strategy used by
392
+ :class:`DistributedDataParallel`, which is not finalized until the
393
+ second forward pass if ``static_graph=False`` or until the third
394
+ forward pass if ``static_graph=True``.
395
+ """
396
+ if not zero._overlap_with_ddp:
397
+ raise ValueError(
398
+ "ZeroRedundancyOptimizer must be constructed with "
399
+ "`overlap_with_ddp=True` to use this hook properly"
400
+ )
401
+ ddp_ref = weakref.ref(ddp)
402
+
403
+ # NOTE: Gloo may hang with this overlapping approach; see https://github.com/pytorch/pytorch/issues/62300
404
+ pg = dist.get_backend(ddp_ref().process_group) # type: ignore[union-attr]
405
+ if pg == dist.Backend.GLOO:
406
+ raise RuntimeError(
407
+ "Gloo backend using Overlapping DDP with ZeRO may meet hangs"
408
+ )
409
+
410
+ if shard_buckets:
411
+ zero._overlap_info.shard_buckets = True
412
+ zero._overlap_info.total_size = 0
413
+
414
+ def hook_with_zero_interleaved_fn(
415
+ state,
416
+ bucket: dist.GradBucket,
417
+ ) -> torch.futures.Future[torch.Tensor]:
418
+ r"""
419
+ Return :class:`Future` that gives gradient bucket tensor and performs partial :class:`ZeroRedundancyOptimizer` :meth:`step`.
420
+
421
+ This function uses the gradients in the given bucket to perform a partial
422
+ :class:`ZeroRedundancyOptimizer` :meth:`step`
423
+
424
+ Arguments:
425
+ state: any state for the hook.
426
+ bucket (dist.GradBucket): the :class:`DistributedDataParallel`
427
+ gradient bucket.
428
+ """
429
+ fut = hook(state, bucket)
430
+ _hook_with_zero_step_setup(ddp_ref, zero, bucket)
431
+ if zero._overlap_info.status != _OverlapStatus.INITIALIZED:
432
+ return fut
433
+
434
+ def zero_step(fut: torch.futures.Future) -> torch.Tensor:
435
+ r"""
436
+ Perform partial :class:`ZeroRedundancyOptimizer` :meth:`step` using gradients in the :class:`DistributedDataParallel`.
437
+
438
+ Returns:
439
+ A :class:`torch.Tensor` representing the contents of the
440
+ gradient bucket.
441
+ """
442
+ overlap_info = zero._overlap_info
443
+ bucket_index = bucket.index()
444
+ rank = zero.global_rank
445
+
446
+ assigned_ranks = overlap_info.assigned_ranks_per_bucket[bucket_index]
447
+ overlap_info.bucket_indices_seen.append(bucket_index)
448
+ if rank in assigned_ranks:
449
+ _perform_local_step(bucket, zero, rank)
450
+
451
+ _broadcast_bucket(bucket_index, zero)
452
+
453
+ num_buckets = len(overlap_info.params_per_bucket)
454
+ if len(overlap_info.bucket_indices_seen) == num_buckets:
455
+ # Ensure that all parameter updates are finished before the
456
+ # next forward pass
457
+ overlap_info.wait_for_broadcasts()
458
+ overlap_info.clear_per_iter_info()
459
+
460
+ return bucket.buffer()
461
+
462
+ return fut.then(zero_step)
463
+
464
+ return hook_with_zero_interleaved_fn
western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/debugging_hooks.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+
3
+ import torch
4
+ from torch.distributed import GradBucket
5
+
6
+
7
+ __all__ = ["noop_hook"]
8
+
9
+
10
+ def noop_hook(_: Any, bucket: GradBucket) -> torch.futures.Future[torch.Tensor]:
11
+ """
12
+ Return a future that wraps the input, so it is a no-op that does not incur any communication overheads.
13
+
14
+ This hook should **only** be used for headroom analysis of allreduce optimization,
15
+ instead of the normal gradient synchronization.
16
+ For example, if only less than 10% speedup of training time can be observed after this hook is registered,
17
+ it usually implies that allreduce is not a performance bottleneck for this case.
18
+ Such instrumentation can be particularly useful
19
+ if GPU traces cannot be easily retrieved or the trace analysis is complicated by
20
+ some factors such as the overlap between allreduce and computation or the desynchronization across ranks.
21
+
22
+ Example::
23
+ >>> # xdoctest: +SKIP
24
+ >>> ddp_model.register_comm_hook(None, noop_hook)
25
+ """
26
+ fut: torch.futures.Future[torch.Tensor] = torch.futures.Future()
27
+ fut.set_result(bucket.buffer())
28
+
29
+ return fut
western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/default_hooks.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from collections.abc import Callable
3
+ from typing import Any, cast
4
+
5
+ import torch
6
+ import torch.distributed as dist
7
+
8
+
9
+ __all__ = [
10
+ "allreduce_hook",
11
+ "fp16_compress_hook",
12
+ "bf16_compress_hook",
13
+ "fp16_compress_wrapper",
14
+ "bf16_compress_wrapper",
15
+ ]
16
+
17
+
18
+ def _allreduce_fut(
19
+ process_group: dist.ProcessGroup, tensor: torch.Tensor
20
+ ) -> torch.futures.Future[torch.Tensor]:
21
+ """Average the input gradient tensor by allreduce and returns a future."""
22
+ group_to_use = process_group if process_group is not None else dist.group.WORLD
23
+
24
+ # Apply the division first to avoid overflow, especially for FP16.
25
+ # pyrefly: ignore [missing-attribute]
26
+ tensor.div_(group_to_use.size())
27
+
28
+ return (
29
+ dist.all_reduce(tensor, group=group_to_use, async_op=True)
30
+ .get_future()
31
+ .then(lambda fut: fut.value()[0])
32
+ )
33
+
34
+
35
+ def allreduce_hook(
36
+ process_group: dist.ProcessGroup, bucket: dist.GradBucket
37
+ ) -> torch.futures.Future[torch.Tensor]:
38
+ """
39
+ Call ``allreduce`` using ``GradBucket`` tensors.
40
+
41
+ Once gradient tensors are aggregated across all workers, its ``then``
42
+ callback takes the mean and returns the result.
43
+
44
+ If user registers this DDP communication hook,
45
+ DDP results is expected to be same as the case where no hook was registered.
46
+ Hence, this won't change behavior of DDP and user can use this as a reference
47
+ or modify this hook to log useful information or any other purposes while
48
+ unaffecting DDP behavior.
49
+
50
+ Example::
51
+ >>> # xdoctest: +SKIP
52
+ >>> ddp_model.register_comm_hook(process_group, allreduce_hook)
53
+ """
54
+ return _allreduce_fut(process_group, bucket.buffer())
55
+
56
+
57
+ def _compress_hook(
58
+ dtype: torch.dtype,
59
+ process_group: dist.ProcessGroup,
60
+ bucket: dist.GradBucket,
61
+ ) -> torch.futures.Future[torch.Tensor]:
62
+ group_to_use = process_group if process_group is not None else dist.group.WORLD
63
+ # pyrefly: ignore [missing-attribute]
64
+ world_size = group_to_use.size()
65
+
66
+ buffer = (
67
+ cast(tuple[torch.Tensor, ...], bucket)[0]
68
+ if isinstance(bucket, tuple)
69
+ else bucket.buffer()
70
+ )
71
+ compressed_tensor = buffer.to(dtype).div_(world_size)
72
+
73
+ def decompress(fut):
74
+ decompressed_tensor = buffer
75
+ # Decompress in place to reduce the peak memory.
76
+ # See: https://github.com/pytorch/pytorch/issues/45968
77
+ value = fut if isinstance(fut, torch.Tensor) else fut.value()[0]
78
+ decompressed_tensor.copy_(value)
79
+ return decompressed_tensor
80
+
81
+ if torch.compiler.is_compiling():
82
+ grad = dist._functional_collectives.all_reduce(
83
+ compressed_tensor,
84
+ "sum",
85
+ # pyrefly: ignore [bad-argument-type]
86
+ group_to_use,
87
+ )
88
+ return decompress(grad)
89
+ else:
90
+ fut = dist.all_reduce(
91
+ compressed_tensor, group=group_to_use, async_op=True
92
+ ).get_future()
93
+ return fut.then(decompress)
94
+
95
+
96
+ def fp16_compress_hook(
97
+ process_group: dist.ProcessGroup,
98
+ bucket: dist.GradBucket,
99
+ ) -> torch.futures.Future[torch.Tensor]:
100
+ """
101
+ Compress by casting ``GradBucket`` to ``torch.float16`` divided by process group size.
102
+
103
+ This DDP communication hook implements a simple gradient compression
104
+ approach that casts ``GradBucket`` tensor to half-precision floating-point format (``torch.float16``)
105
+ and then divides it by the process group size.
106
+ It allreduces those ``float16`` gradient tensors. Once compressed gradient
107
+ tensors are allreduced, the chained callback ``decompress`` casts it back to the input data type (such as ``float32``).
108
+
109
+ Example::
110
+ >>> # xdoctest: +SKIP
111
+ >>> ddp_model.register_comm_hook(process_group, fp16_compress_hook)
112
+ """
113
+ return _compress_hook(torch.float16, process_group, bucket)
114
+
115
+
116
+ def bf16_compress_hook(
117
+ process_group: dist.ProcessGroup,
118
+ bucket: dist.GradBucket,
119
+ ) -> torch.futures.Future[torch.Tensor]:
120
+ """
121
+ Warning: This API is experimental, and it requires NCCL version later than 2.9.6.
122
+
123
+ This DDP communication hook implements a simple gradient compression
124
+ approach that casts ``GradBucket`` tensor to half-precision
125
+ `Brain floating point format <https://en.wikipedia.org/wiki/Bfloat16_floating-point_format>`_ (``torch.bfloat16``)
126
+ and then divides it by the process group size.
127
+ It allreduces those ``bfloat16`` gradient tensors. Once compressed gradient
128
+ tensors are allreduced, the chained callback ``decompress`` casts it back to the input data type (such as ``float32``).
129
+
130
+ Example::
131
+ >>> # xdoctest: +SKIP
132
+ >>> ddp_model.register_comm_hook(process_group, bf16_compress_hook)
133
+ """
134
+ return _compress_hook(torch.bfloat16, process_group, bucket)
135
+
136
+
137
+ def fp16_compress_wrapper(
138
+ hook: Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]],
139
+ ) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]:
140
+ """
141
+ Cast input tensor to ``torch.float16``, cast result of hook back to input dtype.
142
+
143
+ This wrapper casts the input gradient tensor of a given DDP communication hook to half-precision
144
+ floating point format (``torch.float16``), and casts the resulting tensor of the given hook back to
145
+ the input data type, such as ``float32``.
146
+ Therefore, ``fp16_compress_hook`` is equivalent to ``fp16_compress_wrapper(allreduce_hook)``.
147
+
148
+ Example::
149
+ >>> # xdoctest: +SKIP
150
+ >>> state = PowerSGDState(process_group=process_group, matrix_approximation_rank=1, start_powerSGD_iter=10)
151
+ >>> ddp_model.register_comm_hook(state, fp16_compress_wrapper(powerSGD_hook))
152
+ """
153
+
154
+ def fp16_compress_wrapper_hook(
155
+ hook_state, bucket: dist.GradBucket
156
+ ) -> torch.futures.Future[torch.Tensor]:
157
+ # Cast bucket tensor to FP16.
158
+ bucket.set_buffer(bucket.buffer().to(torch.float16))
159
+
160
+ fut = hook(hook_state, bucket)
161
+
162
+ def decompress(fut):
163
+ decompressed_tensor = bucket.buffer()
164
+ # Decompress in place to reduce the peak memory.
165
+ # See: https://github.com/pytorch/pytorch/issues/45968
166
+ decompressed_tensor.copy_(fut.value())
167
+ return decompressed_tensor
168
+
169
+ # Decompress after hook has run.
170
+ return fut.then(decompress)
171
+
172
+ return fp16_compress_wrapper_hook
173
+
174
+
175
+ def bf16_compress_wrapper(
176
+ hook: Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]],
177
+ ) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]:
178
+ """
179
+ Warning: This API is experimental, and it requires NCCL version later than 2.9.6.
180
+
181
+ This wrapper casts the input gradient tensor of a given DDP communication hook to half-precision
182
+ `Brain floating point format <https://en.wikipedia.org/wiki/Bfloat16_floating-point_format>`_ (``torch.bfloat16``),
183
+ and casts the resulting tensor of the given hook back to the input data type, such as ``float32``.
184
+
185
+ Therefore, ``bf16_compress_hook`` is equivalent to ``bf16_compress_wrapper(allreduce_hook)``.
186
+
187
+ Example::
188
+ >>> # xdoctest: +SKIP
189
+ >>> state = PowerSGDState(process_group=process_group, matrix_approximation_rank=1, start_powerSGD_iter=10)
190
+ >>> ddp_model.register_comm_hook(state, bf16_compress_wrapper(powerSGD_hook))
191
+ """
192
+
193
+ def bf16_compress_wrapper_hook(
194
+ hook_state, bucket: dist.GradBucket
195
+ ) -> torch.futures.Future[torch.Tensor]:
196
+ # Cast bucket tensor to BF16.
197
+ bucket.set_buffer(bucket.buffer().to(torch.bfloat16))
198
+
199
+ fut = hook(hook_state, bucket)
200
+
201
+ def decompress(fut):
202
+ decompressed_tensor = bucket.buffer()
203
+ # Decompress in place to reduce the peak memory.
204
+ # See: https://github.com/pytorch/pytorch/issues/45968
205
+ decompressed_tensor.copy_(fut.value())
206
+ return decompressed_tensor
207
+
208
+ # Decompress after hook has run.
209
+ return fut.then(decompress)
210
+
211
+ return bf16_compress_wrapper_hook
western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/mixed_precision_hooks.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from typing import Any, no_type_check
3
+
4
+ import torch
5
+ import torch.distributed as dist
6
+ from torch.autograd import Variable
7
+ from torch.distributed.utils import _free_storage
8
+
9
+
10
+ @dataclass
11
+ class _AllreduceUpcastHookState:
12
+ """
13
+ State to manage DDP mixed precision in backward / gradient communication.
14
+
15
+ This contains a weakref to the DDP module for access to reducer and process
16
+ group, and a stream to run parameter and gradient upcasts.
17
+ """
18
+
19
+ ddp_weakref: Any
20
+ upcast_stream: torch.Stream
21
+ wait_for_stream_enqueued: bool = False
22
+
23
+
24
+ @no_type_check
25
+ def _reducer_allreduce_and_upcast_hook(
26
+ hook_state: _AllreduceUpcastHookState, bucket: dist.GradBucket
27
+ ) -> torch.futures.Future[torch.Tensor]:
28
+ """
29
+ Perform allreduce in precision ``reduce_dtype``, upcast to prepare for optimizer.
30
+
31
+ Performs allreduce in the reduced precision given by DDP's mixed precision
32
+ reduce_dtype, and upcasts parameters and gradients to fp32 in preparation
33
+ to run the optimizer.
34
+ """
35
+ ddp_weakref = hook_state.ddp_weakref
36
+ reducer, process_group = ddp_weakref().reducer, ddp_weakref().process_group
37
+ # Cast bucket if different than param_dtype.
38
+ if (
39
+ ddp_weakref().mixed_precision.param_dtype
40
+ != ddp_weakref().mixed_precision.reduce_dtype
41
+ ):
42
+ # Cast bucket tensor to reduce_dtype
43
+ bucket.set_buffer(
44
+ bucket.buffer().to(ddp_weakref().mixed_precision.reduce_dtype)
45
+ )
46
+ fut = reducer._run_allreduce_hook(bucket)
47
+ ret_fut = torch.futures.Future()
48
+ stream = hook_state.upcast_stream
49
+ with stream:
50
+ fut.wait()
51
+ bucket.buffer().div_(process_group.size())
52
+ ret_fut.set_result(bucket.buffer())
53
+
54
+ # Upcast parameters and gradients so optimizer step can run in fp32.
55
+ for p in bucket.parameters():
56
+ p.data = p._fp_param
57
+ # free storage for mp param as it will be allocated again in next
58
+ # forward pass.
59
+ _free_storage(p._mp_param)
60
+ p.grad.data = p.grad.to(p.data.dtype)
61
+
62
+ # enqueue a callback to wait for this stream at end of backward
63
+ def wait_for_stream_cb():
64
+ torch.accelerator.current_stream().wait_stream(stream)
65
+ # Remove post-backward hooks since they are re-installed in next
66
+ # iteration, similar to FSDP.
67
+ # Parameters that don't require grad still needed to be casted since
68
+ # they may participate in computation. However, they would not be recast
69
+ # by hook above as they don't have a grad hook installed, so cast them
70
+ # back here.
71
+ for _, p in ddp_weakref().module.named_parameters():
72
+ if hasattr(p, "_ddp_mp_hook_state"):
73
+ p._ddp_mp_hook_state[1].remove()
74
+ delattr(p, "_ddp_mp_hook_state")
75
+ if not p.requires_grad and not hasattr(p, "_ddp_ignored"):
76
+ p.data = p._fp_param
77
+
78
+ # reset for next backward pass
79
+ hook_state.wait_for_stream_enqueued = False
80
+
81
+ if not hook_state.wait_for_stream_enqueued:
82
+ Variable._execution_engine.queue_callback(wait_for_stream_cb)
83
+ # mark that the callback is enqueued
84
+ hook_state.wait_for_stream_enqueued = True
85
+
86
+ return ret_fut
western_ai/usa/pytorch/torch/distributed/algorithms/ddp_comm_hooks/optimizer_overlap_hooks.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from collections.abc import Callable
3
+ from dataclasses import dataclass
4
+ from functools import partial
5
+ from typing import Any, no_type_check
6
+
7
+ import torch
8
+ import torch.distributed as dist
9
+ from torch.autograd import Variable
10
+
11
+
12
+ __all__: list[str] = []
13
+
14
+ _FUNCTIONAL_OPTIM_STEP_METHOD_NAME = "step_param"
15
+
16
+
17
+ class _OptimizerHookState:
18
+ """
19
+ Holds state for running optimizer in-line after DDP communication hook.
20
+
21
+ Currently contains only optimizer class which must have a method `step_param`.
22
+ """
23
+
24
+ __slots__ = ["functional_optimizer", "params_to_optimize"]
25
+
26
+ def __init__(self, functional_optim, params=None):
27
+ self.functional_optimizer = functional_optim
28
+ self._check_valid_functional_optim()
29
+ self._set_params_to_optimize(params)
30
+
31
+ def _set_params_to_optimize(self, params):
32
+ if params is not None:
33
+ self.params_to_optimize = set(params)
34
+
35
+ def _check_valid_functional_optim(self):
36
+ if not hasattr(self.functional_optimizer, _FUNCTIONAL_OPTIM_STEP_METHOD_NAME):
37
+ raise ValueError(
38
+ f"Class {type(self.functional_optimizer)} must implement method "
39
+ f"{_FUNCTIONAL_OPTIM_STEP_METHOD_NAME}."
40
+ )
41
+
42
+
43
+ @dataclass
44
+ class _OptimInBackwardHookState:
45
+ optim_stream: torch.Stream
46
+ wait_for_optim_stream_enqueued: bool
47
+
48
+
49
+ @no_type_check
50
+ def _apply_optim_in_backward_hook(
51
+ gradient_is_bucket_view: bool,
52
+ ) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]:
53
+ r"""
54
+ Register hook to apply the optimizer in backward.
55
+
56
+ If torch.distributed.optim._apply_optimizer_in_backward is used to overlap
57
+ optimizer with backward pass, DDP will run the below hook to run optimizer
58
+ step for parameters after gradient communication has taken place.
59
+ """
60
+ optim_in_bwd_state = _OptimInBackwardHookState(
61
+ optim_stream=torch.Stream(),
62
+ wait_for_optim_stream_enqueued=False,
63
+ )
64
+
65
+ def apply_optim_in_backward_hook(
66
+ hook_state: Any,
67
+ bucket: dist.GradBucket,
68
+ optim_stream_state,
69
+ ) -> torch.futures.Future[torch.Tensor]:
70
+ # Run original hook
71
+ ddp_weakref = hook_state
72
+ ddp_inst = ddp_weakref()
73
+ reducer, process_group = ddp_inst.reducer, ddp_inst.process_group
74
+ fut = reducer._run_allreduce_hook(bucket)
75
+ optimizer_stream = optim_stream_state.optim_stream
76
+ with optimizer_stream:
77
+ fut.wait()
78
+ # Apply gradient division since C++ side only allreduces and does
79
+ # not average. TODO: (rohan-varma) the div factor may be different
80
+ # when running with join hook
81
+ bucket.buffer().div_(process_group.size())
82
+ model_params = bucket.parameters()
83
+ grads = bucket.gradients()
84
+ # TODO (rohan-varma): upcast as needed for DDP mixed precision,
85
+ # once optimizer in backward + DDP mixed precision is supported.
86
+ for p, g in zip(model_params, grads):
87
+ if hasattr(p, "_in_backward_optimizers"):
88
+ # Note: need to set grad to the bucket's grad, because
89
+ # running allreduce results in the bucket's grad being
90
+ # reduced, but not grad field.
91
+ if not gradient_is_bucket_view:
92
+ p.grad = g
93
+ for optim in p._in_backward_optimizers:
94
+ optim.step()
95
+
96
+ # Need to return a Future[Tensor] to obey comm hook API contract.
97
+ ret_fut = torch.futures.Future()
98
+ ret_fut.set_result(bucket.buffer())
99
+
100
+ # enqueue a callback to wait for this optimizer stream at the end of
101
+ # backward and set all DDP managed grads to None.
102
+ def wait_for_optim_stream_callback():
103
+ torch.accelerator.current_stream().wait_stream(
104
+ optim_stream_state.optim_stream
105
+ )
106
+ # Set DDP managed grads to None
107
+ for param in ddp_inst._get_data_parallel_params(ddp_inst.module):
108
+ if hasattr(param, "_in_backward_optimizers"):
109
+ param.grad = None
110
+
111
+ # reset for the next backward pass
112
+ optim_stream_state.wait_for_optim_stream_enqueued = False
113
+
114
+ if not optim_stream_state.wait_for_optim_stream_enqueued:
115
+ Variable._execution_engine.queue_callback(wait_for_optim_stream_callback)
116
+ # mark that the callback is enqueued
117
+ optim_stream_state.wait_for_optim_stream_enqueued = True
118
+
119
+ return ret_fut
120
+
121
+ comm_hook = partial(
122
+ apply_optim_in_backward_hook, optim_stream_state=optim_in_bwd_state
123
+ )
124
+ # These are needed for DDP's logging of comm hooks
125
+ comm_hook.__name__ = apply_optim_in_backward_hook.__name__
126
+ comm_hook.__qualname__ = apply_optim_in_backward_hook.__qualname__
127
+
128
+ return comm_hook
129
+
130
+
131
+ def _hook_then_optimizer(
132
+ hook: Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]],
133
+ optimizer_state: _OptimizerHookState,
134
+ ) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]:
135
+ r"""Run optimizer in a functional fashion after DDP communication hook."""
136
+ has_set_params = (
137
+ hasattr(optimizer_state, "params_to_optimize")
138
+ and optimizer_state.params_to_optimize is not None
139
+ )
140
+
141
+ def hook_then_optimizer_wrapper(
142
+ hook_state, bucket: dist.GradBucket
143
+ ) -> torch.futures.Future[torch.Tensor]:
144
+ # Run original hook
145
+ fut = hook(hook_state, bucket)
146
+
147
+ def optimizer_step(fut):
148
+ gradient_tensors = bucket.gradients()
149
+ model_params = bucket.parameters()
150
+ for grad_tensor, model_param in zip(gradient_tensors, model_params):
151
+ if (
152
+ not has_set_params
153
+ or model_param in optimizer_state.params_to_optimize
154
+ ):
155
+ optimizer_state.functional_optimizer.step_param(
156
+ model_param,
157
+ grad_tensor,
158
+ )
159
+ return bucket.buffer()
160
+
161
+ return fut.then(optimizer_step)
162
+
163
+ return hook_then_optimizer_wrapper