Instructions to use Motif-Technologies/optimizer with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Kernels
How to use Motif-Technologies/optimizer with Kernels:
# !pip install kernels from kernels import get_kernel # a version (or an explicit revision) is required; see the "Files and versions" tab for the available ones kernel = get_kernel("Motif-Technologies/optimizer", version=1) - Notebooks
- Google Colab
- Kaggle
change owner_params to owned_params
Browse files
torch-ext/optimizer/muon.py
CHANGED
|
@@ -133,7 +133,7 @@ def _all2all_gather(params, param_to_state, rank, comm_stream, none_grad,
|
|
| 133 |
for v in per_dst), "all params should be sharded to all devices"
|
| 134 |
|
| 135 |
send_buf = torch.cat([torch.cat(v, dim=0) for v in per_dst], dim=0)
|
| 136 |
-
|
| 137 |
p for p in params if param_to_state[id(p)].worker_rank == rank
|
| 138 |
]
|
| 139 |
|
|
@@ -142,7 +142,7 @@ def _all2all_gather(params, param_to_state, rank, comm_stream, none_grad,
|
|
| 142 |
|
| 143 |
for src in range(num_ranks):
|
| 144 |
total = 0
|
| 145 |
-
for p in
|
| 146 |
state = param_to_state[id(p)]
|
| 147 |
assert state.worker_rank == rank
|
| 148 |
total += split_elems_for_src(p, state, src, num_ranks)
|
|
@@ -176,14 +176,14 @@ def _all2all_gather(params, param_to_state, rank, comm_stream, none_grad,
|
|
| 176 |
comm_stream.wait_event(alloc_event)
|
| 177 |
|
| 178 |
off = 0
|
| 179 |
-
write_offsets = {id(p): 0 for p in
|
| 180 |
for src in range(num_ranks):
|
| 181 |
if recv_counts[src] == 0:
|
| 182 |
continue
|
| 183 |
|
| 184 |
block = recv_counts[src]
|
| 185 |
inner_off = 0
|
| 186 |
-
for p in
|
| 187 |
state = param_to_state[id(p)]
|
| 188 |
assert state.worker_rank == rank
|
| 189 |
n = split_elems_for_src(p, state, src, num_ranks)
|
|
@@ -255,7 +255,7 @@ def _all2all_scatter(params, param_to_state, rank, comm_stream, alloc_event):
|
|
| 255 |
with torch.cuda.stream(comm_stream):
|
| 256 |
process_group = param_to_state[id(params[0])].process_group
|
| 257 |
num_ranks = dist.get_world_size(group=process_group)
|
| 258 |
-
|
| 259 |
p for p in params if param_to_state[id(p)].worker_rank == rank
|
| 260 |
]
|
| 261 |
|
|
@@ -263,8 +263,8 @@ def _all2all_scatter(params, param_to_state, rank, comm_stream, alloc_event):
|
|
| 263 |
per_dst = [[] for _ in range(num_ranks)]
|
| 264 |
send_counts = [0] * num_ranks
|
| 265 |
|
| 266 |
-
if
|
| 267 |
-
for p in
|
| 268 |
state = param_to_state[id(p)]
|
| 269 |
if state.compute_event is None:
|
| 270 |
raise RuntimeError(
|
|
|
|
| 133 |
for v in per_dst), "all params should be sharded to all devices"
|
| 134 |
|
| 135 |
send_buf = torch.cat([torch.cat(v, dim=0) for v in per_dst], dim=0)
|
| 136 |
+
owned_params = [
|
| 137 |
p for p in params if param_to_state[id(p)].worker_rank == rank
|
| 138 |
]
|
| 139 |
|
|
|
|
| 142 |
|
| 143 |
for src in range(num_ranks):
|
| 144 |
total = 0
|
| 145 |
+
for p in owned_params:
|
| 146 |
state = param_to_state[id(p)]
|
| 147 |
assert state.worker_rank == rank
|
| 148 |
total += split_elems_for_src(p, state, src, num_ranks)
|
|
|
|
| 176 |
comm_stream.wait_event(alloc_event)
|
| 177 |
|
| 178 |
off = 0
|
| 179 |
+
write_offsets = {id(p): 0 for p in owned_params}
|
| 180 |
for src in range(num_ranks):
|
| 181 |
if recv_counts[src] == 0:
|
| 182 |
continue
|
| 183 |
|
| 184 |
block = recv_counts[src]
|
| 185 |
inner_off = 0
|
| 186 |
+
for p in owned_params:
|
| 187 |
state = param_to_state[id(p)]
|
| 188 |
assert state.worker_rank == rank
|
| 189 |
n = split_elems_for_src(p, state, src, num_ranks)
|
|
|
|
| 255 |
with torch.cuda.stream(comm_stream):
|
| 256 |
process_group = param_to_state[id(params[0])].process_group
|
| 257 |
num_ranks = dist.get_world_size(group=process_group)
|
| 258 |
+
owned_params = [
|
| 259 |
p for p in params if param_to_state[id(p)].worker_rank == rank
|
| 260 |
]
|
| 261 |
|
|
|
|
| 263 |
per_dst = [[] for _ in range(num_ranks)]
|
| 264 |
send_counts = [0] * num_ranks
|
| 265 |
|
| 266 |
+
if owned_params:
|
| 267 |
+
for p in owned_params:
|
| 268 |
state = param_to_state[id(p)]
|
| 269 |
if state.compute_event is None:
|
| 270 |
raise RuntimeError(
|