Kernels
ca1207 commited on
Commit
6943c45
·
1 Parent(s): 589b763

change owner_params to owned_params

Browse files
Files changed (1) hide show
  1. torch-ext/optimizer/muon.py +7 -7
torch-ext/optimizer/muon.py CHANGED
@@ -133,7 +133,7 @@ def _all2all_gather(params, param_to_state, rank, comm_stream, none_grad,
133
  for v in per_dst), "all params should be sharded to all devices"
134
 
135
  send_buf = torch.cat([torch.cat(v, dim=0) for v in per_dst], dim=0)
136
- owner_params = [
137
  p for p in params if param_to_state[id(p)].worker_rank == rank
138
  ]
139
 
@@ -142,7 +142,7 @@ def _all2all_gather(params, param_to_state, rank, comm_stream, none_grad,
142
 
143
  for src in range(num_ranks):
144
  total = 0
145
- for p in owner_params:
146
  state = param_to_state[id(p)]
147
  assert state.worker_rank == rank
148
  total += split_elems_for_src(p, state, src, num_ranks)
@@ -176,14 +176,14 @@ def _all2all_gather(params, param_to_state, rank, comm_stream, none_grad,
176
  comm_stream.wait_event(alloc_event)
177
 
178
  off = 0
179
- write_offsets = {id(p): 0 for p in owner_params}
180
  for src in range(num_ranks):
181
  if recv_counts[src] == 0:
182
  continue
183
 
184
  block = recv_counts[src]
185
  inner_off = 0
186
- for p in owner_params:
187
  state = param_to_state[id(p)]
188
  assert state.worker_rank == rank
189
  n = split_elems_for_src(p, state, src, num_ranks)
@@ -255,7 +255,7 @@ def _all2all_scatter(params, param_to_state, rank, comm_stream, alloc_event):
255
  with torch.cuda.stream(comm_stream):
256
  process_group = param_to_state[id(params[0])].process_group
257
  num_ranks = dist.get_world_size(group=process_group)
258
- owner_params = [
259
  p for p in params if param_to_state[id(p)].worker_rank == rank
260
  ]
261
 
@@ -263,8 +263,8 @@ def _all2all_scatter(params, param_to_state, rank, comm_stream, alloc_event):
263
  per_dst = [[] for _ in range(num_ranks)]
264
  send_counts = [0] * num_ranks
265
 
266
- if owner_params:
267
- for p in owner_params:
268
  state = param_to_state[id(p)]
269
  if state.compute_event is None:
270
  raise RuntimeError(
 
133
  for v in per_dst), "all params should be sharded to all devices"
134
 
135
  send_buf = torch.cat([torch.cat(v, dim=0) for v in per_dst], dim=0)
136
+ owned_params = [
137
  p for p in params if param_to_state[id(p)].worker_rank == rank
138
  ]
139
 
 
142
 
143
  for src in range(num_ranks):
144
  total = 0
145
+ for p in owned_params:
146
  state = param_to_state[id(p)]
147
  assert state.worker_rank == rank
148
  total += split_elems_for_src(p, state, src, num_ranks)
 
176
  comm_stream.wait_event(alloc_event)
177
 
178
  off = 0
179
+ write_offsets = {id(p): 0 for p in owned_params}
180
  for src in range(num_ranks):
181
  if recv_counts[src] == 0:
182
  continue
183
 
184
  block = recv_counts[src]
185
  inner_off = 0
186
+ for p in owned_params:
187
  state = param_to_state[id(p)]
188
  assert state.worker_rank == rank
189
  n = split_elems_for_src(p, state, src, num_ranks)
 
255
  with torch.cuda.stream(comm_stream):
256
  process_group = param_to_state[id(params[0])].process_group
257
  num_ranks = dist.get_world_size(group=process_group)
258
+ owned_params = [
259
  p for p in params if param_to_state[id(p)].worker_rank == rank
260
  ]
261
 
 
263
  per_dst = [[] for _ in range(num_ranks)]
264
  send_counts = [0] * num_ranks
265
 
266
+ if owned_params:
267
+ for p in owned_params:
268
  state = param_to_state[id(p)]
269
  if state.compute_event is None:
270
  raise RuntimeError(