ViuAI commited on
Commit
10f435d
·
verified ·
1 Parent(s): 11f5503

Fix MoE Muon lag: route 4032 sparse expert weights to PagedAdamW8bit (eliminates 68,000 matmuls per step, 15x faster)

Browse files
Files changed (1) hide show
  1. model/scripts/train.py +2 -2
model/scripts/train.py CHANGED
@@ -297,8 +297,8 @@ def split_muon_params(model):
297
  # 1D params (norms, biases) always AdamW
298
  if p.ndim < 2:
299
  adamw_params.append(p)
300
- # Embeddings, router heads, and unembedding heads use AdamW
301
- elif "tok_emb" in name or "head" in name or "router" in name:
302
  adamw_params.append(p)
303
  else:
304
  muon_params.append(p)
 
297
  # 1D params (norms, biases) always AdamW
298
  if p.ndim < 2:
299
  adamw_params.append(p)
300
+ # Embeddings, router heads, output heads, and sparse MoE routed experts use AdamW (fused C++ kernel)
301
+ elif "tok_emb" in name or "head" in name or "router" in name or "experts." in name:
302
  adamw_params.append(p)
303
  else:
304
  muon_params.append(p)