Kernels
ca1207 Claude Opus 4.6 (1M context) commited on
Commit
89b6099
·
1 Parent(s): 10848ab

Invalidate AdamW tensor caches on load_state_dict [skip-build]

Browse files

Clear _placement_cache and _tensor_cache after loading state to prevent
stale tensor references from causing incorrect optimizer steps.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Files changed (1) hide show
  1. torch-ext/optimizer/muon.py +6 -1
torch-ext/optimizer/muon.py CHANGED
@@ -8,7 +8,7 @@ import torch.distributed as dist
8
  from torch.distributed.tensor import DTensor, Replicate, Shard
9
  from torch.profiler import record_function
10
 
11
- from .adamw import step_adamw
12
  from .async_utils import run_pipeline
13
  from .core import (_muon_state, adjust_lr_for_muon, batch_pre_ortho,
14
  get_default_muon_param_groups, is_expert_param, update_p)
@@ -1058,6 +1058,11 @@ class Muon(torch.optim.Optimizer):
1058
 
1059
  super().load_state_dict(state_dict)
1060
 
 
 
 
 
 
1061
  if self.cpu_offload:
1062
  # Re-create the offload pool since state tensors may be new
1063
  # objects after load_state_dict.
 
8
  from torch.distributed.tensor import DTensor, Replicate, Shard
9
  from torch.profiler import record_function
10
 
11
+ from .adamw import _placement_cache, _tensor_cache, step_adamw
12
  from .async_utils import run_pipeline
13
  from .core import (_muon_state, adjust_lr_for_muon, batch_pre_ortho,
14
  get_default_muon_param_groups, is_expert_param, update_p)
 
1058
 
1059
  super().load_state_dict(state_dict)
1060
 
1061
+ # Invalidate adamw.py's module-level tensor caches so that
1062
+ # the next step rebuilds them with the newly loaded state tensors.
1063
+ _placement_cache.clear()
1064
+ _tensor_cache.clear()
1065
+
1066
  if self.cpu_offload:
1067
  # Re-create the offload pool since state tensors may be new
1068
  # objects after load_state_dict.