natmin322 commited on
Commit
6f52dc6
·
1 Parent(s): d7fe0d8

fix: compat shim for ShardedDDPOption and sharded_ddp attribute

Browse files
root_gainlora/src/cl_trainer_gainlora_inflora.py CHANGED
@@ -21,6 +21,13 @@ try:
21
  except ImportError:
22
  ipdb = None
23
 
 
 
 
 
 
 
 
24
  def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100):
25
  predictions_ids = np.where(predictions_ids == ignore_idx, tokenizer.pad_token_id, predictions_ids)
26
 
@@ -643,7 +650,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer):
643
 
644
  optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args)
645
 
646
- if self.sharded_ddp == ShardedDDPOption.SIMPLE:
647
  self.optimizer = OSS(
648
  params=optimizer_grouped_parameters,
649
  optim=optimizer_cls,
@@ -1048,8 +1055,8 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer):
1048
  debug_overflow = DebugUnderflowOverflow(self.model) # noqa
1049
 
1050
  delay_optimizer_creation = (
1051
- self.sharded_ddp is not None
1052
- and self.sharded_ddp != ShardedDDPOption.SIMPLE
1053
  or is_sagemaker_mp_enabled()
1054
  or self.fsdp is not None
1055
  )
 
21
  except ImportError:
22
  ipdb = None
23
 
24
+ # Compat: ShardedDDPOption removed in transformers >= 4.40
25
+ try:
26
+ ShardedDDPOption
27
+ except NameError:
28
+ from types import SimpleNamespace
29
+ ShardedDDPOption = SimpleNamespace(SIMPLE='simple')
30
+
31
  def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100):
32
  predictions_ids = np.where(predictions_ids == ignore_idx, tokenizer.pad_token_id, predictions_ids)
33
 
 
650
 
651
  optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args)
652
 
653
+ if getattr(self, 'sharded_ddp', None) == ShardedDDPOption.SIMPLE:
654
  self.optimizer = OSS(
655
  params=optimizer_grouped_parameters,
656
  optim=optimizer_cls,
 
1055
  debug_overflow = DebugUnderflowOverflow(self.model) # noqa
1056
 
1057
  delay_optimizer_creation = (
1058
+ getattr(self, 'sharded_ddp', None) is not None
1059
+ and getattr(self, 'sharded_ddp', None) != ShardedDDPOption.SIMPLE
1060
  or is_sagemaker_mp_enabled()
1061
  or self.fsdp is not None
1062
  )