fix: compat shim for ShardedDDPOption and sharded_ddp attribute
Browse files
root_gainlora/src/cl_trainer_gainlora_inflora.py
CHANGED
|
@@ -21,6 +21,13 @@ try:
|
|
| 21 |
except ImportError:
|
| 22 |
ipdb = None
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100):
|
| 25 |
predictions_ids = np.where(predictions_ids == ignore_idx, tokenizer.pad_token_id, predictions_ids)
|
| 26 |
|
|
@@ -643,7 +650,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer):
|
|
| 643 |
|
| 644 |
optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args)
|
| 645 |
|
| 646 |
-
if self
|
| 647 |
self.optimizer = OSS(
|
| 648 |
params=optimizer_grouped_parameters,
|
| 649 |
optim=optimizer_cls,
|
|
@@ -1048,8 +1055,8 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer):
|
|
| 1048 |
debug_overflow = DebugUnderflowOverflow(self.model) # noqa
|
| 1049 |
|
| 1050 |
delay_optimizer_creation = (
|
| 1051 |
-
self
|
| 1052 |
-
and self
|
| 1053 |
or is_sagemaker_mp_enabled()
|
| 1054 |
or self.fsdp is not None
|
| 1055 |
)
|
|
|
|
| 21 |
except ImportError:
|
| 22 |
ipdb = None
|
| 23 |
|
| 24 |
+
# Compat: ShardedDDPOption removed in transformers >= 4.40
|
| 25 |
+
try:
|
| 26 |
+
ShardedDDPOption
|
| 27 |
+
except NameError:
|
| 28 |
+
from types import SimpleNamespace
|
| 29 |
+
ShardedDDPOption = SimpleNamespace(SIMPLE='simple')
|
| 30 |
+
|
| 31 |
def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100):
|
| 32 |
predictions_ids = np.where(predictions_ids == ignore_idx, tokenizer.pad_token_id, predictions_ids)
|
| 33 |
|
|
|
|
| 650 |
|
| 651 |
optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args)
|
| 652 |
|
| 653 |
+
if getattr(self, 'sharded_ddp', None) == ShardedDDPOption.SIMPLE:
|
| 654 |
self.optimizer = OSS(
|
| 655 |
params=optimizer_grouped_parameters,
|
| 656 |
optim=optimizer_cls,
|
|
|
|
| 1055 |
debug_overflow = DebugUnderflowOverflow(self.model) # noqa
|
| 1056 |
|
| 1057 |
delay_optimizer_creation = (
|
| 1058 |
+
getattr(self, 'sharded_ddp', None) is not None
|
| 1059 |
+
and getattr(self, 'sharded_ddp', None) != ShardedDDPOption.SIMPLE
|
| 1060 |
or is_sagemaker_mp_enabled()
|
| 1061 |
or self.fsdp is not None
|
| 1062 |
)
|