Jtapsa commited on
Commit
4121139
·
verified ·
1 Parent(s): f1f2803

Update moe.py

Browse files
Files changed (1) hide show
  1. moe.py +4 -1
moe.py CHANGED
@@ -7,7 +7,7 @@ import torch.nn as nn
7
  import torch.nn.functional as F
8
 
9
  from transformers import PreTrainedModel, PretrainedConfig
10
- from transformers import AutoConfig, AutoModelForCausalLM
11
 
12
 
13
  from transformers.modeling_outputs import (
@@ -323,6 +323,7 @@ class MoELMConfig(PretrainedConfig):
323
 
324
  class MoEModel(PreTrainedModel):
325
  config_class = MoELMConfig
 
326
 
327
  def __init__(self, config: MoELMConfig):
328
  super().__init__(config)
@@ -472,6 +473,7 @@ class MoEModel(PreTrainedModel):
472
 
473
  class MoEForCausalLM(PreTrainedModel):
474
  config_class = MoELMConfig
 
475
 
476
  def __init__(self, config: MoELMConfig):
477
  super().__init__(config)
@@ -553,6 +555,7 @@ class MoEForCausalLM(PreTrainedModel):
553
 
554
  AutoConfig.register("moe_lm", MoELMConfig)
555
  AutoModelForCausalLM.register(MoELMConfig, MoEForCausalLM)
 
556
 
557
 
558
 
 
7
  import torch.nn.functional as F
8
 
9
  from transformers import PreTrainedModel, PretrainedConfig
10
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoModel
11
 
12
 
13
  from transformers.modeling_outputs import (
 
323
 
324
  class MoEModel(PreTrainedModel):
325
  config_class = MoELMConfig
326
+ base_model_prefix = "model"
327
 
328
  def __init__(self, config: MoELMConfig):
329
  super().__init__(config)
 
473
 
474
  class MoEForCausalLM(PreTrainedModel):
475
  config_class = MoELMConfig
476
+ base_model_prefix = "model"
477
 
478
  def __init__(self, config: MoELMConfig):
479
  super().__init__(config)
 
555
 
556
  AutoConfig.register("moe_lm", MoELMConfig)
557
  AutoModelForCausalLM.register(MoELMConfig, MoEForCausalLM)
558
+ AutoModel.register(MoELMConfig, MoEModel)
559
 
560
 
561