Update moe.py
Browse files
moe.py
CHANGED
|
@@ -7,7 +7,7 @@ import torch.nn as nn
|
|
| 7 |
import torch.nn.functional as F
|
| 8 |
|
| 9 |
from transformers import PreTrainedModel, PretrainedConfig
|
| 10 |
-
from transformers import AutoConfig, AutoModelForCausalLM
|
| 11 |
|
| 12 |
|
| 13 |
from transformers.modeling_outputs import (
|
|
@@ -323,6 +323,7 @@ class MoELMConfig(PretrainedConfig):
|
|
| 323 |
|
| 324 |
class MoEModel(PreTrainedModel):
|
| 325 |
config_class = MoELMConfig
|
|
|
|
| 326 |
|
| 327 |
def __init__(self, config: MoELMConfig):
|
| 328 |
super().__init__(config)
|
|
@@ -472,6 +473,7 @@ class MoEModel(PreTrainedModel):
|
|
| 472 |
|
| 473 |
class MoEForCausalLM(PreTrainedModel):
|
| 474 |
config_class = MoELMConfig
|
|
|
|
| 475 |
|
| 476 |
def __init__(self, config: MoELMConfig):
|
| 477 |
super().__init__(config)
|
|
@@ -553,6 +555,7 @@ class MoEForCausalLM(PreTrainedModel):
|
|
| 553 |
|
| 554 |
AutoConfig.register("moe_lm", MoELMConfig)
|
| 555 |
AutoModelForCausalLM.register(MoELMConfig, MoEForCausalLM)
|
|
|
|
| 556 |
|
| 557 |
|
| 558 |
|
|
|
|
| 7 |
import torch.nn.functional as F
|
| 8 |
|
| 9 |
from transformers import PreTrainedModel, PretrainedConfig
|
| 10 |
+
from transformers import AutoConfig, AutoModelForCausalLM, AutoModel
|
| 11 |
|
| 12 |
|
| 13 |
from transformers.modeling_outputs import (
|
|
|
|
| 323 |
|
| 324 |
class MoEModel(PreTrainedModel):
|
| 325 |
config_class = MoELMConfig
|
| 326 |
+
base_model_prefix = "model"
|
| 327 |
|
| 328 |
def __init__(self, config: MoELMConfig):
|
| 329 |
super().__init__(config)
|
|
|
|
| 473 |
|
| 474 |
class MoEForCausalLM(PreTrainedModel):
|
| 475 |
config_class = MoELMConfig
|
| 476 |
+
base_model_prefix = "model"
|
| 477 |
|
| 478 |
def __init__(self, config: MoELMConfig):
|
| 479 |
super().__init__(config)
|
|
|
|
| 555 |
|
| 556 |
AutoConfig.register("moe_lm", MoELMConfig)
|
| 557 |
AutoModelForCausalLM.register(MoELMConfig, MoEForCausalLM)
|
| 558 |
+
AutoModel.register(MoELMConfig, MoEModel)
|
| 559 |
|
| 560 |
|
| 561 |
|