Instructions to use inclusionAI/Ling-3.1-flash with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Inference
- HuggingChat
- Notebooks
- Google Colab
- Kaggle
Fix expert_swiglu_limit_list in modeling
#10
by ChangxinTian - opened
- configuration_bailing_moe_v3.py +4 -0
- modeling_bailing_moe_v3.py +30 -11
configuration_bailing_moe_v3.py
CHANGED
|
@@ -61,6 +61,8 @@ class BailingMoeV3Config(PretrainedConfig):
|
|
| 61 |
kda_safe_gate=False,
|
| 62 |
kda_lower_bound=None,
|
| 63 |
short_conv_kernel_size=4,
|
|
|
|
|
|
|
| 64 |
**kwargs,
|
| 65 |
):
|
| 66 |
self.num_hidden_layers = num_hidden_layers
|
|
@@ -121,6 +123,8 @@ class BailingMoeV3Config(PretrainedConfig):
|
|
| 121 |
self.kda_safe_gate = kda_safe_gate
|
| 122 |
self.kda_lower_bound = kda_lower_bound
|
| 123 |
self.short_conv_kernel_size = short_conv_kernel_size
|
|
|
|
|
|
|
| 124 |
super().__init__(
|
| 125 |
pad_token_id=pad_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
|
| 126 |
)
|
|
|
|
| 61 |
kda_safe_gate=False,
|
| 62 |
kda_lower_bound=None,
|
| 63 |
short_conv_kernel_size=4,
|
| 64 |
+
expert_swiglu_limit_list=None,
|
| 65 |
+
share_expert_swiglu_limit_list=None,
|
| 66 |
**kwargs,
|
| 67 |
):
|
| 68 |
self.num_hidden_layers = num_hidden_layers
|
|
|
|
| 123 |
self.kda_safe_gate = kda_safe_gate
|
| 124 |
self.kda_lower_bound = kda_lower_bound
|
| 125 |
self.short_conv_kernel_size = short_conv_kernel_size
|
| 126 |
+
self.expert_swiglu_limit_list = expert_swiglu_limit_list
|
| 127 |
+
self.share_expert_swiglu_limit_list = share_expert_swiglu_limit_list
|
| 128 |
super().__init__(
|
| 129 |
pad_token_id=pad_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
|
| 130 |
)
|
modeling_bailing_moe_v3.py
CHANGED
|
@@ -327,11 +327,12 @@ def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
|
|
| 327 |
|
| 328 |
|
| 329 |
class BailingMoeV3MLP(nn.Module):
|
| 330 |
-
def __init__(self, config: BailingMoeV3Config, intermediate_size: int):
|
| 331 |
super().__init__()
|
| 332 |
self.config = config
|
| 333 |
self.hidden_size = config.hidden_size
|
| 334 |
self.intermediate_size = intermediate_size
|
|
|
|
| 335 |
|
| 336 |
self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 337 |
self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
|
@@ -339,7 +340,12 @@ class BailingMoeV3MLP(nn.Module):
|
|
| 339 |
self.act_fn = ACT2FN[config.hidden_act]
|
| 340 |
|
| 341 |
def forward(self, x):
|
| 342 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 343 |
|
| 344 |
|
| 345 |
class BailingMoeV3Gate(nn.Module):
|
|
@@ -411,21 +417,37 @@ class BailingMoeV3SparseMoeBlock(nn.Module):
|
|
| 411 |
A mixed expert module containing shared experts.
|
| 412 |
"""
|
| 413 |
|
| 414 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
super().__init__()
|
| 416 |
self.config = config
|
|
|
|
|
|
|
|
|
|
|
|
|
| 417 |
self.num_experts_per_tok = config.num_experts_per_tok
|
| 418 |
self._setup_experts()
|
| 419 |
self.gate = BailingMoeV3Gate(config)
|
| 420 |
if config.num_shared_experts is not None:
|
| 421 |
self.shared_experts = BailingMoeV3MLP(
|
| 422 |
-
config=config,
|
|
|
|
|
|
|
| 423 |
)
|
| 424 |
|
| 425 |
def _setup_experts(self):
|
| 426 |
self.experts = nn.ModuleList(
|
| 427 |
[
|
| 428 |
-
BailingMoeV3MLP(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 429 |
for _ in range(self.config.num_experts)
|
| 430 |
]
|
| 431 |
)
|
|
@@ -543,7 +565,6 @@ def apply_rotary_pos_emb_interleave(q, k, cos, sin, position_ids=None, unsqueeze
|
|
| 543 |
TODO let's just use the original freqcis computation to not have the view
|
| 544 |
transpose + reshape! This is not optimized!
|
| 545 |
Applies Rotary Position Embedding to the query and key tensors.
|
| 546 |
-
|
| 547 |
Args:
|
| 548 |
q (`torch.Tensor`): The query tensor.
|
| 549 |
k (`torch.Tensor`): The key tensor.
|
|
@@ -934,7 +955,7 @@ class BailingMoeV3MTPLayer(nn.Module):
|
|
| 934 |
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
| 935 |
self.post_attention_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 936 |
self.attention = BailingMoeV3MultiLatentAttention(config=config, layer_idx=layer_idx)
|
| 937 |
-
self.mlp = BailingMoeV3SparseMoeBlock(config)
|
| 938 |
|
| 939 |
self.hnorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 940 |
self.final_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
|
@@ -1014,7 +1035,7 @@ class BailingMoeV3DecoderLayer(nn.Module):
|
|
| 1014 |
self.attention = BailingMoeV3KimiDeltaAttention(config=config, layer_idx=layer_idx)
|
| 1015 |
|
| 1016 |
self.mlp = (
|
| 1017 |
-
BailingMoeV3SparseMoeBlock(config)
|
| 1018 |
if (config.num_experts is not None and layer_idx >= config.first_k_dense_replace)
|
| 1019 |
else BailingMoeV3MLP(config=config, intermediate_size=config.intermediate_size)
|
| 1020 |
)
|
|
@@ -1552,9 +1573,7 @@ class BailingMoeV3ForCausalLM(BailingMoeV3PreTrainedModel, GenerationMixin):
|
|
| 1552 |
output_router_logits if output_router_logits is not None else self.config.output_router_logits
|
| 1553 |
)
|
| 1554 |
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1555 |
-
compute_mtp_loss =
|
| 1556 |
-
labels is not None and self.num_nextn_predict_layers > 0 and self.mtp_loss_scaling_factor > 0
|
| 1557 |
-
)
|
| 1558 |
# MTP hidden states are kept outside the tuple output, so retain the structured output while
|
| 1559 |
# computing its loss.
|
| 1560 |
model_return_dict = return_dict or compute_mtp_loss
|
|
|
|
| 327 |
|
| 328 |
|
| 329 |
class BailingMoeV3MLP(nn.Module):
|
| 330 |
+
def __init__(self, config: BailingMoeV3Config, intermediate_size: int, swiglu_limit: Optional[float] = None):
|
| 331 |
super().__init__()
|
| 332 |
self.config = config
|
| 333 |
self.hidden_size = config.hidden_size
|
| 334 |
self.intermediate_size = intermediate_size
|
| 335 |
+
self.swiglu_limit = swiglu_limit
|
| 336 |
|
| 337 |
self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 338 |
self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
|
|
|
| 340 |
self.act_fn = ACT2FN[config.hidden_act]
|
| 341 |
|
| 342 |
def forward(self, x):
|
| 343 |
+
gate = self.act_fn(self.gate_proj(x))
|
| 344 |
+
up = self.up_proj(x)
|
| 345 |
+
if self.swiglu_limit is not None:
|
| 346 |
+
gate = gate.clamp(max=self.swiglu_limit)
|
| 347 |
+
up = up.clamp(min=-self.swiglu_limit, max=self.swiglu_limit)
|
| 348 |
+
return self.down_proj(gate * up)
|
| 349 |
|
| 350 |
|
| 351 |
class BailingMoeV3Gate(nn.Module):
|
|
|
|
| 417 |
A mixed expert module containing shared experts.
|
| 418 |
"""
|
| 419 |
|
| 420 |
+
@staticmethod
|
| 421 |
+
def _get_swiglu_limit(limit_list, layer_idx):
|
| 422 |
+
if limit_list is None or not 0 <= layer_idx < len(limit_list):
|
| 423 |
+
return None
|
| 424 |
+
return limit_list[layer_idx] or None
|
| 425 |
+
|
| 426 |
+
def __init__(self, config: BailingMoeV3Config, layer_idx: int):
|
| 427 |
super().__init__()
|
| 428 |
self.config = config
|
| 429 |
+
self.expert_swiglu_limit = self._get_swiglu_limit(getattr(config, "expert_swiglu_limit_list", None), layer_idx)
|
| 430 |
+
self.share_expert_swiglu_limit = self._get_swiglu_limit(
|
| 431 |
+
getattr(config, "share_expert_swiglu_limit_list", None), layer_idx
|
| 432 |
+
)
|
| 433 |
self.num_experts_per_tok = config.num_experts_per_tok
|
| 434 |
self._setup_experts()
|
| 435 |
self.gate = BailingMoeV3Gate(config)
|
| 436 |
if config.num_shared_experts is not None:
|
| 437 |
self.shared_experts = BailingMoeV3MLP(
|
| 438 |
+
config=config,
|
| 439 |
+
intermediate_size=config.moe_shared_expert_intermediate_size * config.num_shared_experts,
|
| 440 |
+
swiglu_limit=self.share_expert_swiglu_limit,
|
| 441 |
)
|
| 442 |
|
| 443 |
def _setup_experts(self):
|
| 444 |
self.experts = nn.ModuleList(
|
| 445 |
[
|
| 446 |
+
BailingMoeV3MLP(
|
| 447 |
+
config=self.config,
|
| 448 |
+
intermediate_size=self.config.moe_intermediate_size,
|
| 449 |
+
swiglu_limit=self.expert_swiglu_limit,
|
| 450 |
+
)
|
| 451 |
for _ in range(self.config.num_experts)
|
| 452 |
]
|
| 453 |
)
|
|
|
|
| 565 |
TODO let's just use the original freqcis computation to not have the view
|
| 566 |
transpose + reshape! This is not optimized!
|
| 567 |
Applies Rotary Position Embedding to the query and key tensors.
|
|
|
|
| 568 |
Args:
|
| 569 |
q (`torch.Tensor`): The query tensor.
|
| 570 |
k (`torch.Tensor`): The key tensor.
|
|
|
|
| 955 |
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
| 956 |
self.post_attention_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 957 |
self.attention = BailingMoeV3MultiLatentAttention(config=config, layer_idx=layer_idx)
|
| 958 |
+
self.mlp = BailingMoeV3SparseMoeBlock(config, layer_idx=layer_idx)
|
| 959 |
|
| 960 |
self.hnorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 961 |
self.final_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
|
|
|
| 1035 |
self.attention = BailingMoeV3KimiDeltaAttention(config=config, layer_idx=layer_idx)
|
| 1036 |
|
| 1037 |
self.mlp = (
|
| 1038 |
+
BailingMoeV3SparseMoeBlock(config, layer_idx=layer_idx)
|
| 1039 |
if (config.num_experts is not None and layer_idx >= config.first_k_dense_replace)
|
| 1040 |
else BailingMoeV3MLP(config=config, intermediate_size=config.intermediate_size)
|
| 1041 |
)
|
|
|
|
| 1573 |
output_router_logits if output_router_logits is not None else self.config.output_router_logits
|
| 1574 |
)
|
| 1575 |
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1576 |
+
compute_mtp_loss = labels is not None and self.num_nextn_predict_layers > 0 and self.mtp_loss_scaling_factor > 0
|
|
|
|
|
|
|
| 1577 |
# MTP hidden states are kept outside the tuple output, so retain the structured output while
|
| 1578 |
# computing its loss.
|
| 1579 |
model_return_dict = return_dict or compute_mtp_loss
|