真文 commited on
Commit
d0ade7e
·
1 Parent(s): 276cca7

fix: swiglu_limit

Browse files
configuration_bailing_moe_v3.py CHANGED
@@ -61,6 +61,8 @@ class BailingMoeV3Config(PretrainedConfig):
61
  kda_safe_gate=False,
62
  kda_lower_bound=None,
63
  short_conv_kernel_size=4,
 
 
64
  **kwargs,
65
  ):
66
  self.num_hidden_layers = num_hidden_layers
@@ -121,6 +123,8 @@ class BailingMoeV3Config(PretrainedConfig):
121
  self.kda_safe_gate = kda_safe_gate
122
  self.kda_lower_bound = kda_lower_bound
123
  self.short_conv_kernel_size = short_conv_kernel_size
 
 
124
  super().__init__(
125
  pad_token_id=pad_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
126
  )
 
61
  kda_safe_gate=False,
62
  kda_lower_bound=None,
63
  short_conv_kernel_size=4,
64
+ expert_swiglu_limit_list=None,
65
+ share_expert_swiglu_limit_list=None,
66
  **kwargs,
67
  ):
68
  self.num_hidden_layers = num_hidden_layers
 
123
  self.kda_safe_gate = kda_safe_gate
124
  self.kda_lower_bound = kda_lower_bound
125
  self.short_conv_kernel_size = short_conv_kernel_size
126
+ self.expert_swiglu_limit_list = expert_swiglu_limit_list
127
+ self.share_expert_swiglu_limit_list = share_expert_swiglu_limit_list
128
  super().__init__(
129
  pad_token_id=pad_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
130
  )
modeling_bailing_moe_v3.py CHANGED
@@ -327,11 +327,12 @@ def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
327
 
328
 
329
  class BailingMoeV3MLP(nn.Module):
330
- def __init__(self, config: BailingMoeV3Config, intermediate_size: int):
331
  super().__init__()
332
  self.config = config
333
  self.hidden_size = config.hidden_size
334
  self.intermediate_size = intermediate_size
 
335
 
336
  self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
337
  self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
@@ -339,7 +340,12 @@ class BailingMoeV3MLP(nn.Module):
339
  self.act_fn = ACT2FN[config.hidden_act]
340
 
341
  def forward(self, x):
342
- return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
 
 
 
 
 
343
 
344
 
345
  class BailingMoeV3Gate(nn.Module):
@@ -411,21 +417,37 @@ class BailingMoeV3SparseMoeBlock(nn.Module):
411
  A mixed expert module containing shared experts.
412
  """
413
 
414
- def __init__(self, config: BailingMoeV3Config):
 
 
 
 
 
 
415
  super().__init__()
416
  self.config = config
 
 
 
 
417
  self.num_experts_per_tok = config.num_experts_per_tok
418
  self._setup_experts()
419
  self.gate = BailingMoeV3Gate(config)
420
  if config.num_shared_experts is not None:
421
  self.shared_experts = BailingMoeV3MLP(
422
- config=config, intermediate_size=config.moe_shared_expert_intermediate_size * config.num_shared_experts
 
 
423
  )
424
 
425
  def _setup_experts(self):
426
  self.experts = nn.ModuleList(
427
  [
428
- BailingMoeV3MLP(config=self.config, intermediate_size=self.config.moe_intermediate_size)
 
 
 
 
429
  for _ in range(self.config.num_experts)
430
  ]
431
  )
@@ -543,7 +565,6 @@ def apply_rotary_pos_emb_interleave(q, k, cos, sin, position_ids=None, unsqueeze
543
  TODO let's just use the original freqcis computation to not have the view
544
  transpose + reshape! This is not optimized!
545
  Applies Rotary Position Embedding to the query and key tensors.
546
-
547
  Args:
548
  q (`torch.Tensor`): The query tensor.
549
  k (`torch.Tensor`): The key tensor.
@@ -934,7 +955,7 @@ class BailingMoeV3MTPLayer(nn.Module):
934
  self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
935
  self.post_attention_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
936
  self.attention = BailingMoeV3MultiLatentAttention(config=config, layer_idx=layer_idx)
937
- self.mlp = BailingMoeV3SparseMoeBlock(config)
938
 
939
  self.hnorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
940
  self.final_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -1014,7 +1035,7 @@ class BailingMoeV3DecoderLayer(nn.Module):
1014
  self.attention = BailingMoeV3KimiDeltaAttention(config=config, layer_idx=layer_idx)
1015
 
1016
  self.mlp = (
1017
- BailingMoeV3SparseMoeBlock(config)
1018
  if (config.num_experts is not None and layer_idx >= config.first_k_dense_replace)
1019
  else BailingMoeV3MLP(config=config, intermediate_size=config.intermediate_size)
1020
  )
@@ -1552,9 +1573,7 @@ class BailingMoeV3ForCausalLM(BailingMoeV3PreTrainedModel, GenerationMixin):
1552
  output_router_logits if output_router_logits is not None else self.config.output_router_logits
1553
  )
1554
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1555
- compute_mtp_loss = (
1556
- labels is not None and self.num_nextn_predict_layers > 0 and self.mtp_loss_scaling_factor > 0
1557
- )
1558
  # MTP hidden states are kept outside the tuple output, so retain the structured output while
1559
  # computing its loss.
1560
  model_return_dict = return_dict or compute_mtp_loss
 
327
 
328
 
329
  class BailingMoeV3MLP(nn.Module):
330
+ def __init__(self, config: BailingMoeV3Config, intermediate_size: int, swiglu_limit: Optional[float] = None):
331
  super().__init__()
332
  self.config = config
333
  self.hidden_size = config.hidden_size
334
  self.intermediate_size = intermediate_size
335
+ self.swiglu_limit = swiglu_limit
336
 
337
  self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
338
  self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
 
340
  self.act_fn = ACT2FN[config.hidden_act]
341
 
342
  def forward(self, x):
343
+ gate = self.act_fn(self.gate_proj(x))
344
+ up = self.up_proj(x)
345
+ if self.swiglu_limit is not None:
346
+ gate = gate.clamp(max=self.swiglu_limit)
347
+ up = up.clamp(min=-self.swiglu_limit, max=self.swiglu_limit)
348
+ return self.down_proj(gate * up)
349
 
350
 
351
  class BailingMoeV3Gate(nn.Module):
 
417
  A mixed expert module containing shared experts.
418
  """
419
 
420
+ @staticmethod
421
+ def _get_swiglu_limit(limit_list, layer_idx):
422
+ if limit_list is None or not 0 <= layer_idx < len(limit_list):
423
+ return None
424
+ return limit_list[layer_idx] or None
425
+
426
+ def __init__(self, config: BailingMoeV3Config, layer_idx: int):
427
  super().__init__()
428
  self.config = config
429
+ self.expert_swiglu_limit = self._get_swiglu_limit(getattr(config, "expert_swiglu_limit_list", None), layer_idx)
430
+ self.share_expert_swiglu_limit = self._get_swiglu_limit(
431
+ getattr(config, "share_expert_swiglu_limit_list", None), layer_idx
432
+ )
433
  self.num_experts_per_tok = config.num_experts_per_tok
434
  self._setup_experts()
435
  self.gate = BailingMoeV3Gate(config)
436
  if config.num_shared_experts is not None:
437
  self.shared_experts = BailingMoeV3MLP(
438
+ config=config,
439
+ intermediate_size=config.moe_shared_expert_intermediate_size * config.num_shared_experts,
440
+ swiglu_limit=self.share_expert_swiglu_limit,
441
  )
442
 
443
  def _setup_experts(self):
444
  self.experts = nn.ModuleList(
445
  [
446
+ BailingMoeV3MLP(
447
+ config=self.config,
448
+ intermediate_size=self.config.moe_intermediate_size,
449
+ swiglu_limit=self.expert_swiglu_limit,
450
+ )
451
  for _ in range(self.config.num_experts)
452
  ]
453
  )
 
565
  TODO let's just use the original freqcis computation to not have the view
566
  transpose + reshape! This is not optimized!
567
  Applies Rotary Position Embedding to the query and key tensors.
 
568
  Args:
569
  q (`torch.Tensor`): The query tensor.
570
  k (`torch.Tensor`): The key tensor.
 
955
  self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
956
  self.post_attention_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
957
  self.attention = BailingMoeV3MultiLatentAttention(config=config, layer_idx=layer_idx)
958
+ self.mlp = BailingMoeV3SparseMoeBlock(config, layer_idx=layer_idx)
959
 
960
  self.hnorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
961
  self.final_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
 
1035
  self.attention = BailingMoeV3KimiDeltaAttention(config=config, layer_idx=layer_idx)
1036
 
1037
  self.mlp = (
1038
+ BailingMoeV3SparseMoeBlock(config, layer_idx=layer_idx)
1039
  if (config.num_experts is not None and layer_idx >= config.first_k_dense_replace)
1040
  else BailingMoeV3MLP(config=config, intermediate_size=config.intermediate_size)
1041
  )
 
1573
  output_router_logits if output_router_logits is not None else self.config.output_router_logits
1574
  )
1575
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1576
+ compute_mtp_loss = labels is not None and self.num_nextn_predict_layers > 0 and self.mtp_loss_scaling_factor > 0
 
 
1577
  # MTP hidden states are kept outside the tuple output, so retain the structured output while
1578
  # computing its loss.
1579
  model_return_dict = return_dict or compute_mtp_loss