kovwang commited on
Commit
468d8dd
·
verified ·
1 Parent(s): 14bb1e9

Upload modeling_deepseek.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. modeling_deepseek.py +16 -2
modeling_deepseek.py CHANGED
@@ -521,14 +521,28 @@ class DeepseekV3MoE(nn.Module):
521
  config=config, intermediate_size=intermediate_size
522
  )
523
 
524
- def forward(self, hidden_states):
525
  identity = hidden_states
526
  orig_shape = hidden_states.shape
527
  topk_idx, topk_weight = self.gate(hidden_states)
528
  hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
529
  flat_topk_idx = topk_idx.view(-1)
530
- if not self.training:
 
 
 
 
 
 
 
 
 
 
 
 
 
531
  y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
 
532
  if self.config.n_shared_experts is not None:
533
  y = y + self.shared_experts(identity)
534
  return y
 
521
  config=config, intermediate_size=intermediate_size
522
  )
523
 
524
+ def forward(self, hidden_states):
525
  identity = hidden_states
526
  orig_shape = hidden_states.shape
527
  topk_idx, topk_weight = self.gate(hidden_states)
528
  hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
529
  flat_topk_idx = topk_idx.view(-1)
530
+
531
+ if self.training:
532
+ # 训练模式:处理MoE前向传播
533
+ hidden_states = hidden_states.repeat_interleave(self.num_experts_per_tok, dim=0)
534
+ y = torch.empty_like(hidden_states)
535
+ for i, expert in enumerate(self.experts):
536
+ if expert is not None: # 处理expert parallelism的情况
537
+ mask = (flat_topk_idx == i)
538
+ if mask.any():
539
+ y[mask] = expert(hidden_states[mask])
540
+ y = (y.view(*topk_weight.shape, -1) * topk_weight.unsqueeze(-1)).sum(dim=1)
541
+ y = y.view(*orig_shape)
542
+ else:
543
+ # 推理模式:使用moe_infer
544
  y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
545
+
546
  if self.config.n_shared_experts is not None:
547
  y = y + self.shared_experts(identity)
548
  return y