from typing import Callable, Dict, List, Optional, Tuple, Union import torch import torch.distributed as dist import torch.nn as nn import torch.nn.functional as F from torch.utils.data import Dataset from transformers.modeling_utils import PreTrainedModel from transformers.trainer import Trainer from train_cot.qwen3_vl_cot_embedding import pooling_last def _dist_info(group=None) -> Tuple[int, int]: if group is None or not (dist.is_available() and dist.is_initialized()): return 0, 1 return dist.get_rank(group=group), dist.get_world_size(group=group) def build_negative_group(group_size: int): """按 group_size 张卡划分负样本共享范围。 group_size = 1 -> 返回 None,只用本卡 batch 内的负样本 group_size = world_size -> 所有卡共享(<=0 也按全部处理) 其余 -> 每 group_size 张卡一组,组内共享(如 8 = 单机内共享) """ if not (dist.is_available() and dist.is_initialized()): return None world_size = dist.get_world_size() if world_size == 1 or group_size == 1: return None if group_size <= 0 or group_size > world_size: group_size = world_size if world_size % group_size != 0: raise ValueError( f"neg_gather_size={group_size} 无法整除 world_size={world_size}" ) if group_size == world_size: return dist.group.WORLD # new_group 是集合操作,每个进程都要按同样顺序把所有组建一遍 my_group = None for start in range(0, world_size, group_size): ranks = list(range(start, start + group_size)) new_group = dist.new_group(ranks=ranks) if dist.get_rank() in ranks: my_group = new_group return my_group def _dist_gather(t: torch.Tensor, group=None) -> torch.Tensor: rank, group_size = _dist_info(group) if group_size == 1: return t t = t.contiguous() gathered = [torch.empty_like(t) for _ in range(group_size)] dist.all_gather(gathered, t, group=group) # all_gather 不回传梯度,本卡那一段换回原 tensor 才能反向 gathered[rank] = t return torch.cat(gathered, dim=0) def infonce_loss( query_embeds: torch.Tensor, pos_embeds: torch.Tensor, neg_embeds: torch.Tensor, temperature: float = 0.02, group=None, ) -> torch.Tensor: """ InfoNCE with in-batch negatives + per-sample hard negative. Candidates per query_i (G = 组内卡数,group=None 时 G=1): positives : pos_i negatives : 组内所有 pos_j (j≠i) + 组内所有 neg_j logit matrix shape: (N, 2*G*N) labels: group_rank*N + arange(N) """ q = F.normalize(query_embeds, dim=-1) p = F.normalize(pos_embeds, dim=-1) n = F.normalize(neg_embeds, dim=-1) # doc pool = [组内各卡的 pos, 组内各卡的 neg] docs = torch.cat([_dist_gather(p, group), _dist_gather(n, group)], dim=0) logits = q @ docs.T / temperature rank, _ = _dist_info(group) labels = torch.arange(len(q), device=q.device) + rank * len(q) return F.cross_entropy(logits, labels) class VLM_CoT_Trainer(Trainer): """CoT embedding 训练器。 总 loss = infonce_weight * InfoNCE(embedding) + cot_weight * CrossEntropy(CoT)。 query 侧一次前向同时产出 CoT logits 与 embedding,doc 侧只取 embedding。 """ def __init__( self, model: Union["PreTrainedModel", nn.Module] = None, training_args: "TrainingArguments" = None, # type: ignore data_collator: Optional["DataCollator"] = None, # type: ignore train_dataset: Optional[Dataset] = None, eval_data_collator: Optional["DataCollator"] = None, # type: ignore eval_dataset: Optional[Union[Dataset, Dict[str, Dataset]]] = None, emb_dim: Optional[int] = None, temperature: float = 0.02, infonce_weight: float = 1.0, cot_weight: float = 1.0, neg_gather_size: int = 1, tokenizer: Optional["PreTrainedTokenizerBase"] = None, # type: ignore model_init: Optional[Callable[[], "PreTrainedModel"]] = None, compute_metrics: Optional[Callable[["EvalPrediction"], Dict]] = None, # type: ignore callbacks: Optional[List["TrainerCallback"]] = None, # type: ignore optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None), preprocess_logits_for_metrics: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None, ): super().__init__( model=model, args=training_args, data_collator=data_collator, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=tokenizer, model_init=model_init, compute_metrics=compute_metrics, callbacks=callbacks, optimizers=optimizers, preprocess_logits_for_metrics=preprocess_logits_for_metrics ) self.model = model self.emb_dim = emb_dim self.training_args = training_args self.data_collator = data_collator self.device = training_args.device self.eval_data_collator = eval_data_collator self.temperature = temperature self.infonce_weight = infonce_weight self.cot_weight = cot_weight self.neg_group = build_negative_group(neg_gather_size) self.sub_loss_dict = {} if self.args.process_index == 0: _, group_size = _dist_info(self.neg_group) pool = 2 * group_size * training_args.per_device_train_batch_size print(f"[infonce] neg_gather_size={neg_gather_size} -> 组内 {group_size} 卡共享," f"候选池 {pool},每个 query {pool - 1} 个负样本") def _pool(self, out) -> torch.Tensor: emb = pooling_last(out.last_hidden_state, out.attention_mask) if self.emb_dim and self.emb_dim < emb.shape[-1]: emb = emb[:, :self.emb_dim] return emb def _forward_and_pool(self, model, inputs_dict) -> torch.Tensor: return self._pool(model(**inputs_dict)) def compute_loss(self, model, inputs, num_items_in_batch=None, return_outputs=False): # query 侧:teacher forcing 走一遍,既拿 CoT loss 又拿 embedding query_outputs = model(**inputs["query_inputs"], labels=inputs["query_labels"]) anchor_emb = self._pool(query_outputs) positive_emb = self._forward_and_pool(model, inputs["positive_inputs"]) negative_emb = self._forward_and_pool(model, inputs["negative_inputs"]) # 计算损失 loss_infonce = infonce_loss( anchor_emb, positive_emb, negative_emb, temperature=self.temperature, group=self.neg_group, ) loss_cot = query_outputs.lm_loss loss = self.infonce_weight * loss_infonce + self.cot_weight * loss_cot if self.model.training: self.sub_loss_dict = { "train/loss_infonce": loss_infonce.item(), "train/loss_cot": loss_cot.item(), } return loss def log(self, logs: Dict[str, float], *args, **kwargs) -> None: # 把两路子 loss 一起写进日志/tensorboard,方便观察配比 if self.sub_loss_dict: logs.update(self.sub_loss_dict) super().log(logs, *args, **kwargs) def evaluate( self, eval_dataset: Optional[Union[Dataset, Dict[str, Dataset]]] = None, ignore_keys: Optional[List[str]] = None, metric_key_prefix: str = "eval", ) -> Dict[str, float]: # 现在不在训练中做 eval,直接跳过。 return {} def get_grad_norm(self, model, norm_type=2): total_norm = 0 for name, param in model.named_parameters(): if param.grad is not None: param_norm = param.grad.data.norm(norm_type) total_norm += param_norm.item() ** norm_type else: print(f"参数{name}梯度为None,跳过") total_norm = total_norm ** (1. / norm_type) return total_norm def check_model_nan_grad(self, loss, model): isnan_exists = False if loss is not None and torch.isnan(loss): loss = torch.tensor(0.0, requires_grad=True, device=self.device) print("loss存在NaN,跳过更新") isnan_exists = True for name, param in model.named_parameters(): if param.grad is not None: if torch.isnan(param.grad).any(): print(f"参数{name}梯度存在NaN,跳过更新") isnan_exists = True if isnan_exists: break if isnan_exists: model.zero_grad()