liqi / code /multi_vlm2vec_train /train_cot /trainer_qwen3vl_cot.py
WaitXian's picture
Upload liqi files
0ab2566 verified
Raw History Blame Contribute Delete
8.86 kB
from typing import Callable, Dict, List, Optional, Tuple, Union
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset
from transformers.modeling_utils import PreTrainedModel
from transformers.trainer import Trainer
from train_cot.qwen3_vl_cot_embedding import pooling_last
def _dist_info(group=None) -> Tuple[int, int]:
if group is None or not (dist.is_available() and dist.is_initialized()):
return 0, 1
return dist.get_rank(group=group), dist.get_world_size(group=group)
def build_negative_group(group_size: int):
"""按 group_size 张卡划分负样本共享范围。
group_size = 1 -> 返回 None,只用本卡 batch 内的负样本
group_size = world_size -> 所有卡共享(<=0 也按全部处理)
其余 -> 每 group_size 张卡一组,组内共享(如 8 = 单机内共享)
"""
if not (dist.is_available() and dist.is_initialized()):
return None
world_size = dist.get_world_size()
if world_size == 1 or group_size == 1:
return None
if group_size <= 0 or group_size > world_size:
group_size = world_size
if world_size % group_size != 0:
raise ValueError(
f"neg_gather_size={group_size} 无法整除 world_size={world_size}"
)
if group_size == world_size:
return dist.group.WORLD
# new_group 是集合操作,每个进程都要按同样顺序把所有组建一遍
my_group = None
for start in range(0, world_size, group_size):
ranks = list(range(start, start + group_size))
new_group = dist.new_group(ranks=ranks)
if dist.get_rank() in ranks:
my_group = new_group
return my_group
def _dist_gather(t: torch.Tensor, group=None) -> torch.Tensor:
rank, group_size = _dist_info(group)
if group_size == 1:
return t
t = t.contiguous()
gathered = [torch.empty_like(t) for _ in range(group_size)]
dist.all_gather(gathered, t, group=group)
# all_gather 不回传梯度,本卡那一段换回原 tensor 才能反向
gathered[rank] = t
return torch.cat(gathered, dim=0)
def infonce_loss(
query_embeds: torch.Tensor,
pos_embeds: torch.Tensor,
neg_embeds: torch.Tensor,
temperature: float = 0.02,
group=None,
) -> torch.Tensor:
"""
InfoNCE with in-batch negatives + per-sample hard negative.
Candidates per query_i (G = 组内卡数,group=None 时 G=1):
positives : pos_i
negatives : 组内所有 pos_j (j≠i) + 组内所有 neg_j
logit matrix shape: (N, 2*G*N) labels: group_rank*N + arange(N)
"""
q = F.normalize(query_embeds, dim=-1)
p = F.normalize(pos_embeds, dim=-1)
n = F.normalize(neg_embeds, dim=-1)
# doc pool = [组内各卡的 pos, 组内各卡的 neg]
docs = torch.cat([_dist_gather(p, group), _dist_gather(n, group)], dim=0)
logits = q @ docs.T / temperature
rank, _ = _dist_info(group)
labels = torch.arange(len(q), device=q.device) + rank * len(q)
return F.cross_entropy(logits, labels)
class VLM_CoT_Trainer(Trainer):
"""CoT embedding 训练器。
总 loss = infonce_weight * InfoNCE(embedding) + cot_weight * CrossEntropy(CoT)。
query 侧一次前向同时产出 CoT logits 与 embedding,doc 侧只取 embedding。
"""
def __init__(
self,
model: Union["PreTrainedModel", nn.Module] = None,
training_args: "TrainingArguments" = None, # type: ignore
data_collator: Optional["DataCollator"] = None, # type: ignore
train_dataset: Optional[Dataset] = None,
eval_data_collator: Optional["DataCollator"] = None, # type: ignore
eval_dataset: Optional[Union[Dataset, Dict[str, Dataset]]] = None,
emb_dim: Optional[int] = None,
temperature: float = 0.02,
infonce_weight: float = 1.0,
cot_weight: float = 1.0,
neg_gather_size: int = 1,
tokenizer: Optional["PreTrainedTokenizerBase"] = None, # type: ignore
model_init: Optional[Callable[[], "PreTrainedModel"]] = None,
compute_metrics: Optional[Callable[["EvalPrediction"], Dict]] = None, # type: ignore
callbacks: Optional[List["TrainerCallback"]] = None, # type: ignore
optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None),
preprocess_logits_for_metrics: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None,
):
super().__init__(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
processing_class=tokenizer,
model_init=model_init,
compute_metrics=compute_metrics,
callbacks=callbacks,
optimizers=optimizers,
preprocess_logits_for_metrics=preprocess_logits_for_metrics
)
self.model = model
self.emb_dim = emb_dim
self.training_args = training_args
self.data_collator = data_collator
self.device = training_args.device
self.eval_data_collator = eval_data_collator
self.temperature = temperature
self.infonce_weight = infonce_weight
self.cot_weight = cot_weight
self.neg_group = build_negative_group(neg_gather_size)
self.sub_loss_dict = {}
if self.args.process_index == 0:
_, group_size = _dist_info(self.neg_group)
pool = 2 * group_size * training_args.per_device_train_batch_size
print(f"[infonce] neg_gather_size={neg_gather_size} -> 组内 {group_size} 卡共享,"
f"候选池 {pool},每个 query {pool - 1} 个负样本")
def _pool(self, out) -> torch.Tensor:
emb = pooling_last(out.last_hidden_state, out.attention_mask)
if self.emb_dim and self.emb_dim < emb.shape[-1]:
emb = emb[:, :self.emb_dim]
return emb
def _forward_and_pool(self, model, inputs_dict) -> torch.Tensor:
return self._pool(model(**inputs_dict))
def compute_loss(self, model, inputs, num_items_in_batch=None, return_outputs=False):
# query 侧:teacher forcing 走一遍,既拿 CoT loss 又拿 embedding
query_outputs = model(**inputs["query_inputs"], labels=inputs["query_labels"])
anchor_emb = self._pool(query_outputs)
positive_emb = self._forward_and_pool(model, inputs["positive_inputs"])
negative_emb = self._forward_and_pool(model, inputs["negative_inputs"])
# 计算损失
loss_infonce = infonce_loss(
anchor_emb, positive_emb, negative_emb,
temperature=self.temperature, group=self.neg_group,
)
loss_cot = query_outputs.lm_loss
loss = self.infonce_weight * loss_infonce + self.cot_weight * loss_cot
if self.model.training:
self.sub_loss_dict = {
"train/loss_infonce": loss_infonce.item(),
"train/loss_cot": loss_cot.item(),
}
return loss
def log(self, logs: Dict[str, float], *args, **kwargs) -> None:
# 把两路子 loss 一起写进日志/tensorboard,方便观察配比
if self.sub_loss_dict:
logs.update(self.sub_loss_dict)
super().log(logs, *args, **kwargs)
def evaluate(
self,
eval_dataset: Optional[Union[Dataset, Dict[str, Dataset]]] = None,
ignore_keys: Optional[List[str]] = None,
metric_key_prefix: str = "eval",
) -> Dict[str, float]:
# 现在不在训练中做 eval,直接跳过。
return {}
def get_grad_norm(self, model, norm_type=2):
total_norm = 0
for name, param in model.named_parameters():
if param.grad is not None:
param_norm = param.grad.data.norm(norm_type)
total_norm += param_norm.item() ** norm_type
else:
print(f"参数{name}梯度为None,跳过")
total_norm = total_norm ** (1. / norm_type)
return total_norm
def check_model_nan_grad(self, loss, model):
isnan_exists = False
if loss is not None and torch.isnan(loss):
loss = torch.tensor(0.0, requires_grad=True, device=self.device)
print("loss存在NaN,跳过更新")
isnan_exists = True
for name, param in model.named_parameters():
if param.grad is not None:
if torch.isnan(param.grad).any():
print(f"参数{name}梯度存在NaN,跳过更新")
isnan_exists = True
if isnan_exists:
break
if isnan_exists:
model.zero_grad()