import torch from kdflow.loss import build_loss_fn from kdflow.algorithms import register_algorithm from kdflow.loss.chunked_loss import chunked_loss from kdflow.loss.cross_entropy import compute_cross_entropy from kdflow.metrics.entropy import compute_entropy from kdflow.utils.logging_utils import init_logger logger = init_logger(__name__) @register_algorithm("simple_ctkd") class SimpleCrossTokenizerKD: """Simply find the overlap tokens between student and teacher tokenizer, and only compute KD loss on this sub-vocabulary. Motivation: modern LLMs have a large amount of shared tokens. """ def __init__( self, strategy, student_model, teacher_lm_head, student_tokenizer, teacher_tokenizer, **kwargs ): self.strategy = strategy self.args = strategy.args self.student = student_model self.teacher_lm_head = teacher_lm_head self.student_tokenizer = student_tokenizer self.teacher_tokenizer = teacher_tokenizer self.student_overlap_token_ids, self.teacher_overlap_token_ids = self._find_overlap_tokens() self.loss_fn = build_loss_fn(self.args.kd.kd_loss_fn, self.args) self.metric_fns = [compute_entropy] if self.args.scenario == "on_policy_kd" else [] def _find_overlap_tokens(self): student_vocab = {k.replace("Ġ", "▁"): v for k, v in self.student_tokenizer.get_vocab().items()} teacher_vocab = {k.replace("Ġ", "▁"): v for k, v in self.teacher_tokenizer.get_vocab().items()} overlap_tokens = set(student_vocab.keys()) & set(teacher_vocab.keys()) student_ids = [student_vocab[token] for token in overlap_tokens] teacher_ids = [teacher_vocab[token] for token in overlap_tokens] stu_eos, tea_eos = self.student_tokenizer.eos_token_id, self.teacher_tokenizer.eos_token_id if stu_eos not in student_ids or tea_eos not in teacher_ids: student_ids.append(stu_eos) teacher_ids.append(tea_eos) device = self.teacher_lm_head.weight.device logger.info(f"Num of overlap_tokens between student & teacher: {len(student_ids)}") return torch.tensor(student_ids, dtype=torch.long, device=device), torch.tensor(teacher_ids, dtype=torch.long, device=device) def _align_sequences(self, tea_seq, stu_seq): i, j = 0, 0 t2s_align, s2t_align = [], [] history_tea_seq, history_stu_seq = "", "" tea_eos = self.teacher_tokenizer.eos_token stu_eos = self.student_tokenizer.eos_token tea_seq = [token.replace('▁', '').replace('Ġ', '') for token in tea_seq] stu_seq = [token.replace('▁', '').replace('Ġ', '') for token in stu_seq] if tea_seq == stu_seq: indices = list(range(len(tea_seq))) return indices, indices while i < len(tea_seq) and j < len(stu_seq): is_eos_match = (tea_seq[i] == tea_eos and stu_seq[j] == stu_eos) if history_tea_seq == history_stu_seq and ( tea_seq[i] == stu_seq[j] or is_eos_match ): common_text = tea_seq[i] history_tea_seq += common_text history_stu_seq += common_text t2s_align.append(i) s2t_align.append(j) i += 1 j += 1 elif len(history_tea_seq) > len(history_stu_seq): history_stu_seq += stu_seq[j] j += 1 elif len(history_tea_seq) < len(history_stu_seq): history_tea_seq += tea_seq[i] i += 1 else: history_tea_seq += tea_seq[i] history_stu_seq += stu_seq[j] i += 1 j += 1 return t2s_align, s2t_align def _get_alignment_indices( self, teacher_label_ids, student_label_ids, teacher_loss_mask, student_loss_mask, ): """Return indices into the flattened teacher/student response tensors.""" teacher_tokens = self.teacher_tokenizer.convert_ids_to_tokens( teacher_label_ids[teacher_loss_mask].cpu().tolist() ) student_tokens = self.student_tokenizer.convert_ids_to_tokens( student_label_ids[student_loss_mask].cpu().tolist() ) return self._align_sequences(teacher_tokens, student_tokens) def training_step(self, micro_batch): student_input_ids = micro_batch["stu_input_ids"] student_attn_mask = micro_batch["stu_attn_mask"] student_loss_mask = micro_batch["stu_loss_mask"].bool() teacher_input_ids = micro_batch["tea_input_ids"] teacher_attn_mask = micro_batch["tea_attn_mask"] teacher_loss_mask = micro_batch["tea_loss_mask"].bool() teacher_hiddens = micro_batch.get("teacher_hiddens", None) avg_token_num = micro_batch["avg_micro_batch_token_num"] assert teacher_hiddens is not None, "micro_batch must contain `teacher_hiddens` for KD" student_shifted_label_ids = student_input_ids.roll(shifts=-1, dims=1) teacher_shifted_label_ids = teacher_input_ids.roll(shifts=-1, dims=1) teacher_aligned_idx, student_aligned_idx = self._get_alignment_indices( teacher_shifted_label_ids, student_shifted_label_ids, teacher_loss_mask, student_loss_mask, ) student_label_ids = student_shifted_label_ids[student_loss_mask] mm_kwargs = micro_batch.get("stu_multi_modal_inputs") or {} output = self.student( student_input_ids, attention_mask=student_attn_mask, allgather_logits=True, ring_attn_group=self.strategy.ring_attn_group, **mm_kwargs, ) student_hiddens = output["hidden_states"][-1][student_loss_mask] del output teacher_hiddens = teacher_hiddens.to(self.teacher_lm_head.weight) align_ratio = torch.tensor(len(student_aligned_idx) / len(student_label_ids)) aligned_student_hiddens = student_hiddens[student_aligned_idx] aligned_teacher_hiddens = teacher_hiddens[teacher_aligned_idx] # A length-truncated cross-tokenizer response may contain no 1:1 byte # span (and no appended EOS fallback). Keep the distributed optimizer # step well-defined instead of passing a zero chunk size downstream. if aligned_student_hiddens.shape[0] == 0: zero_loss = student_hiddens.sum() * 0.0 return { "train/loss": zero_loss, "train/kd_loss": zero_loss, "distill/alignment_ratio": align_ratio, } # Non-chunked case can be regarded as a special case of chunked loss (i.e., chunk_size = seq_len) chunk_size = self.args.train.chunked_loss_size or aligned_student_hiddens.shape[0] student_overlap_ids = self.student_overlap_token_ids teacher_overlap_ids = self.teacher_overlap_token_ids student_lm_head = self.student.model.lm_head teacher_lm_head = self.teacher_lm_head def student_logits_fn(hidden_chunk, skip=False): return student_lm_head(hidden_chunk, skip=skip)[:, student_overlap_ids] def teacher_logits_fn(start, end): return teacher_lm_head(aligned_teacher_hiddens[start:end])[:, teacher_overlap_ids] kd_loss, metric_sums = chunked_loss( aligned_student_hiddens, student_lm_head, self.loss_fn, student_logits_fn=student_logits_fn, teacher_logits_fn=teacher_logits_fn, chunk_size=chunk_size, reduction="sum", metric_fns=self.metric_fns, return_metrics=True, ) kd_loss = kd_loss / avg_token_num loss_info = { "train/loss": kd_loss, "train/kd_loss": kd_loss, "distill/alignment_ratio": align_ratio, } loss_info.update({key: value / avg_token_num for key, value in metric_sums.items()}) if self.args.kd.kd_ratio < 1: ce_loss = chunked_loss( student_hiddens, self.student.model.lm_head, compute_cross_entropy, label=student_label_ids, chunk_size=chunk_size, reduction="sum" ) / avg_token_num loss = (1 - self.args.kd.kd_ratio) * ce_loss + self.args.kd.kd_ratio * kd_loss loss_info["train/loss"] = loss loss_info["train/ce_loss"] = ce_loss return loss_info