File size: 4,552 Bytes
0ca529a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | import torch
from kdflow.algorithms import register_algorithm
from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD
from kdflow.token_alignment import (
align_byte_span_batch_1to1,
align_byte_spans_1to1,
find_byte_overlap_token_ids,
get_tokenizer_byte_codec,
)
from kdflow.utils.logging_utils import init_logger
logger = init_logger(__name__)
@register_algorithm("byte_align_ctkd")
class ByteAlignCrossTokenizerKD(SimpleCrossTokenizerKD):
"""Simple cross-tokenizer KD with UTF-8 byte-span 1:1 alignment."""
def __init__(
self,
strategy,
student_model,
teacher_lm_head,
student_tokenizer,
teacher_tokenizer,
**kwargs,
):
# These are initialized before ``super`` because its constructor calls
# the virtual ``_find_overlap_tokens`` method.
self.student_byte_codec = get_tokenizer_byte_codec(student_tokenizer)
self.teacher_byte_codec = get_tokenizer_byte_codec(teacher_tokenizer)
super().__init__(
strategy=strategy,
student_model=student_model,
teacher_lm_head=teacher_lm_head,
student_tokenizer=student_tokenizer,
teacher_tokenizer=teacher_tokenizer,
**kwargs,
)
def _find_overlap_tokens(self):
# Preserve the exact historical mapping for all existing ByteLevel
# pairs. Only mixed/SentencePiece pairs use the byte-canonical map.
if (
self.student_byte_codec.name == "byte_level"
and self.teacher_byte_codec.name == "byte_level"
):
return super()._find_overlap_tokens()
student_ids, teacher_ids = find_byte_overlap_token_ids(
student_tokenizer=self.student_tokenizer,
teacher_tokenizer=self.teacher_tokenizer,
student_codec=self.student_byte_codec,
teacher_codec=self.teacher_byte_codec,
)
device = self.teacher_lm_head.weight.device
logger.info(
"Num of byte-equivalent overlap tokens between student & teacher: "
f"{len(student_ids)}"
)
return (
torch.tensor(student_ids, dtype=torch.long, device=device),
torch.tensor(teacher_ids, dtype=torch.long, device=device),
)
def _align_sequences(self, tea_seq, stu_seq):
return align_byte_spans_1to1(
teacher_tokens=tea_seq,
student_tokens=stu_seq,
teacher_eos_token=self.teacher_tokenizer.eos_token,
student_eos_token=self.student_tokenizer.eos_token,
teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
)
def _get_alignment_indices(
self,
teacher_label_ids,
student_label_ids,
teacher_loss_mask,
student_loss_mask,
):
"""Align each micro-batch row independently, then return flat indices."""
if teacher_label_ids.shape[0] != student_label_ids.shape[0]:
raise ValueError(
"Teacher and student micro-batches must contain the same number "
"of samples for byte_align_ctkd."
)
teacher_batch_tokens = []
student_batch_tokens = []
for row_index in range(teacher_label_ids.shape[0]):
teacher_row_ids = teacher_label_ids[row_index][
teacher_loss_mask[row_index]
]
student_row_ids = student_label_ids[row_index][
student_loss_mask[row_index]
]
teacher_row_token_ids = teacher_row_ids.cpu().tolist()
student_row_token_ids = student_row_ids.cpu().tolist()
teacher_batch_tokens.append(
self.teacher_tokenizer.convert_ids_to_tokens(
teacher_row_token_ids
)
)
student_batch_tokens.append(
self.student_tokenizer.convert_ids_to_tokens(
student_row_token_ids
)
)
return align_byte_span_batch_1to1(
teacher_batch_tokens=teacher_batch_tokens,
student_batch_tokens=student_batch_tokens,
teacher_eos_token=self.teacher_tokenizer.eos_token,
student_eos_token=self.student_tokenizer.eos_token,
teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
)
|