File size: 4,552 Bytes
0ca529a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import torch

from kdflow.algorithms import register_algorithm
from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD
from kdflow.token_alignment import (
    align_byte_span_batch_1to1,
    align_byte_spans_1to1,
    find_byte_overlap_token_ids,
    get_tokenizer_byte_codec,
)
from kdflow.utils.logging_utils import init_logger


logger = init_logger(__name__)


@register_algorithm("byte_align_ctkd")
class ByteAlignCrossTokenizerKD(SimpleCrossTokenizerKD):
    """Simple cross-tokenizer KD with UTF-8 byte-span 1:1 alignment."""

    def __init__(
        self,
        strategy,
        student_model,
        teacher_lm_head,
        student_tokenizer,
        teacher_tokenizer,
        **kwargs,
    ):
        # These are initialized before ``super`` because its constructor calls
        # the virtual ``_find_overlap_tokens`` method.
        self.student_byte_codec = get_tokenizer_byte_codec(student_tokenizer)
        self.teacher_byte_codec = get_tokenizer_byte_codec(teacher_tokenizer)
        super().__init__(
            strategy=strategy,
            student_model=student_model,
            teacher_lm_head=teacher_lm_head,
            student_tokenizer=student_tokenizer,
            teacher_tokenizer=teacher_tokenizer,
            **kwargs,
        )

    def _find_overlap_tokens(self):
        # Preserve the exact historical mapping for all existing ByteLevel
        # pairs. Only mixed/SentencePiece pairs use the byte-canonical map.
        if (
            self.student_byte_codec.name == "byte_level"
            and self.teacher_byte_codec.name == "byte_level"
        ):
            return super()._find_overlap_tokens()

        student_ids, teacher_ids = find_byte_overlap_token_ids(
            student_tokenizer=self.student_tokenizer,
            teacher_tokenizer=self.teacher_tokenizer,
            student_codec=self.student_byte_codec,
            teacher_codec=self.teacher_byte_codec,
        )
        device = self.teacher_lm_head.weight.device
        logger.info(
            "Num of byte-equivalent overlap tokens between student & teacher: "
            f"{len(student_ids)}"
        )
        return (
            torch.tensor(student_ids, dtype=torch.long, device=device),
            torch.tensor(teacher_ids, dtype=torch.long, device=device),
        )

    def _align_sequences(self, tea_seq, stu_seq):
        return align_byte_spans_1to1(
            teacher_tokens=tea_seq,
            student_tokens=stu_seq,
            teacher_eos_token=self.teacher_tokenizer.eos_token,
            student_eos_token=self.student_tokenizer.eos_token,
            teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
            student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
        )

    def _get_alignment_indices(
        self,
        teacher_label_ids,
        student_label_ids,
        teacher_loss_mask,
        student_loss_mask,
    ):
        """Align each micro-batch row independently, then return flat indices."""
        if teacher_label_ids.shape[0] != student_label_ids.shape[0]:
            raise ValueError(
                "Teacher and student micro-batches must contain the same number "
                "of samples for byte_align_ctkd."
            )

        teacher_batch_tokens = []
        student_batch_tokens = []
        for row_index in range(teacher_label_ids.shape[0]):
            teacher_row_ids = teacher_label_ids[row_index][
                teacher_loss_mask[row_index]
            ]
            student_row_ids = student_label_ids[row_index][
                student_loss_mask[row_index]
            ]
            teacher_row_token_ids = teacher_row_ids.cpu().tolist()
            student_row_token_ids = student_row_ids.cpu().tolist()
            teacher_batch_tokens.append(
                self.teacher_tokenizer.convert_ids_to_tokens(
                    teacher_row_token_ids
                )
            )
            student_batch_tokens.append(
                self.student_tokenizer.convert_ids_to_tokens(
                    student_row_token_ids
                )
            )

        return align_byte_span_batch_1to1(
            teacher_batch_tokens=teacher_batch_tokens,
            student_batch_tokens=student_batch_tokens,
            teacher_eos_token=self.teacher_tokenizer.eos_token,
            student_eos_token=self.student_tokenizer.eos_token,
            teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
            student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
        )