File size: 3,448 Bytes
0ca529a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
from typing import Tuple

import torch

from kdflow.algorithms import register_algorithm
from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD
from kdflow.metrics.entropy import compute_entropy


def select_student_topk_logits(
    student_logits: torch.Tensor,
    teacher_logits: torch.Tensor,
    topk: int,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    """Select the student's top-k shared-vocabulary entries on both sides.

    The returned tensors contain matching vocabulary entries. Downstream KD
    losses apply softmax/log-softmax to these reduced logits, which
    re-normalizes both distributions over exactly this top-k support.
    """
    if student_logits.shape != teacher_logits.shape:
        raise ValueError(
            "Student and teacher shared-vocabulary logits must have the same "
            f"shape, got {student_logits.shape} and {teacher_logits.shape}."
        )
    if topk <= 0 or topk > student_logits.shape[-1]:
        raise ValueError(
            f"topk must be in [1, {student_logits.shape[-1]}], got {topk}."
        )

    student_topk_logits, topk_indices = torch.topk(
        student_logits,
        k=topk,
        dim=-1,
    )
    teacher_topk_logits = teacher_logits.gather(-1, topk_indices)
    return student_topk_logits, teacher_topk_logits, topk_indices


@register_algorithm("topk_byte_align_ctkd")
class TopKByteAlignCrossTokenizerKD(ByteAlignCrossTokenizerKD):
    """Byte-aligned CTKD restricted to each student's shared-vocab top-k."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.ctkd_topk = self.args.kd.ctkd_topk
        overlap_vocab_size = self.student_overlap_token_ids.numel()
        if self.ctkd_topk < 2:
            raise ValueError(
                "ctkd_topk must be at least 2 because re-normalizing a "
                "single logit produces a degenerate distribution."
            )
        if self.ctkd_topk > overlap_vocab_size:
            raise ValueError(
                f"ctkd_topk ({self.ctkd_topk}) exceeds the shared vocabulary "
                f"size ({overlap_vocab_size})."
            )
        if (
            self.args.kd.kd_loss_fn == "hrl"
            and self.args.kd.hrl_topk > self.ctkd_topk
        ):
            raise ValueError(
                f"hrl_topk ({self.args.kd.hrl_topk}) cannot exceed ctkd_topk "
                f"({self.ctkd_topk})."
            )

        self._shared_vocab_loss_fn = self.loss_fn
        self.loss_fn = self._topk_loss
        if self.metric_fns:
            self.metric_fns = [self._topk_entropy]

    def _topk_loss(
        self,
        student_logits: torch.Tensor,
        teacher_logits: torch.Tensor,
        **kwargs,
    ) -> torch.Tensor:
        student_topk_logits, teacher_topk_logits, _ = select_student_topk_logits(
            student_logits,
            teacher_logits,
            self.ctkd_topk,
        )
        return self._shared_vocab_loss_fn(
            student_topk_logits,
            teacher_topk_logits,
            **kwargs,
        )

    def _topk_entropy(
        self,
        student_logits: torch.Tensor,
        teacher_logits: torch.Tensor,
        **kwargs,
    ):
        student_topk_logits, _, _ = select_student_topk_logits(
            student_logits,
            teacher_logits,
            self.ctkd_topk,
        )
        return compute_entropy(student_logits=student_topk_logits)