File size: 892 Bytes
619cff6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
"""Paper Eq.7-8 scores on the released SimCT candidate/alignment interface."""
import types

import torch

from . import register_algorithm
from .simct import SimCT


def paper_virtual_vocab(upstream, segments, student_logits, teacher_logits, student_ids, teacher_ids):
    # Normalize over the FULL native vocabulary at EACH autoregressive position.
    # The original builder then selects shared entries and averages span scores.
    return type(upstream)._build_virtual_vocab_logits(upstream, segments,
        torch.log_softmax(student_logits.float(), -1),
        torch.log_softmax(teacher_logits.float(), -1), student_ids, teacher_ids)


@register_algorithm('simct_paper')
class SimCTPaper(SimCT):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.upstream._build_virtual_vocab_logits = types.MethodType(paper_virtual_vocab, self.upstream)