File size: 2,923 Bytes
2bff5be
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""A causal output adapter that mixes Franok's vocabulary and context-copy distributions."""

import math
import torch
from torch import nn
from torch.nn import functional as F


class PointerHead(nn.Module):
    def __init__(self, width=512, rank=32):
        super().__init__()
        self.q = nn.Linear(width, rank, bias=False)
        self.k = nn.Linear(width, rank, bias=False)
        self.prev = nn.Linear(width, rank, bias=False)
        self.gate = nn.Linear(width, 1)
        nn.init.zeros_(self.gate.weight)
        nn.init.constant_(self.gate.bias, -2.8)
        self.suffix = nn.Parameter(torch.zeros(3))
        self.user = nn.Parameter(torch.zeros(()))

    def forward(self, hidden, tokens, query_positions):
        """Return source-position probabilities and the logit of the copy gate.

        hidden[b, t] is the frozen decoder's final normalized state after reading
        tokens[b, t]. A prediction at t may copy a token from positions 0 through t.
        """
        batch, length, _ = hidden.shape
        batch_indices = torch.arange(batch, device=tokens.device)[:, None]
        positions = torch.arange(length, device=tokens.device)[None, :]
        keys = self.k(hidden) + F.pad(self.prev(hidden[:, :-1]), (0, 0, 1, 0))
        selected = hidden[batch_indices, query_positions]
        queries = self.q(selected)
        scores = queries @ keys.transpose(1, 2) / math.sqrt(keys.shape[-1])

        boundaries = torch.where((tokens == 3) | (tokens == 4), positions, 0)
        last_role = tokens.gather(1, boundaries.cummax(-1).values)
        scores = scores + self.user * (last_role == 3)[:, None, :]

        # A matching suffix makes its continuation a candidate at the next position.
        target = query_positions[:, :, None]
        source = positions[:, None, :]
        matched = torch.ones_like(scores, dtype=torch.bool)
        for n, coefficient in enumerate(self.suffix, 1):
            last = tokens[batch_indices, (query_positions - n + 1).clamp_min(0)]
            prior = F.pad(tokens, (n, 0))[:, :length]
            matched = (
                matched
                & (last[:, :, None] == prior[:, None, :])
                & (source >= n)
                & (target >= n - 1)
            )
            scores = scores + coefficient * matched

        valid = (source <= target) & (tokens[:, None, :] >= 7)
        weights = scores.masked_fill(~valid, -1e9).softmax(-1) * valid
        gate = self.gate(selected).squeeze(-1).masked_fill(~valid.any(-1), -torch.inf)
        return weights, gate


def target_log_probs(base_log_probs, weights, gate, tokens, targets):
    """Log probability of each known next token under the normalized mixture."""
    copy_mass = (weights * (tokens[:, None, :] == targets[:, :, None])).sum(-1)
    return torch.logaddexp(
        F.logsigmoid(-gate) + base_log_probs,
        F.logsigmoid(gate) + copy_mass.clamp_min(1e-20).log(),
    )