File size: 2,923 Bytes
2bff5be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """A causal output adapter that mixes Franok's vocabulary and context-copy distributions."""
import math
import torch
from torch import nn
from torch.nn import functional as F
class PointerHead(nn.Module):
def __init__(self, width=512, rank=32):
super().__init__()
self.q = nn.Linear(width, rank, bias=False)
self.k = nn.Linear(width, rank, bias=False)
self.prev = nn.Linear(width, rank, bias=False)
self.gate = nn.Linear(width, 1)
nn.init.zeros_(self.gate.weight)
nn.init.constant_(self.gate.bias, -2.8)
self.suffix = nn.Parameter(torch.zeros(3))
self.user = nn.Parameter(torch.zeros(()))
def forward(self, hidden, tokens, query_positions):
"""Return source-position probabilities and the logit of the copy gate.
hidden[b, t] is the frozen decoder's final normalized state after reading
tokens[b, t]. A prediction at t may copy a token from positions 0 through t.
"""
batch, length, _ = hidden.shape
batch_indices = torch.arange(batch, device=tokens.device)[:, None]
positions = torch.arange(length, device=tokens.device)[None, :]
keys = self.k(hidden) + F.pad(self.prev(hidden[:, :-1]), (0, 0, 1, 0))
selected = hidden[batch_indices, query_positions]
queries = self.q(selected)
scores = queries @ keys.transpose(1, 2) / math.sqrt(keys.shape[-1])
boundaries = torch.where((tokens == 3) | (tokens == 4), positions, 0)
last_role = tokens.gather(1, boundaries.cummax(-1).values)
scores = scores + self.user * (last_role == 3)[:, None, :]
# A matching suffix makes its continuation a candidate at the next position.
target = query_positions[:, :, None]
source = positions[:, None, :]
matched = torch.ones_like(scores, dtype=torch.bool)
for n, coefficient in enumerate(self.suffix, 1):
last = tokens[batch_indices, (query_positions - n + 1).clamp_min(0)]
prior = F.pad(tokens, (n, 0))[:, :length]
matched = (
matched
& (last[:, :, None] == prior[:, None, :])
& (source >= n)
& (target >= n - 1)
)
scores = scores + coefficient * matched
valid = (source <= target) & (tokens[:, None, :] >= 7)
weights = scores.masked_fill(~valid, -1e9).softmax(-1) * valid
gate = self.gate(selected).squeeze(-1).masked_fill(~valid.any(-1), -torch.inf)
return weights, gate
def target_log_probs(base_log_probs, weights, gate, tokens, targets):
"""Log probability of each known next token under the normalized mixture."""
copy_mass = (weights * (tokens[:, None, :] == targets[:, :, None])).sum(-1)
return torch.logaddexp(
F.logsigmoid(-gate) + base_log_probs,
F.logsigmoid(gate) + copy_mass.clamp_min(1e-20).log(),
)
|