Safetensors
English
RonanMcGovern's picture
Scripts under the duplex package name; card final pass
d90680b verified
Raw History Blame Contribute Delete
4.37 kB
"""Smoothed bigram and trigram floors for codebook 0.
A floor says how well codebook 0 can be predicted from the previous one or two codes
alone. The duplex-0 review found its model at trigram level; every evaluation now
reports codebook 0 next to these floors on the same held-out tokens.
Each order interpolates with the order below (Jelinek-Mercer): the unigram is
add-one smoothed, the bigram mixes its relative frequency with the unigram, the
trigram with the bigram, and an unseen history falls back to the order below. The
two weights are chosen on held-out training sequences with counts from the rest,
then the counts are rebuilt from every training sequence. A sequence is one track's
codebook 0 in one window, so history restarts at each window as the model's does.
"""
import numpy as np
V = 2048
GRID = (*np.round(np.arange(0.05, 1.0, 0.05), 2), 0.97, 0.99)
class _Counts:
def __init__(self, keys):
self.keys, self.counts = np.unique(keys, return_counts=True)
def __call__(self, keys):
index = np.searchsorted(self.keys, keys).clip(max=max(len(self.keys) - 1, 0))
found = len(self.keys) > 0
hit = (self.keys[index] == keys) if found else np.zeros(len(keys), bool)
return np.where(hit, self.counts[index] if found else 0, 0)
def _grams(sequences):
sequences = [np.asarray(s, dtype=np.int64) for s in sequences if len(s)]
pairs = [s[:-1] * V + s[1:] for s in sequences if len(s) > 1]
triples = [(s[:-2] * V + s[1:-1]) * V + s[2:] for s in sequences if len(s) > 2]
empty = np.zeros(0, dtype=np.int64)
return (
np.concatenate(sequences) if sequences else empty,
np.concatenate(pairs) if pairs else empty,
np.concatenate(triples) if triples else empty,
)
class Floors:
def __init__(self, train, holdout):
"""train and holdout are lists of codebook-0 sequences."""
self._fit(train)
self.bigram_weight = self.trigram_weight = 0.0
if any(len(s) > 1 for s in holdout):
self.bigram_weight = min(
GRID, key=lambda w: self._total(holdout, w, 0.0)[0]
)
self.trigram_weight = min(
GRID, key=lambda w: self._total(holdout, self.bigram_weight, w)[1]
)
self._fit(list(train) + list(holdout))
def _fit(self, sequences):
unigrams, pairs, triples = _grams(sequences)
self.unigram = np.bincount(unigrams, minlength=V).astype(np.float64)
self.total = len(unigrams)
self.pairs, self.triples = _Counts(pairs), _Counts(triples)
# History counts: a code or pair followed by another code.
self.pair_history = _Counts(pairs // V)
self.triple_history = _Counts(triples // V)
def _total(self, sequences, bigram_weight, trigram_weight):
bigram, trigram = self.nats(sequences, bigram_weight, trigram_weight)
return bigram.sum(), trigram.sum()
def nats(self, sequences, bigram_weight=None, trigram_weight=None):
"""Per-token negative log likelihoods of each sequence, concatenated in order,
under the bigram and the trigram floor."""
lb = self.bigram_weight if bigram_weight is None else bigram_weight
lt = self.trigram_weight if trigram_weight is None else trigram_weight
bigrams, trigrams = [], []
for s in sequences:
s = np.asarray(s, dtype=np.int64)
uni = (self.unigram[s] + 1) / (self.total + V)
bi = uni.copy()
if len(s) > 1:
seen = self.pair_history(s[:-1])
count = self.pairs(s[:-1] * V + s[1:])
mixed = lb * count / np.maximum(seen, 1) + (1 - lb) * uni[1:]
bi[1:] = np.where(seen > 0, mixed, uni[1:])
tri = bi.copy()
if len(s) > 2:
history = s[:-2] * V + s[1:-1]
seen = self.triple_history(history)
count = self.triples(history * V + s[2:])
mixed = lt * count / np.maximum(seen, 1) + (1 - lt) * bi[2:]
tri[2:] = np.where(seen > 0, mixed, bi[2:])
bigrams.append(-np.log(bi))
trigrams.append(-np.log(tri))
empty = np.zeros(0)
return (
np.concatenate(bigrams) if bigrams else empty,
np.concatenate(trigrams) if trigrams else empty,
)