"""Smoothed bigram and trigram floors for codebook 0. A floor says how well codebook 0 can be predicted from the previous one or two codes alone. The duplex-0 review found its model at trigram level; every evaluation now reports codebook 0 next to these floors on the same held-out tokens. Each order interpolates with the order below (Jelinek-Mercer): the unigram is add-one smoothed, the bigram mixes its relative frequency with the unigram, the trigram with the bigram, and an unseen history falls back to the order below. The two weights are chosen on held-out training sequences with counts from the rest, then the counts are rebuilt from every training sequence. A sequence is one track's codebook 0 in one window, so history restarts at each window as the model's does. """ import numpy as np V = 2048 GRID = (*np.round(np.arange(0.05, 1.0, 0.05), 2), 0.97, 0.99) class _Counts: def __init__(self, keys): self.keys, self.counts = np.unique(keys, return_counts=True) def __call__(self, keys): index = np.searchsorted(self.keys, keys).clip(max=max(len(self.keys) - 1, 0)) found = len(self.keys) > 0 hit = (self.keys[index] == keys) if found else np.zeros(len(keys), bool) return np.where(hit, self.counts[index] if found else 0, 0) def _grams(sequences): sequences = [np.asarray(s, dtype=np.int64) for s in sequences if len(s)] pairs = [s[:-1] * V + s[1:] for s in sequences if len(s) > 1] triples = [(s[:-2] * V + s[1:-1]) * V + s[2:] for s in sequences if len(s) > 2] empty = np.zeros(0, dtype=np.int64) return ( np.concatenate(sequences) if sequences else empty, np.concatenate(pairs) if pairs else empty, np.concatenate(triples) if triples else empty, ) class Floors: def __init__(self, train, holdout): """train and holdout are lists of codebook-0 sequences.""" self._fit(train) self.bigram_weight = self.trigram_weight = 0.0 if any(len(s) > 1 for s in holdout): self.bigram_weight = min( GRID, key=lambda w: self._total(holdout, w, 0.0)[0] ) self.trigram_weight = min( GRID, key=lambda w: self._total(holdout, self.bigram_weight, w)[1] ) self._fit(list(train) + list(holdout)) def _fit(self, sequences): unigrams, pairs, triples = _grams(sequences) self.unigram = np.bincount(unigrams, minlength=V).astype(np.float64) self.total = len(unigrams) self.pairs, self.triples = _Counts(pairs), _Counts(triples) # History counts: a code or pair followed by another code. self.pair_history = _Counts(pairs // V) self.triple_history = _Counts(triples // V) def _total(self, sequences, bigram_weight, trigram_weight): bigram, trigram = self.nats(sequences, bigram_weight, trigram_weight) return bigram.sum(), trigram.sum() def nats(self, sequences, bigram_weight=None, trigram_weight=None): """Per-token negative log likelihoods of each sequence, concatenated in order, under the bigram and the trigram floor.""" lb = self.bigram_weight if bigram_weight is None else bigram_weight lt = self.trigram_weight if trigram_weight is None else trigram_weight bigrams, trigrams = [], [] for s in sequences: s = np.asarray(s, dtype=np.int64) uni = (self.unigram[s] + 1) / (self.total + V) bi = uni.copy() if len(s) > 1: seen = self.pair_history(s[:-1]) count = self.pairs(s[:-1] * V + s[1:]) mixed = lb * count / np.maximum(seen, 1) + (1 - lb) * uni[1:] bi[1:] = np.where(seen > 0, mixed, uni[1:]) tri = bi.copy() if len(s) > 2: history = s[:-2] * V + s[1:-1] seen = self.triple_history(history) count = self.triples(history * V + s[2:]) mixed = lt * count / np.maximum(seen, 1) + (1 - lt) * bi[2:] tri[2:] = np.where(seen > 0, mixed, bi[2:]) bigrams.append(-np.log(bi)) trigrams.append(-np.log(tri)) empty = np.zeros(0) return ( np.concatenate(bigrams) if bigrams else empty, np.concatenate(trigrams) if trigrams else empty, )