Download scripts/duplex/evaluation/ngram.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 4.37 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/evaluation/ngram.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/evaluation/ngram.py
-
curl -L -o ngram.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/evaluation/ngram.py
4.37 kB
| """Smoothed bigram and trigram floors for codebook 0. | |
| A floor says how well codebook 0 can be predicted from the previous one or two codes | |
| alone. The duplex-0 review found its model at trigram level; every evaluation now | |
| reports codebook 0 next to these floors on the same held-out tokens. | |
| Each order interpolates with the order below (Jelinek-Mercer): the unigram is | |
| add-one smoothed, the bigram mixes its relative frequency with the unigram, the | |
| trigram with the bigram, and an unseen history falls back to the order below. The | |
| two weights are chosen on held-out training sequences with counts from the rest, | |
| then the counts are rebuilt from every training sequence. A sequence is one track's | |
| codebook 0 in one window, so history restarts at each window as the model's does. | |
| """ | |
| import numpy as np | |
| V = 2048 | |
| GRID = (*np.round(np.arange(0.05, 1.0, 0.05), 2), 0.97, 0.99) | |
| class _Counts: | |
| def __init__(self, keys): | |
| self.keys, self.counts = np.unique(keys, return_counts=True) | |
| def __call__(self, keys): | |
| index = np.searchsorted(self.keys, keys).clip(max=max(len(self.keys) - 1, 0)) | |
| found = len(self.keys) > 0 | |
| hit = (self.keys[index] == keys) if found else np.zeros(len(keys), bool) | |
| return np.where(hit, self.counts[index] if found else 0, 0) | |
| def _grams(sequences): | |
| sequences = [np.asarray(s, dtype=np.int64) for s in sequences if len(s)] | |
| pairs = [s[:-1] * V + s[1:] for s in sequences if len(s) > 1] | |
| triples = [(s[:-2] * V + s[1:-1]) * V + s[2:] for s in sequences if len(s) > 2] | |
| empty = np.zeros(0, dtype=np.int64) | |
| return ( | |
| np.concatenate(sequences) if sequences else empty, | |
| np.concatenate(pairs) if pairs else empty, | |
| np.concatenate(triples) if triples else empty, | |
| ) | |
| class Floors: | |
| def __init__(self, train, holdout): | |
| """train and holdout are lists of codebook-0 sequences.""" | |
| self._fit(train) | |
| self.bigram_weight = self.trigram_weight = 0.0 | |
| if any(len(s) > 1 for s in holdout): | |
| self.bigram_weight = min( | |
| GRID, key=lambda w: self._total(holdout, w, 0.0)[0] | |
| ) | |
| self.trigram_weight = min( | |
| GRID, key=lambda w: self._total(holdout, self.bigram_weight, w)[1] | |
| ) | |
| self._fit(list(train) + list(holdout)) | |
| def _fit(self, sequences): | |
| unigrams, pairs, triples = _grams(sequences) | |
| self.unigram = np.bincount(unigrams, minlength=V).astype(np.float64) | |
| self.total = len(unigrams) | |
| self.pairs, self.triples = _Counts(pairs), _Counts(triples) | |
| # History counts: a code or pair followed by another code. | |
| self.pair_history = _Counts(pairs // V) | |
| self.triple_history = _Counts(triples // V) | |
| def _total(self, sequences, bigram_weight, trigram_weight): | |
| bigram, trigram = self.nats(sequences, bigram_weight, trigram_weight) | |
| return bigram.sum(), trigram.sum() | |
| def nats(self, sequences, bigram_weight=None, trigram_weight=None): | |
| """Per-token negative log likelihoods of each sequence, concatenated in order, | |
| under the bigram and the trigram floor.""" | |
| lb = self.bigram_weight if bigram_weight is None else bigram_weight | |
| lt = self.trigram_weight if trigram_weight is None else trigram_weight | |
| bigrams, trigrams = [], [] | |
| for s in sequences: | |
| s = np.asarray(s, dtype=np.int64) | |
| uni = (self.unigram[s] + 1) / (self.total + V) | |
| bi = uni.copy() | |
| if len(s) > 1: | |
| seen = self.pair_history(s[:-1]) | |
| count = self.pairs(s[:-1] * V + s[1:]) | |
| mixed = lb * count / np.maximum(seen, 1) + (1 - lb) * uni[1:] | |
| bi[1:] = np.where(seen > 0, mixed, uni[1:]) | |
| tri = bi.copy() | |
| if len(s) > 2: | |
| history = s[:-2] * V + s[1:-1] | |
| seen = self.triple_history(history) | |
| count = self.triples(history * V + s[2:]) | |
| mixed = lt * count / np.maximum(seen, 1) + (1 - lt) * bi[2:] | |
| tri[2:] = np.where(seen > 0, mixed, bi[2:]) | |
| bigrams.append(-np.log(bi)) | |
| trigrams.append(-np.log(tri)) | |
| empty = np.zeros(0) | |
| return ( | |
| np.concatenate(bigrams) if bigrams else empty, | |
| np.concatenate(trigrams) if trigrams else empty, | |
| ) | |