File size: 2,634 Bytes
c394b65 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """Compact English-only unigram tokenizer ("ETK2").
Input is already `normalize`d ([a-z0-9] words separated by single spaces), so no
byte fallback or unicode handling is needed. Word boundaries are marked with
`_` (SentencePiece's `▁`).
File layout: b"ETK2" | u16 n_pieces | n x ( u8 len | ascii bytes | i8 score*8 )
Piece 0 is <unk>.
"""
from __future__ import annotations
import struct
MAGIC = b"ETK2"
SCORE_SCALE = 8.0
UNK_SCORE = -30.0
class TinyTokenizer:
def __init__(self, data: bytes):
if data[:4] != MAGIC:
raise ValueError("bad tokenizer file")
(n,) = struct.unpack_from("<H", data, 4)
off = 6
self.pieces: list[str] = []
self.scores: list[float] = []
for _ in range(n):
ln = data[off]
off += 1
self.pieces.append(data[off : off + ln].decode("ascii"))
off += ln
self.scores.append(struct.unpack_from("<b", data, off)[0] / SCORE_SCALE)
off += 1
self.index = {p: i for i, p in enumerate(self.pieces) if i > 0}
self.unk_id = 0
self.max_len = max(len(p) for p in self.pieces)
@classmethod
def load(cls, path: str) -> "TinyTokenizer":
with open(path, "rb") as f:
return cls(f.read())
@staticmethod
def dump(pieces: list[str], scores: list[float]) -> bytes:
out = bytearray(MAGIC + struct.pack("<H", len(pieces)))
for p, s in zip(pieces, scores):
b = p.encode("ascii")
q = max(-128, min(127, round(s * SCORE_SCALE)))
out += struct.pack("<B", len(b)) + b + struct.pack("<b", q)
return bytes(out)
def encode(self, norm_text: str, max_tokens: int | None = None) -> list[int]:
s = "_" + norm_text.replace(" ", "_")
n = len(s)
if n <= 1:
return []
best = [-1e18] * (n + 1)
best[0] = 0.0
bp = [0] * (n + 1)
bid = [self.unk_id] * (n + 1)
for i in range(1, n + 1):
for j in range(max(0, i - self.max_len), i):
tid = self.index.get(s[j:i])
if tid is not None:
sc = best[j] + self.scores[tid]
if sc > best[i]:
best[i], bp[i], bid[i] = sc, j, tid
sc = best[i - 1] + UNK_SCORE
if sc > best[i]:
best[i], bp[i], bid[i] = sc, i - 1, self.unk_id
ids: list[int] = []
i = n
while i > 0:
ids.append(bid[i])
i = bp[i]
ids.reverse()
return ids if max_tokens is None else ids[-max_tokens:]
|