File size: 2,634 Bytes
c394b65
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
"""Compact English-only unigram tokenizer ("ETK2").

Input is already `normalize`d ([a-z0-9] words separated by single spaces), so no
byte fallback or unicode handling is needed.  Word boundaries are marked with
`_` (SentencePiece's `▁`).

File layout:  b"ETK2" | u16 n_pieces | n x ( u8 len | ascii bytes | i8 score*8 )
Piece 0 is <unk>.
"""

from __future__ import annotations

import struct

MAGIC = b"ETK2"
SCORE_SCALE = 8.0
UNK_SCORE = -30.0


class TinyTokenizer:
    def __init__(self, data: bytes):
        if data[:4] != MAGIC:
            raise ValueError("bad tokenizer file")
        (n,) = struct.unpack_from("<H", data, 4)
        off = 6
        self.pieces: list[str] = []
        self.scores: list[float] = []
        for _ in range(n):
            ln = data[off]
            off += 1
            self.pieces.append(data[off : off + ln].decode("ascii"))
            off += ln
            self.scores.append(struct.unpack_from("<b", data, off)[0] / SCORE_SCALE)
            off += 1
        self.index = {p: i for i, p in enumerate(self.pieces) if i > 0}
        self.unk_id = 0
        self.max_len = max(len(p) for p in self.pieces)

    @classmethod
    def load(cls, path: str) -> "TinyTokenizer":
        with open(path, "rb") as f:
            return cls(f.read())

    @staticmethod
    def dump(pieces: list[str], scores: list[float]) -> bytes:
        out = bytearray(MAGIC + struct.pack("<H", len(pieces)))
        for p, s in zip(pieces, scores):
            b = p.encode("ascii")
            q = max(-128, min(127, round(s * SCORE_SCALE)))
            out += struct.pack("<B", len(b)) + b + struct.pack("<b", q)
        return bytes(out)

    def encode(self, norm_text: str, max_tokens: int | None = None) -> list[int]:
        s = "_" + norm_text.replace(" ", "_")
        n = len(s)
        if n <= 1:
            return []
        best = [-1e18] * (n + 1)
        best[0] = 0.0
        bp = [0] * (n + 1)
        bid = [self.unk_id] * (n + 1)
        for i in range(1, n + 1):
            for j in range(max(0, i - self.max_len), i):
                tid = self.index.get(s[j:i])
                if tid is not None:
                    sc = best[j] + self.scores[tid]
                    if sc > best[i]:
                        best[i], bp[i], bid[i] = sc, j, tid
            sc = best[i - 1] + UNK_SCORE
            if sc > best[i]:
                best[i], bp[i], bid[i] = sc, i - 1, self.unk_id
        ids: list[int] = []
        i = n
        while i > 0:
            ids.append(bid[i])
            i = bp[i]
        ids.reverse()
        return ids if max_tokens is None else ids[-max_tokens:]