File size: 1,973 Bytes
3711808
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""InDepth tokenizer — word-level vocab + hashed char-trigram subword features.

Rules (mirrored bit-exact in the browser engine embedded in InDepth.html):
- words: lowercase, split on [^a-z0-9]+, drop empties, max 16 tokens
- unknown words -> <unk> id 1 (trigram features still apply)
- char trigrams over "<word>", capped at 16 per word
- bucket id = 1 + (FNV-1a 32-bit of the trigram mod (buckets-1)); 0 = null/pad
"""
import json
import re
from pathlib import Path

SUBWORD_SLOTS = 16
UNK = 1


def fnv1a(s):
    h = 2166136261
    for ch in s:
        h ^= ord(ch)
        h = (h * 16777619) & 0xFFFFFFFF
    return h


def word_trigram_buckets(word, buckets):
    tagged = "<" + word + ">"
    grams = [tagged[i:i + 3] for i in range(len(tagged) - 2)][:SUBWORD_SLOTS]
    ids = [0] * SUBWORD_SLOTS
    for i, g in enumerate(grams):
        ids[i] = 1 + (fnv1a(g) % (buckets - 1))
    return ids


def encode_prompt(prompt, stoi, buckets, max_len):
    """-> (tokens, subwords) flat lists of length max_len and max_len * SUBWORD_SLOTS."""
    words = [t for t in re.split(r"[^a-z0-9]+", prompt.lower()) if t][:max_len]
    tokens = [stoi.get(w, UNK) for w in words]
    tri = [[0] * SUBWORD_SLOTS for _ in range(max_len)]
    for i, w in enumerate(words):
        tri[i] = word_trigram_buckets(w, buckets)
    return tokens, tri


class IndepthTokenizer:
    def __init__(self, model_dir):
        vocab = json.loads((Path(model_dir) / "vocab.json").read_text())
        self.stoi = {w: i for i, w in enumerate(vocab["vocab"])}
        self.max_len = vocab["max_len"]
        self.buckets = vocab.get("subword_buckets", 4096)

    def batch(self, prompts):
        rows, subs = [], []
        for prompt in prompts:
            tokens, tri = encode_prompt(prompt, self.stoi, self.buckets, self.max_len)
            rows.append(tokens + [0] * (self.max_len - len(tokens)))
            subs.append(tri)  # (max_len, SUBWORD_SLOTS) per prompt
        return rows, subs