File size: 5,487 Bytes
f8a85a5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
"""
Test the tokenizer: BPE round-trips, special tokens, and conversation rendering.
Trains a tiny throwaway tokenizer in-process so the test is hermetic
(no dependency on ~/.cache/nanochat).

python -m pytest tests/test_tokenizer.py -v
"""

import pytest
from nanochat.tokenizer import RustBPETokenizer, SPECIAL_TOKENS

# a small corpus is enough to exercise the BPE machinery
CORPUS = [
    "The quick brown fox jumps over the lazy dog.",
    "hello world, hello tokenizer, hello hello hello",
    "Numbers like 12345 and unicode like naïve café 你好 🙂 should survive.",
    "def f(x):\n    return x + 1\n",
] * 8


@pytest.fixture(scope="module")
def tokenizer():
    vocab_size = 256 + len(SPECIAL_TOKENS) + 35 # bytes + specials + a few merges
    return RustBPETokenizer.train_from_iterator(iter(CORPUS), vocab_size)


def test_vocab_size(tokenizer):
    assert tokenizer.get_vocab_size() == 256 + len(SPECIAL_TOKENS) + 35


def test_encode_decode_roundtrip(tokenizer):
    for text in ["hello world", "naïve café 你好 🙂", "unseen tokens: zqxjkv"]:
        ids = tokenizer.encode(text)
        assert tokenizer.decode(ids) == text


def test_special_tokens(tokenizer):
    # all special tokens encode to a unique single id
    ids = [tokenizer.encode_special(t) for t in SPECIAL_TOKENS]
    assert len(set(ids)) == len(SPECIAL_TOKENS)
    assert tokenizer.get_bos_token_id() == tokenizer.encode_special("<|bos|>")
    # specials are NOT special-cased in ordinary text encoding
    ids = tokenizer.encode("<|bos|>")
    assert len(ids) > 1, "special token strings in plain text should not collapse to one token"


def test_encode_prepend_append(tokenizer):
    bos = tokenizer.get_bos_token_id()
    ids = tokenizer.encode("hello", prepend="<|bos|>", append="<|user_end|>")
    assert ids[0] == bos
    assert ids[-1] == tokenizer.encode_special("<|user_end|>")


def test_encode_batch(tokenizer):
    texts = ["hello", "world"]
    ids = tokenizer.encode(texts)
    assert isinstance(ids, list) and len(ids) == 2
    assert ids[0] == tokenizer.encode("hello")


def test_render_conversation_masks(tokenizer):
    conversation = {"messages": [
        {"role": "user", "content": "hi"},
        {"role": "assistant", "content": "hello!"},
        {"role": "user", "content": "bye"},
        {"role": "assistant", "content": "later"},
    ]}
    ids, mask = tokenizer.render_conversation(conversation)
    assert len(ids) == len(mask)
    # first token is bos and is not supervised
    assert ids[0] == tokenizer.get_bos_token_id() and mask[0] == 0
    # supervised tokens are exactly: assistant content + assistant_end tokens
    assistant_end = tokenizer.encode_special("<|assistant_end|>")
    supervised_ids = [i for i, m in zip(ids, mask) if m == 1]
    expected = tokenizer.encode("hello!") + [assistant_end] + tokenizer.encode("later") + [assistant_end]
    assert supervised_ids == expected
    # user content tokens are never supervised
    user_start = tokenizer.encode_special("<|user_start|>")
    assert all(m == 0 for i, m in zip(ids, mask) if i == user_start)


def test_render_conversation_system_message_merged(tokenizer):
    without_system = {"messages": [
        {"role": "user", "content": "sys prompt\n\nhi"},
        {"role": "assistant", "content": "yo"},
    ]}
    with_system = {"messages": [
        {"role": "system", "content": "sys prompt"},
        {"role": "user", "content": "hi"},
        {"role": "assistant", "content": "yo"},
    ]}
    assert tokenizer.render_conversation(with_system) == tokenizer.render_conversation(without_system)


def test_render_conversation_tool_parts(tokenizer):
    # python tool calls are supervised, python outputs (come from the interpreter) are not
    conversation = {"messages": [
        {"role": "user", "content": "add"},
        {"role": "assistant", "content": [
            {"type": "text", "text": "sure"},
            {"type": "python", "text": "1+1"},
            {"type": "python_output", "text": "2"},
            {"type": "text", "text": "it is 2"},
        ]},
    ]}
    ids, mask = tokenizer.render_conversation(conversation)
    python_start = tokenizer.encode_special("<|python_start|>")
    output_start = tokenizer.encode_special("<|output_start|>")
    output_end = tokenizer.encode_special("<|output_end|>")
    # the tool call and its delimiters are supervised
    assert mask[ids.index(python_start)] == 1
    # the interpreter output and its delimiters are not
    start, end = ids.index(output_start), ids.index(output_end)
    assert all(m == 0 for m in mask[start:end + 1])


def test_render_conversation_truncation(tokenizer):
    conversation = {"messages": [
        {"role": "user", "content": "hello " * 100},
        {"role": "assistant", "content": "world " * 100},
    ]}
    ids, mask = tokenizer.render_conversation(conversation, max_tokens=32)
    assert len(ids) == 32 and len(mask) == 32


def test_render_for_completion(tokenizer):
    conversation = {"messages": [
        {"role": "user", "content": "hi"},
        {"role": "assistant", "content": "this gets stripped"},
    ]}
    ids = tokenizer.render_for_completion(conversation)
    # ends with assistant_start, primed for a completion
    assert ids[-1] == tokenizer.encode_special("<|assistant_start|>")
    # the assistant response itself must not be present
    stripped = tokenizer.encode("this gets stripped")
    assert not any(ids[i:i + len(stripped)] == stripped for i in range(len(ids)))