File size: 4,234 Bytes
8a73bb7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
"""
NanoI1 chat format (ChatML on the special tokens reserved in the tokenizer). Every conversation
starts with <|endoftext|>, the same boundary token that preceded each document in pretraining:

    <|endoftext|><|im_start|>system\\n{system}<|im_end|>\\n
    <|im_start|>user\\n{question}<|im_end|>\\n
    <|im_start|>assistant\\n{answer}<|im_end|>\\n

Training, inference (chat.py) and evaluation (evaluate.py / eval_chat.py) all go through this one
module, so the token sequence the model is trained on is exactly what it sees in use:

* `render(messages)` builds the text; `encode(tokenizer, messages)` tokenizes it and returns a
  loss mask that is 1 only on assistant content and the closing <|im_end|> of assistant turns.
* Encoding a rendered string as a whole gives the same ids as `encode` (the tokenizer splits on
  special tokens, and labels are assigned by character offsets), so there is no train/serve skew.
* Special-token strings inside user/assistant text are removed (`sanitize`) so data or users
  cannot inject turn boundaries.
"""

from typing import Dict, List, Optional, Tuple

IM_START, IM_END, EOT, PAD = "<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|pad|>"
ROLES = ("system", "user", "assistant")
ROLE_ALIASES = {"human": "user", "gpt": "assistant", "bot": "assistant", "model": "assistant",
                "chatgpt": "assistant", "prompter": "user", "instruction": "user", "response": "assistant"}

_SPECIALS: List[str] = []


def _specials(tokenizer) -> List[str]:
    global _SPECIALS
    if not _SPECIALS:
        vocab = tokenizer.get_vocab()
        _SPECIALS = [t for t in vocab if t.startswith("<|") and t.endswith("|>")] + ["<think>", "</think>"]
    return _SPECIALS


def sanitize(text: str, tokenizer=None) -> str:
    toks = _specials(tokenizer) if tokenizer is not None else [IM_START, IM_END, EOT, PAD]
    for t in toks:
        if t in text:
            text = text.replace(t, "")
    return text


def normalize_role(role: str) -> Optional[str]:
    r = (role or "").strip().lower()
    r = ROLE_ALIASES.get(r, r)
    return r if r in ROLES else None


def render(messages: List[Dict[str, str]], add_generation_prompt: bool = False) -> str:
    out = [EOT]
    for m in messages:
        out.append(f"{IM_START}{m['role']}\n{m['content']}{IM_END}\n")
    if add_generation_prompt:
        out.append(f"{IM_START}assistant\n")
    return "".join(out)


def encode(tokenizer, messages: List[Dict[str, str]], add_generation_prompt: bool = False
           ) -> Tuple[List[int], List[int]]:
    """Token ids of render(messages) and a same-length 0/1 mask marking the assistant tokens to train on."""
    text = render(messages, add_generation_prompt)
    # character spans that should be learned: assistant content + its closing <|im_end|>
    spans, pos = [], len(EOT)
    for m in messages:
        head = f"{IM_START}{m['role']}\n"
        start = pos + len(head)
        end = start + len(m["content"]) + len(IM_END)
        if m["role"] == "assistant":
            spans.append((start, end))
        pos = end + 1  # trailing "\n"
    enc = tokenizer.encode(text, add_special_tokens=False)
    ids, mask = list(enc.ids), []
    si = 0
    for (a, b) in enc.offsets:
        while si < len(spans) and spans[si][1] <= a:
            si += 1
        inside = si < len(spans) and a >= spans[si][0] and b <= spans[si][1] and b > a
        mask.append(1 if inside else 0)
    return ids, mask


def prompt_ids(tokenizer, messages: List[Dict[str, str]]) -> List[int]:
    """Ids for generation: the conversation so far followed by '<|im_start|>assistant\\n'."""
    return tokenizer.encode(render(messages, add_generation_prompt=True), add_special_tokens=False).ids


def stop_ids(tokenizer) -> List[int]:
    return [i for i in (tokenizer.token_to_id(IM_END), tokenizer.token_to_id(EOT)) if i is not None]


# Jinja equivalent, written into tokenizer_config.json so HF `apply_chat_template` matches exactly.
JINJA_TEMPLATE = (
    "{{ '<|endoftext|>' }}"
    "{% for message in messages %}"
    "{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}"
    "{% endfor %}"
    "{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}"
)