NanoI1-155M-Instruct / chat_format.py
anuj0456's picture
Upload folder using huggingface_hub
be97e4b verified
Raw History Blame Contribute Delete
4.23 kB
"""
NanoI1 chat format (ChatML on the special tokens reserved in the tokenizer). Every conversation
starts with <|endoftext|>, the same boundary token that preceded each document in pretraining:
<|endoftext|><|im_start|>system\\n{system}<|im_end|>\\n
<|im_start|>user\\n{question}<|im_end|>\\n
<|im_start|>assistant\\n{answer}<|im_end|>\\n
Training, inference (chat.py) and evaluation (evaluate.py / eval_chat.py) all go through this one
module, so the token sequence the model is trained on is exactly what it sees in use:
* `render(messages)` builds the text; `encode(tokenizer, messages)` tokenizes it and returns a
loss mask that is 1 only on assistant content and the closing <|im_end|> of assistant turns.
* Encoding a rendered string as a whole gives the same ids as `encode` (the tokenizer splits on
special tokens, and labels are assigned by character offsets), so there is no train/serve skew.
* Special-token strings inside user/assistant text are removed (`sanitize`) so data or users
cannot inject turn boundaries.
"""
from typing import Dict, List, Optional, Tuple
IM_START, IM_END, EOT, PAD = "<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|pad|>"
ROLES = ("system", "user", "assistant")
ROLE_ALIASES = {"human": "user", "gpt": "assistant", "bot": "assistant", "model": "assistant",
"chatgpt": "assistant", "prompter": "user", "instruction": "user", "response": "assistant"}
_SPECIALS: List[str] = []
def _specials(tokenizer) -> List[str]:
global _SPECIALS
if not _SPECIALS:
vocab = tokenizer.get_vocab()
_SPECIALS = [t for t in vocab if t.startswith("<|") and t.endswith("|>")] + ["<think>", "</think>"]
return _SPECIALS
def sanitize(text: str, tokenizer=None) -> str:
toks = _specials(tokenizer) if tokenizer is not None else [IM_START, IM_END, EOT, PAD]
for t in toks:
if t in text:
text = text.replace(t, "")
return text
def normalize_role(role: str) -> Optional[str]:
r = (role or "").strip().lower()
r = ROLE_ALIASES.get(r, r)
return r if r in ROLES else None
def render(messages: List[Dict[str, str]], add_generation_prompt: bool = False) -> str:
out = [EOT]
for m in messages:
out.append(f"{IM_START}{m['role']}\n{m['content']}{IM_END}\n")
if add_generation_prompt:
out.append(f"{IM_START}assistant\n")
return "".join(out)
def encode(tokenizer, messages: List[Dict[str, str]], add_generation_prompt: bool = False
) -> Tuple[List[int], List[int]]:
"""Token ids of render(messages) and a same-length 0/1 mask marking the assistant tokens to train on."""
text = render(messages, add_generation_prompt)
# character spans that should be learned: assistant content + its closing <|im_end|>
spans, pos = [], len(EOT)
for m in messages:
head = f"{IM_START}{m['role']}\n"
start = pos + len(head)
end = start + len(m["content"]) + len(IM_END)
if m["role"] == "assistant":
spans.append((start, end))
pos = end + 1 # trailing "\n"
enc = tokenizer.encode(text, add_special_tokens=False)
ids, mask = list(enc.ids), []
si = 0
for (a, b) in enc.offsets:
while si < len(spans) and spans[si][1] <= a:
si += 1
inside = si < len(spans) and a >= spans[si][0] and b <= spans[si][1] and b > a
mask.append(1 if inside else 0)
return ids, mask
def prompt_ids(tokenizer, messages: List[Dict[str, str]]) -> List[int]:
"""Ids for generation: the conversation so far followed by '<|im_start|>assistant\\n'."""
return tokenizer.encode(render(messages, add_generation_prompt=True), add_special_tokens=False).ids
def stop_ids(tokenizer) -> List[int]:
return [i for i in (tokenizer.token_to_id(IM_END), tokenizer.token_to_id(EOT)) if i is not None]
# Jinja equivalent, written into tokenizer_config.json so HF `apply_chat_template` matches exactly.
JINJA_TEMPLATE = (
"{{ '<|endoftext|>' }}"
"{% for message in messages %}"
"{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}"
"{% endfor %}"
"{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}"
)