""" NanoI1 chat format (ChatML on the special tokens reserved in the tokenizer). Every conversation starts with <|endoftext|>, the same boundary token that preceded each document in pretraining: <|endoftext|><|im_start|>system\\n{system}<|im_end|>\\n <|im_start|>user\\n{question}<|im_end|>\\n <|im_start|>assistant\\n{answer}<|im_end|>\\n Training, inference (chat.py) and evaluation (evaluate.py / eval_chat.py) all go through this one module, so the token sequence the model is trained on is exactly what it sees in use: * `render(messages)` builds the text; `encode(tokenizer, messages)` tokenizes it and returns a loss mask that is 1 only on assistant content and the closing <|im_end|> of assistant turns. * Encoding a rendered string as a whole gives the same ids as `encode` (the tokenizer splits on special tokens, and labels are assigned by character offsets), so there is no train/serve skew. * Special-token strings inside user/assistant text are removed (`sanitize`) so data or users cannot inject turn boundaries. """ from typing import Dict, List, Optional, Tuple IM_START, IM_END, EOT, PAD = "<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|pad|>" ROLES = ("system", "user", "assistant") ROLE_ALIASES = {"human": "user", "gpt": "assistant", "bot": "assistant", "model": "assistant", "chatgpt": "assistant", "prompter": "user", "instruction": "user", "response": "assistant"} _SPECIALS: List[str] = [] def _specials(tokenizer) -> List[str]: global _SPECIALS if not _SPECIALS: vocab = tokenizer.get_vocab() _SPECIALS = [t for t in vocab if t.startswith("<|") and t.endswith("|>")] + ["", ""] return _SPECIALS def sanitize(text: str, tokenizer=None) -> str: toks = _specials(tokenizer) if tokenizer is not None else [IM_START, IM_END, EOT, PAD] for t in toks: if t in text: text = text.replace(t, "") return text def normalize_role(role: str) -> Optional[str]: r = (role or "").strip().lower() r = ROLE_ALIASES.get(r, r) return r if r in ROLES else None def render(messages: List[Dict[str, str]], add_generation_prompt: bool = False) -> str: out = [EOT] for m in messages: out.append(f"{IM_START}{m['role']}\n{m['content']}{IM_END}\n") if add_generation_prompt: out.append(f"{IM_START}assistant\n") return "".join(out) def encode(tokenizer, messages: List[Dict[str, str]], add_generation_prompt: bool = False ) -> Tuple[List[int], List[int]]: """Token ids of render(messages) and a same-length 0/1 mask marking the assistant tokens to train on.""" text = render(messages, add_generation_prompt) # character spans that should be learned: assistant content + its closing <|im_end|> spans, pos = [], len(EOT) for m in messages: head = f"{IM_START}{m['role']}\n" start = pos + len(head) end = start + len(m["content"]) + len(IM_END) if m["role"] == "assistant": spans.append((start, end)) pos = end + 1 # trailing "\n" enc = tokenizer.encode(text, add_special_tokens=False) ids, mask = list(enc.ids), [] si = 0 for (a, b) in enc.offsets: while si < len(spans) and spans[si][1] <= a: si += 1 inside = si < len(spans) and a >= spans[si][0] and b <= spans[si][1] and b > a mask.append(1 if inside else 0) return ids, mask def prompt_ids(tokenizer, messages: List[Dict[str, str]]) -> List[int]: """Ids for generation: the conversation so far followed by '<|im_start|>assistant\\n'.""" return tokenizer.encode(render(messages, add_generation_prompt=True), add_special_tokens=False).ids def stop_ids(tokenizer) -> List[int]: return [i for i in (tokenizer.token_to_id(IM_END), tokenizer.token_to_id(EOT)) if i is not None] # Jinja equivalent, written into tokenizer_config.json so HF `apply_chat_template` matches exactly. JINJA_TEMPLATE = ( "{{ '<|endoftext|>' }}" "{% for message in messages %}" "{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}" "{% endfor %}" "{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}" )