Download chat_format.py from KiteFishAI/NanoI1-155M-Base: direct link, hf CLI and curl.
- Browser
- Download file 4.23 kB
-
https://huggingface.co/KiteFishAI/NanoI1-155M-Base/resolve/main/chat_format.py
- Command line
-
hf download hf://KiteFishAI/NanoI1-155M-Base/chat_format.py
-
curl -L -o chat_format.py https://huggingface.co/KiteFishAI/NanoI1-155M-Base/resolve/main/chat_format.py
4.23 kB
| """ | |
| NanoI1 chat format (ChatML on the special tokens reserved in the tokenizer). Every conversation | |
| starts with <|endoftext|>, the same boundary token that preceded each document in pretraining: | |
| <|endoftext|><|im_start|>system\\n{system}<|im_end|>\\n | |
| <|im_start|>user\\n{question}<|im_end|>\\n | |
| <|im_start|>assistant\\n{answer}<|im_end|>\\n | |
| Training, inference (chat.py) and evaluation (evaluate.py / eval_chat.py) all go through this one | |
| module, so the token sequence the model is trained on is exactly what it sees in use: | |
| * `render(messages)` builds the text; `encode(tokenizer, messages)` tokenizes it and returns a | |
| loss mask that is 1 only on assistant content and the closing <|im_end|> of assistant turns. | |
| * Encoding a rendered string as a whole gives the same ids as `encode` (the tokenizer splits on | |
| special tokens, and labels are assigned by character offsets), so there is no train/serve skew. | |
| * Special-token strings inside user/assistant text are removed (`sanitize`) so data or users | |
| cannot inject turn boundaries. | |
| """ | |
| from typing import Dict, List, Optional, Tuple | |
| IM_START, IM_END, EOT, PAD = "<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|pad|>" | |
| ROLES = ("system", "user", "assistant") | |
| ROLE_ALIASES = {"human": "user", "gpt": "assistant", "bot": "assistant", "model": "assistant", | |
| "chatgpt": "assistant", "prompter": "user", "instruction": "user", "response": "assistant"} | |
| _SPECIALS: List[str] = [] | |
| def _specials(tokenizer) -> List[str]: | |
| global _SPECIALS | |
| if not _SPECIALS: | |
| vocab = tokenizer.get_vocab() | |
| _SPECIALS = [t for t in vocab if t.startswith("<|") and t.endswith("|>")] + ["<think>", "</think>"] | |
| return _SPECIALS | |
| def sanitize(text: str, tokenizer=None) -> str: | |
| toks = _specials(tokenizer) if tokenizer is not None else [IM_START, IM_END, EOT, PAD] | |
| for t in toks: | |
| if t in text: | |
| text = text.replace(t, "") | |
| return text | |
| def normalize_role(role: str) -> Optional[str]: | |
| r = (role or "").strip().lower() | |
| r = ROLE_ALIASES.get(r, r) | |
| return r if r in ROLES else None | |
| def render(messages: List[Dict[str, str]], add_generation_prompt: bool = False) -> str: | |
| out = [EOT] | |
| for m in messages: | |
| out.append(f"{IM_START}{m['role']}\n{m['content']}{IM_END}\n") | |
| if add_generation_prompt: | |
| out.append(f"{IM_START}assistant\n") | |
| return "".join(out) | |
| def encode(tokenizer, messages: List[Dict[str, str]], add_generation_prompt: bool = False | |
| ) -> Tuple[List[int], List[int]]: | |
| """Token ids of render(messages) and a same-length 0/1 mask marking the assistant tokens to train on.""" | |
| text = render(messages, add_generation_prompt) | |
| # character spans that should be learned: assistant content + its closing <|im_end|> | |
| spans, pos = [], len(EOT) | |
| for m in messages: | |
| head = f"{IM_START}{m['role']}\n" | |
| start = pos + len(head) | |
| end = start + len(m["content"]) + len(IM_END) | |
| if m["role"] == "assistant": | |
| spans.append((start, end)) | |
| pos = end + 1 # trailing "\n" | |
| enc = tokenizer.encode(text, add_special_tokens=False) | |
| ids, mask = list(enc.ids), [] | |
| si = 0 | |
| for (a, b) in enc.offsets: | |
| while si < len(spans) and spans[si][1] <= a: | |
| si += 1 | |
| inside = si < len(spans) and a >= spans[si][0] and b <= spans[si][1] and b > a | |
| mask.append(1 if inside else 0) | |
| return ids, mask | |
| def prompt_ids(tokenizer, messages: List[Dict[str, str]]) -> List[int]: | |
| """Ids for generation: the conversation so far followed by '<|im_start|>assistant\\n'.""" | |
| return tokenizer.encode(render(messages, add_generation_prompt=True), add_special_tokens=False).ids | |
| def stop_ids(tokenizer) -> List[int]: | |
| return [i for i in (tokenizer.token_to_id(IM_END), tokenizer.token_to_id(EOT)) if i is not None] | |
| # Jinja equivalent, written into tokenizer_config.json so HF `apply_chat_template` matches exactly. | |
| JINJA_TEMPLATE = ( | |
| "{{ '<|endoftext|>' }}" | |
| "{% for message in messages %}" | |
| "{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}" | |
| "{% endfor %}" | |
| "{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}" | |
| ) | |