from colorama import init, Fore, Style from collections import Counter import pickle, regex, time init(autoreset=True) # the main GPT text split patterns, see # https://github.com/openai/tiktoken/blob/main/tiktoken_ext/openai_public.py GPT4_SPLIT_PATTERN = "|".join( [ r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", r"""\p{N}{1,3}""", r""" ?[^\s\p{L}\p{N}]+[\r\n/]*""", r"""\s*[\r\n]+""", r"""\s+(?!\S)""", r"""\s+""", ] ) def calc_total_time(seconds): # separate the integer part (for hours, minutes, and seconds) from the fractional part (for milliseconds) sec_int, millis = divmod(seconds, 1) millis = int(millis * 1000) # convert the fractional part to milliseconds min, sec = divmod(int(sec_int), 60) hour, min = divmod(min, 60) hours, minutes, seconds = int(hour), int(min), int(sec) t = [ f"{hours} hour" + ("s" if hours > 1 else "") if hours > 0 else None, f"{minutes} minute" + ("s" if minutes > 1 else "") if minutes > 0 else None, f"{seconds} second" + ("s" if seconds > 1 else "") if seconds > 0 else None, f"{millis} ms" if millis > 0 else None ] t = list(filter(None, t)) return ", ".join(t) if t else "0 seconds" # https://github.com/karpathy/minbpe/pull/82/files#diff-2f6d110dc37c6714f3b44335b029a950adfb0c58e2c3013e030a9bbdd76ed02d def get_stats(ids, counts=None, weight=1): """ Given a list of integers, return a dictionary of counts of consecutive pairs, multiplied by weight Example: [1, 2, 3, 1, 2] -> {(1, 2): 2*weight, (2, 3): 1*weight, (3, 1): 1*weight} Optionally allows to update an existing dictionary of counts """ counts = {} if counts is None else counts for pair in zip(ids, ids[1:]): # iterate consecutive elements counts[pair] = counts.get(pair, 0) + weight return counts def merge(ids, pair, idx): """ In the list of integers (ids), replace all consecutive occurrences of pair with the new integer token idx Example: ids=[1, 2, 3, 1, 2], pair=(1, 2), idx=4 -> [4, 3, 4] """ newids = [] i = 0 while i < len(ids): # if not at the very last position AND the pair matches, replace it if ids[i] == pair[0] and i < len(ids) - 1 and ids[i+1] == pair[1]: newids.append(idx) i += 2 else: newids.append(ids[i]) i += 1 return newids class Encoder: def __init__(self, pattern=None): """ - pattern: optional string to override the default (GPT-4 split pattern) - special_tokens: str -> int dictionary of special tokens example: {'<|endoftext|>': 100257} """ self.pattern = GPT4_SPLIT_PATTERN if pattern is None else pattern self.compiled_pattern = regex.compile(self.pattern) self.special_tokens = {} self.inverse_special_tokens = {} self.vocab = {idx: bytes([idx]) for idx in range(256)} # idx -> bytes def train(self, text, vocab_size=256, chunk_range=100_000): """ - path: [name, is_dir] - vocab_size: max number of merges to be made - 256 bytes - text_range: how much many chars from the text should be used to train (default: None means entire text) """ assert vocab_size >= 256 start_time = time.time() print( "encoding text with", f"{Fore.WHITE}{Style.BRIGHT}{len(text)/1e6}M", "total characters and", f"{Fore.WHITE}{Style.BRIGHT}{len(set(text))}", "unique characters" ) if chunk_range is not None: print("ranged chunks has", f"{Fore.WHITE}{Style.BRIGHT}{chunk_range/1e6}M", "chunks") # split the text up into text chunks t = time.time() text_chunks = regex.findall(self.compiled_pattern, text) print("findall:", calc_total_time(time.time() - t)) del text t = time.time() ids = Counter([i for i in text_chunks if len(i) > 1]) print("total unique chunks:", len(ids.most_common())) ids, idsw = map(list, zip(*ids.most_common(chunk_range))) print("dedup:", calc_total_time(time.time() - t)) del text_chunks # input text preprocessing print(f"encoding text chunks... {Fore.WHITE}{Style.DIM}(takes a ~minute)") t = time.time() for i, ch in enumerate(ids): ids[i] = list(ch.encode("utf-8")) print("encode utf-8:", calc_total_time(time.time() - t)) # start training print("training on vocab size", f"{Fore.WHITE}{Style.BRIGHT}{vocab_size}") n_merges = vocab_size - 256 last_print_time = time.time() # iteratively merge the most common pairs to create new tokens for i in range(n_merges): # count the number of times every consecutive pair appears stats = {} # passing in stats will update it in place, adding up counts for j, chunk_ids in enumerate(ids): get_stats(chunk_ids, stats, idsw[j]) # find the pair with the highest count pair = max(stats, key=stats.get) # mint a new token: assign it the next available id idx = 256 + i self.vocab[idx] = self.vocab[pair[0]] + self.vocab[pair[1]] # replace all occurrences of pair in ids with idx ids = [merge(chunk_ids, pair, idx) for chunk_ids in ids] # verbose current_print_time = time.time() print( f"{Fore.WHITE}{Style.BRIGHT}merge", f"{Fore.WHITE}{Style.DIM}[{i+1}/{n_merges}]" ":", f"{pair} -> {idx}", f"{Fore.WHITE}{Style.DIM}({self.vocab[idx]})", f"had {Fore.WHITE}{Style.BRIGHT}{stats[pair]}{Style.RESET_ALL} occurrences" f"{Style.RESET_ALL},", f"{Fore.WHITE}{Style.DIM}time taken: {calc_total_time(current_print_time-last_print_time)}" ) last_print_time = current_print_time # print the total time taken to do all the merges print("vocab size:", f"{Fore.WHITE}{Style.BRIGHT}{len(self.vocab)}") print("time taken:", f"{Fore.WHITE}{Style.BRIGHT}{calc_total_time(time.time()-start_time)}") # special_tokens is a dictionary of str -> int # example: {"<|endoftext|>": 100257} def register_special_tokens(self, *special_tokens): self.special_tokens = dict([(x, i + len(self.vocab)) for i, x in enumerate(special_tokens)]) self.inverse_special_tokens = {v: k for k, v in self.special_tokens.items()} # given ids (list of integers), return Python string def decode(self, ids): part_bytes = [] for idx in ids: if idx in self.vocab: part_bytes.append(self.vocab[idx]) elif idx in self.inverse_special_tokens: part_bytes.append(self.inverse_special_tokens[idx].encode("utf-8")) else: raise ValueError(f"invalid token id: {idx}") text_bytes = b"".join(part_bytes) return text_bytes.decode("utf-8", errors="replace") # return the token ids # https://github.com/karpathy/minbpe/pull/84/files#diff-6b5737d60acbc8d11dba46334d76c559796c1aca8d51e13ed069236f947b9e1f def _encode_chunk(self, text_bytes, inverse_vocab): ids = list(text_bytes) if len(ids) < 2: return ids first, last = 0, 2 while first <= len(ids): if len(ids[first:last]) < 2: break i0 = ids[first:last][0] i1 = ids[first:last][1] if self.vocab[i0] + self.vocab[i1] in inverse_vocab.keys(): ids[first:last] = [inverse_vocab[self.vocab[i0] + self.vocab[i1]]] first, last = 0, 2 else: first += 1 last += 1 return ids def encode_ordinary(self, text, inverse_vocab): """Encoding that ignores any special tokens.""" # split text into chunks of text by categories defined in regex pattern text_chunks = regex.findall(self.compiled_pattern, text) # all chunks of text are encoded separately, then results are joined ids = [] for chunk in text_chunks: chunk_bytes = chunk.encode("utf-8") # raw bytes chunk_ids = self._encode_chunk(chunk_bytes, inverse_vocab) ids.extend(chunk_ids) return ids def encode(self, str, isfile=False, allowed_special="none_raise"): """ Unlike encode_ordinary, this function handles special tokens. allowed_special: can be "all"|"none"|"none_raise" or a custom set of special tokens if none_raise, then an error is raised if any special token is encountered in text this is the default tiktoken behavior right now as well any other behavior is either annoying, or a major footgun """ text = str if isfile: with open(str, "r", encoding="utf-8") as f: text = f.read() # decode the user desire w.r.t. handling of special tokens special = None if allowed_special == "all": special = self.special_tokens elif allowed_special == "none": special = {} elif allowed_special == "none_raise": special = {} assert all(token not in text for token in self.special_tokens) elif isinstance(allowed_special, set): special = {k: v for k, v in self.special_tokens.items() if k in allowed_special} else: raise ValueError(f"allowed_special={allowed_special} not understood") inverse_vocab = {v: k for k, v in self.vocab.items()} # shortcut: if no special tokens, just use the ordinary encoding if not special: return self.encode_ordinary(text, inverse_vocab) # otherwise, we have to be careful with potential special tokens in text # we handle special tokens by splitting the text # based on the occurrence of any exact match with any of the special tokens # we can use regex.split for this. note that surrounding the pattern with () # makes it into a capturing group, so the special tokens will be included special_pattern = "(" + "|".join(regex.escape(k) for k in special) + ")" special_chunks = regex.split(special_pattern, text) del text # now all the special characters are separated from the rest of the text # all chunks of text are encoded separately, then results are joined ids = [] for part in special_chunks: if part in special: # this is a special token, encode it separately as a special case ids.append(special[part]) else: # this is an ordinary sequence, encode it normally ids.extend(self.encode_ordinary(part, inverse_vocab)) return ids def save(self, checkpoint): """ Saves two files: checkpoint.bin - model file is the critical one, intended for load() """ # write the model: to be used in load() later with open(checkpoint + ".bin", "wb") as f: pickle.dump({ "pattern": self.pattern, "special": self.special_tokens, "vocab": self.vocab }, f) # write the model into text file with open(checkpoint + ".txt", "w", encoding="utf-8") as f: f.write(f"pattern:\n{self.pattern}\n\n") f.write(f"special:\n{self.special_tokens}\n\n") f.write(f"vocab:\n{str(self.vocab)[1:-1].replace(', ', '\n')}\n") def load(self, checkpoint: str): # read the model file with open(checkpoint, "rb") as f: model = pickle.load(f) self.pattern = model["pattern"] self.special_tokens = model["special"] self.inverse_special_tokens = {v: k for k, v in self.special_tokens.items()} self.vocab = model["vocab"]