Text Generation
English
Pierre / encoder.py
Srijan-Srivastava's picture
Upload 10 files
d3a14da verified
Raw History Blame Contribute Delete
10.7 kB
from colorama import init, Fore, Style
from collections import Counter
import pickle, regex, time
init(autoreset=True)
# the main GPT text split patterns, see
# https://github.com/openai/tiktoken/blob/main/tiktoken_ext/openai_public.py
GPT4_SPLIT_PATTERN = "|".join(
[
r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
r"""\p{N}{1,3}""",
r""" ?[^\s\p{L}\p{N}]+[\r\n/]*""",
r"""\s*[\r\n]+""",
r"""\s+(?!\S)""",
r"""\s+""",
]
)
def calc_total_time(seconds):
# separate the integer part (for hours, minutes, and seconds) from the fractional part (for milliseconds)
sec_int, millis = divmod(seconds, 1)
millis = int(millis * 1000) # convert the fractional part to milliseconds
min, sec = divmod(int(sec_int), 60)
hour, min = divmod(min, 60)
hours, minutes, seconds = int(hour), int(min), int(sec)
t = [
f"{hours} hour" + ("s" if hours > 1 else "") if hours > 0 else None,
f"{minutes} minute" + ("s" if minutes > 1 else "") if minutes > 0 else None,
f"{seconds} second" + ("s" if seconds > 1 else "") if seconds > 0 else None,
f"{millis} ms" if millis > 0 else None
]
t = list(filter(None, t))
return ", ".join(t) if t else "0 seconds"
# https://github.com/karpathy/minbpe/pull/82/files#diff-2f6d110dc37c6714f3b44335b029a950adfb0c58e2c3013e030a9bbdd76ed02d
def get_stats(ids, counts=None, weight=1):
"""
Given a list of integers, return a dictionary of counts of consecutive pairs, multiplied by weight
Example: [1, 2, 3, 1, 2] -> {(1, 2): 2*weight, (2, 3): 1*weight, (3, 1): 1*weight}
Optionally allows to update an existing dictionary of counts
"""
counts = {} if counts is None else counts
for pair in zip(ids, ids[1:]): # iterate consecutive elements
counts[pair] = counts.get(pair, 0) + weight
return counts
def merge(ids, pair, idx):
"""
In the list of integers (ids), replace all consecutive occurrences
of pair with the new integer token idx
Example: ids=[1, 2, 3, 1, 2], pair=(1, 2), idx=4 -> [4, 3, 4]
"""
newids = []
i = 0
while i < len(ids):
# if not at the very last position AND the pair matches, replace it
if ids[i] == pair[0] and i < len(ids) - 1 and ids[i+1] == pair[1]:
newids.append(idx)
i += 2
else:
newids.append(ids[i])
i += 1
return newids
class Encoder:
def __init__(self, pattern=None):
"""
- pattern: optional string to override the default (GPT-4 split pattern)
- special_tokens: str -> int dictionary of special tokens
example: {'<|endoftext|>': 100257}
"""
self.pattern = GPT4_SPLIT_PATTERN if pattern is None else pattern
self.compiled_pattern = regex.compile(self.pattern)
self.special_tokens = {}
self.inverse_special_tokens = {}
self.vocab = {idx: bytes([idx]) for idx in range(256)} # idx -> bytes
def train(self, text, vocab_size=256, chunk_range=100_000):
"""
- path: [name, is_dir]
- vocab_size: max number of merges to be made - 256 bytes
- text_range: how much many chars from the text should be used to train (default: None means entire text)
"""
assert vocab_size >= 256
start_time = time.time()
print(
"encoding text with", f"{Fore.WHITE}{Style.BRIGHT}{len(text)/1e6}M", "total characters and",
f"{Fore.WHITE}{Style.BRIGHT}{len(set(text))}", "unique characters"
)
if chunk_range is not None:
print("ranged chunks has", f"{Fore.WHITE}{Style.BRIGHT}{chunk_range/1e6}M", "chunks")
# split the text up into text chunks
t = time.time()
text_chunks = regex.findall(self.compiled_pattern, text)
print("findall:", calc_total_time(time.time() - t))
del text
t = time.time()
ids = Counter([i for i in text_chunks if len(i) > 1])
print("total unique chunks:", len(ids.most_common()))
ids, idsw = map(list, zip(*ids.most_common(chunk_range)))
print("dedup:", calc_total_time(time.time() - t))
del text_chunks
# input text preprocessing
print(f"encoding text chunks... {Fore.WHITE}{Style.DIM}(takes a ~minute)")
t = time.time()
for i, ch in enumerate(ids):
ids[i] = list(ch.encode("utf-8"))
print("encode utf-8:", calc_total_time(time.time() - t))
# start training
print("training on vocab size", f"{Fore.WHITE}{Style.BRIGHT}{vocab_size}")
n_merges = vocab_size - 256
last_print_time = time.time()
# iteratively merge the most common pairs to create new tokens
for i in range(n_merges):
# count the number of times every consecutive pair appears
stats = {}
# passing in stats will update it in place, adding up counts
for j, chunk_ids in enumerate(ids):
get_stats(chunk_ids, stats, idsw[j])
# find the pair with the highest count
pair = max(stats, key=stats.get)
# mint a new token: assign it the next available id
idx = 256 + i
self.vocab[idx] = self.vocab[pair[0]] + self.vocab[pair[1]]
# replace all occurrences of pair in ids with idx
ids = [merge(chunk_ids, pair, idx) for chunk_ids in ids]
# verbose
current_print_time = time.time()
print(
f"{Fore.WHITE}{Style.BRIGHT}merge",
f"{Fore.WHITE}{Style.DIM}[{i+1}/{n_merges}]"
":",
f"{pair} -> {idx}",
f"{Fore.WHITE}{Style.DIM}({self.vocab[idx]})",
f"had {Fore.WHITE}{Style.BRIGHT}{stats[pair]}{Style.RESET_ALL} occurrences"
f"{Style.RESET_ALL},",
f"{Fore.WHITE}{Style.DIM}time taken: {calc_total_time(current_print_time-last_print_time)}"
)
last_print_time = current_print_time
# print the total time taken to do all the merges
print("vocab size:", f"{Fore.WHITE}{Style.BRIGHT}{len(self.vocab)}")
print("time taken:", f"{Fore.WHITE}{Style.BRIGHT}{calc_total_time(time.time()-start_time)}")
# special_tokens is a dictionary of str -> int
# example: {"<|endoftext|>": 100257}
def register_special_tokens(self, *special_tokens):
self.special_tokens = dict([(x, i + len(self.vocab)) for i, x in enumerate(special_tokens)])
self.inverse_special_tokens = {v: k for k, v in self.special_tokens.items()}
# given ids (list of integers), return Python string
def decode(self, ids):
part_bytes = []
for idx in ids:
if idx in self.vocab:
part_bytes.append(self.vocab[idx])
elif idx in self.inverse_special_tokens:
part_bytes.append(self.inverse_special_tokens[idx].encode("utf-8"))
else:
raise ValueError(f"invalid token id: {idx}")
text_bytes = b"".join(part_bytes)
return text_bytes.decode("utf-8", errors="replace")
# return the token ids
# https://github.com/karpathy/minbpe/pull/84/files#diff-6b5737d60acbc8d11dba46334d76c559796c1aca8d51e13ed069236f947b9e1f
def _encode_chunk(self, text_bytes, inverse_vocab):
ids = list(text_bytes)
if len(ids) < 2:
return ids
first, last = 0, 2
while first <= len(ids):
if len(ids[first:last]) < 2:
break
i0 = ids[first:last][0]
i1 = ids[first:last][1]
if self.vocab[i0] + self.vocab[i1] in inverse_vocab.keys():
ids[first:last] = [inverse_vocab[self.vocab[i0] + self.vocab[i1]]]
first, last = 0, 2
else:
first += 1
last += 1
return ids
def encode_ordinary(self, text, inverse_vocab):
"""Encoding that ignores any special tokens."""
# split text into chunks of text by categories defined in regex pattern
text_chunks = regex.findall(self.compiled_pattern, text)
# all chunks of text are encoded separately, then results are joined
ids = []
for chunk in text_chunks:
chunk_bytes = chunk.encode("utf-8") # raw bytes
chunk_ids = self._encode_chunk(chunk_bytes, inverse_vocab)
ids.extend(chunk_ids)
return ids
def encode(self, str, isfile=False, allowed_special="none_raise"):
"""
Unlike encode_ordinary, this function handles special tokens.
allowed_special: can be "all"|"none"|"none_raise" or a custom set of special tokens
if none_raise, then an error is raised if any special token is encountered in text
this is the default tiktoken behavior right now as well
any other behavior is either annoying, or a major footgun
"""
text = str
if isfile:
with open(str, "r", encoding="utf-8") as f:
text = f.read()
# decode the user desire w.r.t. handling of special tokens
special = None
if allowed_special == "all":
special = self.special_tokens
elif allowed_special == "none":
special = {}
elif allowed_special == "none_raise":
special = {}
assert all(token not in text for token in self.special_tokens)
elif isinstance(allowed_special, set):
special = {k: v for k, v in self.special_tokens.items() if k in allowed_special}
else:
raise ValueError(f"allowed_special={allowed_special} not understood")
inverse_vocab = {v: k for k, v in self.vocab.items()}
# shortcut: if no special tokens, just use the ordinary encoding
if not special:
return self.encode_ordinary(text, inverse_vocab)
# otherwise, we have to be careful with potential special tokens in text
# we handle special tokens by splitting the text
# based on the occurrence of any exact match with any of the special tokens
# we can use regex.split for this. note that surrounding the pattern with ()
# makes it into a capturing group, so the special tokens will be included
special_pattern = "(" + "|".join(regex.escape(k) for k in special) + ")"
special_chunks = regex.split(special_pattern, text)
del text
# now all the special characters are separated from the rest of the text
# all chunks of text are encoded separately, then results are joined
ids = []
for part in special_chunks:
if part in special:
# this is a special token, encode it separately as a special case
ids.append(special[part])
else:
# this is an ordinary sequence, encode it normally
ids.extend(self.encode_ordinary(part, inverse_vocab))
return ids
def save(self, checkpoint):
"""
Saves two files: checkpoint.bin
- model file is the critical one, intended for load()
"""
# write the model: to be used in load() later
with open(checkpoint + ".bin", "wb") as f:
pickle.dump({
"pattern": self.pattern,
"special": self.special_tokens,
"vocab": self.vocab
}, f)
# write the model into text file
with open(checkpoint + ".txt", "w", encoding="utf-8") as f:
f.write(f"pattern:\n{self.pattern}\n\n")
f.write(f"special:\n{self.special_tokens}\n\n")
f.write(f"vocab:\n{str(self.vocab)[1:-1].replace(', ', '\n')}\n")
def load(self, checkpoint: str):
# read the model file
with open(checkpoint, "rb") as f:
model = pickle.load(f)
self.pattern = model["pattern"]
self.special_tokens = model["special"]
self.inverse_special_tokens = {v: k for k, v in self.special_tokens.items()}
self.vocab = model["vocab"]