| import tiktoken | |
| _encoding = tiktoken.encoding_for_model('gpt-4') | |
| _tokens = map(lambda x: str(x), _encoding.token_byte_values()) | |
| tokens = list(_tokens) | |
| def encode(s: str): | |
| return _encoding.encode(s) | |
| def decode(l: list[int]): | |
| return _encoding.decode(l) | |