Spaces:
Sleeping
Sleeping
| import tiktoken | |
| import json | |
| def count_tokens(string: str, encoding_name: str) -> int: | |
| """Returns the number of tokens in a text string.""" | |
| encoding = tiktoken.encoding_for_model(encoding_name) | |
| num_tokens = len(encoding.encode(string)) | |
| return num_tokens | |
| def load_json(json_file): | |
| with open(json_file) as f: | |
| return json.load(f) | |