Spaces:

sid-0313
/

rag-based-search

No application file

rag-based-search / src /utils /utilities.py

Create utilities.py

d91a750 verified over 2 years ago

156 Bytes

import tiktoken

def count_num_tokens(text: str, model: str) -> int:
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(text))

	import tiktoken

	def count_num_tokens(text: str, model: str) -> int:
	encoding = tiktoken.encoding_for_model(model)
	return len(encoding.encode(text))