acpcchatbot_testing_first / app /utils /token_counter.py
acpcchatbotadmin's picture
Upload 40 files
481dfe6 verified
Raw History Blame Contribute Delete
612 Bytes
import tiktoken
# gpt-4o uses o200k_base encoding
_encoder = tiktoken.get_encoding("o200k_base")
def count_tokens(text: str) -> int:
"""Count tokens in a string using gpt-4o's tokenizer."""
return len(_encoder.encode(text))
def count_messages_tokens(messages: list) -> int:
"""Count total tokens across a list of LangChain BaseMessage objects."""
total = 0
for msg in messages:
content = msg.content if hasattr(msg, "content") else str(msg)
total += count_tokens(content)
total += 4 # overhead per message (role + formatting)
return total