File size: 612 Bytes
481dfe6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | import tiktoken
# gpt-4o uses o200k_base encoding
_encoder = tiktoken.get_encoding("o200k_base")
def count_tokens(text: str) -> int:
"""Count tokens in a string using gpt-4o's tokenizer."""
return len(_encoder.encode(text))
def count_messages_tokens(messages: list) -> int:
"""Count total tokens across a list of LangChain BaseMessage objects."""
total = 0
for msg in messages:
content = msg.content if hasattr(msg, "content") else str(msg)
total += count_tokens(content)
total += 4 # overhead per message (role + formatting)
return total
|