import tiktoken # gpt-4o uses o200k_base encoding _encoder = tiktoken.get_encoding("o200k_base") def count_tokens(text: str) -> int: """Count tokens in a string using gpt-4o's tokenizer.""" return len(_encoder.encode(text)) def count_messages_tokens(messages: list) -> int: """Count total tokens across a list of LangChain BaseMessage objects.""" total = 0 for msg in messages: content = msg.content if hasattr(msg, "content") else str(msg) total += count_tokens(content) total += 4 # overhead per message (role + formatting) return total