File size: 612 Bytes
481dfe6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
import tiktoken

# gpt-4o uses o200k_base encoding
_encoder = tiktoken.get_encoding("o200k_base")


def count_tokens(text: str) -> int:
    """Count tokens in a string using gpt-4o's tokenizer."""
    return len(_encoder.encode(text))


def count_messages_tokens(messages: list) -> int:
    """Count total tokens across a list of LangChain BaseMessage objects."""
    total = 0
    for msg in messages:
        content = msg.content if hasattr(msg, "content") else str(msg)
        total += count_tokens(content)
        total += 4  # overhead per message (role + formatting)
    return total