Download app/utils/token_counter.py from acpcchatbotadmin/acpcchatbot_testing_first: direct link, hf CLI and curl.
- Browser
- Download file 612 Bytes
-
https://huggingface.co/spaces/acpcchatbotadmin/acpcchatbot_testing_first/resolve/main/app/utils/token_counter.py
- Command line
-
hf download hf://spaces/acpcchatbotadmin/acpcchatbot_testing_first/app/utils/token_counter.py
-
curl -L -o token_counter.py https://huggingface.co/spaces/acpcchatbotadmin/acpcchatbot_testing_first/resolve/main/app/utils/token_counter.py
612 Bytes
| import tiktoken | |
| # gpt-4o uses o200k_base encoding | |
| _encoder = tiktoken.get_encoding("o200k_base") | |
| def count_tokens(text: str) -> int: | |
| """Count tokens in a string using gpt-4o's tokenizer.""" | |
| return len(_encoder.encode(text)) | |
| def count_messages_tokens(messages: list) -> int: | |
| """Count total tokens across a list of LangChain BaseMessage objects.""" | |
| total = 0 | |
| for msg in messages: | |
| content = msg.content if hasattr(msg, "content") else str(msg) | |
| total += count_tokens(content) | |
| total += 4 # overhead per message (role + formatting) | |
| return total | |