DU / llm.py
Dmitry Kirsanov
locally debuggable, lots of development
43e002c
Raw History Blame Contribute Delete
6.22 kB
import os, re
import requests
try:
import anthropic
except ModuleNotFoundError:
# Error handling
print("No module anthropic")
pass
try:
import google.generativeai as palm
except ModuleNotFoundError:
# Error handling
print("No module google")
pass
try:
import openai
except ModuleNotFoundError:
# Error handling
print("No module openai")
pass
try:
import tiktoken
except ModuleNotFoundError:
# Error handling
print("No module tiktoken")
pass
import logger
class LLM:
def __init__(self, context_length, api_endpoint, max_output_tokens):
self.context_length = context_length
self.api_endpoint = api_endpoint
self.max_output_tokens = max_output_tokens
def send_question(self, question):
raise NotImplementedError("Subclass must implement send_question method")
def count_tokens(self, text):
raise NotImplementedError("Subclass must implement count_tokens method")
def get_models(self):
raise NotImplementedError("Subclass must implement get_models method")
class Anthropic(LLM):
def __init__(self):
super().__init__(context_length=200000, api_endpoint="https://api.anthropic.com", max_output_tokens=4096)
anthropic.api_key = "YOUR_ANTHROPIC_API_KEY" #FIXME
def send_question(self, question):
num_tokens = self.count_tokens(question)
if num_tokens > self.context_length - self.max_output_tokens:
raise ValueError(f"Question exceeds context length limit of {self.context_length} tokens")
response = anthropic.completions.create(
prompt=f"{anthropic.HUMAN_PROMPT} {question} {anthropic.AI_PROMPT}",
model="claude-3-opus",
max_tokens_to_sample=self.max_output_tokens,
)
output_truncated = len(response.completion) == self.max_output_tokens
return response.completion, output_truncated
def count_tokens(self, text):
encoding = tiktoken.get_encoding("cl100k_base") # FIXME: Claude didn't release its tokenizer; consider using https://github.com/javirandor/anthropic-tokenizer
return len(encoding.encode(text))
class Google(LLM):
def __init__(self):
super().__init__(context_length=256000, api_endpoint="https://generativelanguage.googleapis.com", max_output_tokens=8192)
palm.configure(api_key="YOUR_GOOGLE_API_KEY") #FIXME
def send_question(self, question):
num_tokens = self.count_tokens(question)
if num_tokens > self.context_length - self.max_output_tokens:
raise ValueError(f"Question exceeds context length limit of {self.context_length} tokens")
response = palm.generate_text(
model="models/text-bison-001",
prompt=question,
max_output_tokens=self.max_output_tokens,
)
output_truncated = len(response.result) == self.max_output_tokens
return response.result, output_truncated
def count_tokens(self, text):
encoding = tiktoken.get_encoding("p50k_base") # FIXME: Gemini didn't release its tokenizer; as of apr 2024, the only thing i could google is https://www.googlecloudcommunity.com/gc/AI-ML/Please-share-Gemini-tokenize-information/m-p/709495
return len(encoding.encode(text))
class OpenAI(LLM):
def __init__(self):
super().__init__(context_length=128000, api_endpoint="https://api.openai.com/v1/chat/completions", max_output_tokens=4096)
api_key = os.environ.get("OPENAI_API_KEY")
logger.info(f"Using OpenAI key {api_key[-4:]}")
self.client = openai.OpenAI(api_key = api_key)
logger.info(f"Created OpenAI client")
logger.info(f"Available models: {self.client.models.list()}")
def send_question(self, question):
num_tokens = self.count_tokens(question)
if num_tokens > self.context_length - self.max_output_tokens:
raise ValueError(f"Question has {num_tokens} tokens, exceeds context length limit of {self.context_length} tokens, allowing for {self.max_output_tokens} tokens of answer")
response = self.client.chat.completions.create(
model=self.modelname,
messages=[{"role": "user", "content": question}],
max_tokens=self.max_output_tokens,
)
truncated = response.choices[0].finish_reason == 'length'
if truncated:
print (f"Answer was truncated due to length (model name: {self.modelname}, max output tokens: {self.max_output_tokens}). Modify the instructions to produce a shorter answer or to give answer in parts.")
return response.choices[0].message.content, truncated
def count_tokens(self, text):
encoding = tiktoken.get_encoding("cl100k_base") # GPT-4 uses the cl100k_base encoding https://github.com/openai/tiktoken
return len(encoding.encode(text))
def get_models(self):
return ["gpt-3.5-turbo", "GPT-4o", "gpt-4-32k", "gpt-4", "gpt-4-0125-preview", "gpt-4-turbo-2024-04-09"]
def set_model(self, model):
self.modelname = model
class Debug(LLM):
def __init__(self):
pass
def send_question(self, question):
list_regex = re.compile('Output the line "?([a-zA-Z]+):"? followed by a list')
match = re.search(list_regex, question)
if match:
title = match.group(1)
return f"""{title}:
- item1
- item2
- item3
""", False
if "annotate each sentence" in question and '"actors" attribute' in question:
doc = re.search(r'Here is the .*<document>(.+)</document>', question, flags=re.DOTALL).group(1)
doc = doc.replace('<sentence', '<sentence actors="item1, item2"')
return f'<document>{doc}</document>', False
if "break this document into sections" in question:
doc = re.search(r'Here is the .*<document>(.+)</document>', question, flags=re.DOTALL).group(1)
s = len(doc) // 2
return f'<document><section>{doc[:s]}</section><section>{doc[s:]}</section></document>', False
return("Mock answer from Debug LLM"), False