import os, re import requests try: import anthropic except ModuleNotFoundError: # Error handling print("No module anthropic") pass try: import google.generativeai as palm except ModuleNotFoundError: # Error handling print("No module google") pass try: import openai except ModuleNotFoundError: # Error handling print("No module openai") pass try: import tiktoken except ModuleNotFoundError: # Error handling print("No module tiktoken") pass import logger class LLM: def __init__(self, context_length, api_endpoint, max_output_tokens): self.context_length = context_length self.api_endpoint = api_endpoint self.max_output_tokens = max_output_tokens def send_question(self, question): raise NotImplementedError("Subclass must implement send_question method") def count_tokens(self, text): raise NotImplementedError("Subclass must implement count_tokens method") def get_models(self): raise NotImplementedError("Subclass must implement get_models method") class Anthropic(LLM): def __init__(self): super().__init__(context_length=200000, api_endpoint="https://api.anthropic.com", max_output_tokens=4096) anthropic.api_key = "YOUR_ANTHROPIC_API_KEY" #FIXME def send_question(self, question): num_tokens = self.count_tokens(question) if num_tokens > self.context_length - self.max_output_tokens: raise ValueError(f"Question exceeds context length limit of {self.context_length} tokens") response = anthropic.completions.create( prompt=f"{anthropic.HUMAN_PROMPT} {question} {anthropic.AI_PROMPT}", model="claude-3-opus", max_tokens_to_sample=self.max_output_tokens, ) output_truncated = len(response.completion) == self.max_output_tokens return response.completion, output_truncated def count_tokens(self, text): encoding = tiktoken.get_encoding("cl100k_base") # FIXME: Claude didn't release its tokenizer; consider using https://github.com/javirandor/anthropic-tokenizer return len(encoding.encode(text)) class Google(LLM): def __init__(self): super().__init__(context_length=256000, api_endpoint="https://generativelanguage.googleapis.com", max_output_tokens=8192) palm.configure(api_key="YOUR_GOOGLE_API_KEY") #FIXME def send_question(self, question): num_tokens = self.count_tokens(question) if num_tokens > self.context_length - self.max_output_tokens: raise ValueError(f"Question exceeds context length limit of {self.context_length} tokens") response = palm.generate_text( model="models/text-bison-001", prompt=question, max_output_tokens=self.max_output_tokens, ) output_truncated = len(response.result) == self.max_output_tokens return response.result, output_truncated def count_tokens(self, text): encoding = tiktoken.get_encoding("p50k_base") # FIXME: Gemini didn't release its tokenizer; as of apr 2024, the only thing i could google is https://www.googlecloudcommunity.com/gc/AI-ML/Please-share-Gemini-tokenize-information/m-p/709495 return len(encoding.encode(text)) class OpenAI(LLM): def __init__(self): super().__init__(context_length=128000, api_endpoint="https://api.openai.com/v1/chat/completions", max_output_tokens=4096) api_key = os.environ.get("OPENAI_API_KEY") logger.info(f"Using OpenAI key {api_key[-4:]}") self.client = openai.OpenAI(api_key = api_key) logger.info(f"Created OpenAI client") logger.info(f"Available models: {self.client.models.list()}") def send_question(self, question): num_tokens = self.count_tokens(question) if num_tokens > self.context_length - self.max_output_tokens: raise ValueError(f"Question has {num_tokens} tokens, exceeds context length limit of {self.context_length} tokens, allowing for {self.max_output_tokens} tokens of answer") response = self.client.chat.completions.create( model=self.modelname, messages=[{"role": "user", "content": question}], max_tokens=self.max_output_tokens, ) truncated = response.choices[0].finish_reason == 'length' if truncated: print (f"Answer was truncated due to length (model name: {self.modelname}, max output tokens: {self.max_output_tokens}). Modify the instructions to produce a shorter answer or to give answer in parts.") return response.choices[0].message.content, truncated def count_tokens(self, text): encoding = tiktoken.get_encoding("cl100k_base") # GPT-4 uses the cl100k_base encoding https://github.com/openai/tiktoken return len(encoding.encode(text)) def get_models(self): return ["gpt-3.5-turbo", "GPT-4o", "gpt-4-32k", "gpt-4", "gpt-4-0125-preview", "gpt-4-turbo-2024-04-09"] def set_model(self, model): self.modelname = model class Debug(LLM): def __init__(self): pass def send_question(self, question): list_regex = re.compile('Output the line "?([a-zA-Z]+):"? followed by a list') match = re.search(list_regex, question) if match: title = match.group(1) return f"""{title}: - item1 - item2 - item3 """, False if "annotate each sentence" in question and '"actors" attribute' in question: doc = re.search(r'Here is the .*(.+)', question, flags=re.DOTALL).group(1) doc = doc.replace('{doc}', False if "break this document into sections" in question: doc = re.search(r'Here is the .*(.+)', question, flags=re.DOTALL).group(1) s = len(doc) // 2 return f'
{doc[:s]}
{doc[s:]}
', False return("Mock answer from Debug LLM"), False