File size: 6,220 Bytes
43e002c
097e81a
43e002c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d465175
 
097e81a
 
 
 
 
 
 
 
 
 
 
 
d1d70b6
 
43e002c
d1d70b6
097e81a
 
0b4b0ed
097e81a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d1d70b6
097e81a
 
0b4b0ed
097e81a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d1d70b6
097e81a
 
c1789c7
d465175
9665739
d465175
9514ac4
097e81a
 
 
 
d1d70b6
097e81a
 
d1d70b6
097e81a
 
 
d1d70b6
3300976
 
d1d70b6
 
097e81a
 
 
d1d70b6
 
 
8c80870
d1d70b6
 
 
 
43e002c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
import os, re
import requests
try:
    import anthropic
except ModuleNotFoundError:
    # Error handling
    print("No module anthropic")
    pass
try:
    import google.generativeai as palm
except ModuleNotFoundError:
    # Error handling
    print("No module google")
    pass
try:
    import openai
except ModuleNotFoundError:
    # Error handling
    print("No module openai")
    pass
try:
    import tiktoken
except ModuleNotFoundError:
    # Error handling
    print("No module tiktoken")
    pass
import logger

class LLM:
    def __init__(self, context_length, api_endpoint, max_output_tokens):
        self.context_length = context_length
        self.api_endpoint = api_endpoint
        self.max_output_tokens = max_output_tokens
        
    def send_question(self, question):
        raise NotImplementedError("Subclass must implement send_question method")
        
    def count_tokens(self, text):
        raise NotImplementedError("Subclass must implement count_tokens method")

    def get_models(self):
        raise NotImplementedError("Subclass must implement get_models method")
    
class Anthropic(LLM):
    def __init__(self):
        super().__init__(context_length=200000, api_endpoint="https://api.anthropic.com", max_output_tokens=4096)
        anthropic.api_key = "YOUR_ANTHROPIC_API_KEY" #FIXME
        
    def send_question(self, question):
        num_tokens = self.count_tokens(question)
        if num_tokens > self.context_length - self.max_output_tokens:
            raise ValueError(f"Question exceeds context length limit of {self.context_length} tokens")
        
        response = anthropic.completions.create(
            prompt=f"{anthropic.HUMAN_PROMPT} {question} {anthropic.AI_PROMPT}",
            model="claude-3-opus",
            max_tokens_to_sample=self.max_output_tokens,
        )
        output_truncated = len(response.completion) == self.max_output_tokens
        return response.completion, output_truncated
    
    def count_tokens(self, text):
        encoding = tiktoken.get_encoding("cl100k_base")  # FIXME: Claude didn't release its tokenizer; consider using https://github.com/javirandor/anthropic-tokenizer 
        return len(encoding.encode(text))

class Google(LLM):
    def __init__(self):
        super().__init__(context_length=256000, api_endpoint="https://generativelanguage.googleapis.com", max_output_tokens=8192)
        palm.configure(api_key="YOUR_GOOGLE_API_KEY") #FIXME
        
    def send_question(self, question):
        num_tokens = self.count_tokens(question)
        if num_tokens > self.context_length - self.max_output_tokens:
            raise ValueError(f"Question exceeds context length limit of {self.context_length} tokens")
        
        response = palm.generate_text(
            model="models/text-bison-001",
            prompt=question,
            max_output_tokens=self.max_output_tokens,
        )
        output_truncated = len(response.result) == self.max_output_tokens
        return response.result, output_truncated
    
    def count_tokens(self, text):
        encoding = tiktoken.get_encoding("p50k_base")  # FIXME: Gemini didn't release its tokenizer; as of apr 2024, the only thing i could google is https://www.googlecloudcommunity.com/gc/AI-ML/Please-share-Gemini-tokenize-information/m-p/709495
        return len(encoding.encode(text))

class OpenAI(LLM):
    def __init__(self):
        super().__init__(context_length=128000, api_endpoint="https://api.openai.com/v1/chat/completions", max_output_tokens=4096)
        api_key = os.environ.get("OPENAI_API_KEY")
        logger.info(f"Using OpenAI key {api_key[-4:]}")
        self.client = openai.OpenAI(api_key = api_key)
        logger.info(f"Created OpenAI client")
        logger.info(f"Available models: {self.client.models.list()}")
        
    def send_question(self, question):
        num_tokens = self.count_tokens(question)
        if num_tokens > self.context_length - self.max_output_tokens:
            raise ValueError(f"Question has {num_tokens} tokens, exceeds context length limit of {self.context_length} tokens, allowing for {self.max_output_tokens} tokens of answer")

        response = self.client.chat.completions.create(
            model=self.modelname,
            messages=[{"role": "user", "content": question}],
            max_tokens=self.max_output_tokens,
        )
        truncated = response.choices[0].finish_reason == 'length'
        if truncated:
            print (f"Answer was truncated due to length (model name: {self.modelname}, max output tokens: {self.max_output_tokens}). Modify the instructions to produce a shorter answer or to give answer in parts.")

        return response.choices[0].message.content, truncated
    
    def count_tokens(self, text):
        encoding = tiktoken.get_encoding("cl100k_base")  # GPT-4 uses the cl100k_base encoding https://github.com/openai/tiktoken
        return len(encoding.encode(text))
    
    def get_models(self):
        return ["gpt-3.5-turbo", "GPT-4o", "gpt-4-32k", "gpt-4", "gpt-4-0125-preview", "gpt-4-turbo-2024-04-09"]
    
    def set_model(self, model):
        self.modelname = model




class Debug(LLM):
    def __init__(self):
        pass

    def send_question(self, question):
        list_regex = re.compile('Output the line "?([a-zA-Z]+):"? followed by a list')
        match = re.search(list_regex, question)
        if match:
            title = match.group(1)
            return f"""{title}:
- item1
- item2
- item3
""", False
        
        if "annotate each sentence" in question and '"actors" attribute' in question:
            doc = re.search(r'Here is the .*<document>(.+)</document>', question, flags=re.DOTALL).group(1)
            doc = doc.replace('<sentence', '<sentence actors="item1, item2"')
            return f'<document>{doc}</document>', False
        
        if "break this document into sections" in question:
            doc = re.search(r'Here is the .*<document>(.+)</document>', question, flags=re.DOTALL).group(1)
            s = len(doc) // 2
            return f'<document><section>{doc[:s]}</section><section>{doc[s:]}</section></document>', False

        return("Mock answer from Debug LLM"), False