File size: 9,715 Bytes
6868171
867a3e3
 
 
 
 
8c5d1ba
6868171
867a3e3
6868171
867a3e3
6868171
867a3e3
 
 
8c5d1ba
867a3e3
 
 
 
 
 
 
 
d97f90c
 
 
 
 
 
 
 
 
867a3e3
 
 
 
 
 
 
 
 
 
 
6868171
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47fdc91
 
6868171
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8a1cbe0
 
6868171
867a3e3
 
 
 
 
 
 
 
 
 
 
6868171
867a3e3
6868171
867a3e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d82c6f1
e00e24d
 
 
d82c6f1
867a3e3
d82c6f1
 
867a3e3
 
d82c6f1
867a3e3
 
d82c6f1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
867a3e3
 
 
 
 
 
 
d82c6f1
867a3e3
e00e24d
d82c6f1
 
 
 
 
 
 
271158e
d82c6f1
867a3e3
 
 
e00e24d
84c9646
 
 
867a3e3
84c9646
 
d82c6f1
867a3e3
84c9646
 
d82c6f1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
import itertools
import os
import gradio as gr
import openai
from langchain.chains import RetrievalQA
from langchain.document_loaders.directory import DirectoryLoader
from langchain.memory import ConversationBufferWindowMemory
from langchain.text_splitter import CharacterTextSplitter
import textwrap
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Chroma
from langchain_community.document_loaders import UnstructuredURLLoader
from langchain_core.callbacks import StreamingStdOutCallbackHandler
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from pdfminer.high_level import extract_text


os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
openai.api_key = os.getenv('OPENAI_API_KEY')
#loader = DirectoryLoader('documents/')
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
#     if pdf_doc not in txt_documents:
#         print("Converting " + pdf_doc + ' pdf to txt')
#         text = extract_text(pdfs_dir + pdf_doc + '.pdf')
#         with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
#             f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
#     if previous_row is not None and previous_row['intent'] == row['intent']:
#         intent_index = intent_index + 1
#     else:
#         intent_index = 0
#     with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
#         f.write(row['response'])
#     previous_row = row
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
# converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
#     if pdf_doc not in txt_documents:
#         print("Converting " + pdf_doc + ' pdf to txt')
#         text = extract_text(pdfs_dir + pdf_doc + '.pdf')
#         with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
#             f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
#     if previous_row is not None and previous_row['intent'] == row['intent']:
#         intent_index = intent_index + 1
#     else:
#         intent_index = 0
#     with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
#         f.write(row['response'])
#     previous_row = row
loader = DirectoryLoader(txts_dir)
txt_documents = loader.load()
loader = UnstructuredURLLoader(urls=['https://sherlock-ph.readthedocs.io/en/latest/',
                                     'https://tkmatrix.readthedocs.io/en/latest/',
                                     'https://dearwatson.readthedocs.io/en/latest/',
                                     'https://dearwatson.readthedocs.io/en/latest/installation.html',
                                     'https://dearwatson.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/installation.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/the-pipeline.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/search-proc.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/toi-178-programmatical.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/README_270.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/README_540.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-search.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-search.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting2.html',
                                     'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-validation.html',
                                     'https://github.com/franpoz/SHERLOCK',
                                     'https://github.com/PlanetHunters/watson',
                                     'https://github.com/PlanetHunters/tkmatrix',
                                     'https://github.com/iaa-so-training/sherlock-tutorial/tree/main',
                                     'https://github.com/franpoz/SHERLOCK/blob/master/sherlockpipe/properties.yaml'])
url_documents = loader.load()
def wrap_text_preserve_newlines(text, width=110):
    # Split the input text into lines based on newline characters
    lines = text.split('\n')
    # Wrap each line individually
    wrapped_lines = [textwrap.fill(line, width=width) for line in lines]
    # Join the wrapped lines back together using newline characters
    wrapped_text = '\n'.join(wrapped_lines)
    return wrapped_text

max_length = 1000
text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10)
docs = text_splitter.split_documents(itertools.chain(txt_documents, url_documents))

model_name = "decapoda-research/llama-7b-hf"
model_name = "gpt-3.5-turbo"
#model_name = "gpt2"
#tokenizer = GPT2Tokenizer.from_pretrained(model_name)
#tokenizer = tiktoken.get_encoding("gpt2")
#assert tokenizer.decode(tokenizer.encode("hola mundo")) == "hola mundo"
# tokenizer._pad_token = tokenizer.eos_token
# tokenizer._pad_token_id = 50256
#model = GPT2LMHeadModel.from_pretrained(model_name)
embeddings = OpenAIEmbeddings()
vector_store = Chroma.from_documents(docs, embeddings)
model_kwargs = {"max_length": max_length, "pad_token_id": 50256, "pad_token": "[PAD]", "padding": 'longest'}
# llm = HuggingFaceHub(repo_id=model_name, model_kwargs={"temperature": 0.1, "max_length": 500,
#                                                                    "pad_token_id": 50256, "pad_token": "[PAD]",
#                                                        "padding": 'longest' #for open end generation
#                                                               })
llm = ChatOpenAI(streaming=True, model_name=model_name, temperature=0.9)
prompt_template = """You are going to 
receive a context with information about related works, papers and tools for the PhD of Martín Dévora Pajares 
(exoplanets field), also appearing as M. Dévora-Pajares. 
You can only answer things based on the question and a context to answer it.
Context: {context}
Question: {question}"""
#Memory: {chat_history}"""

PROMPT = PromptTemplate(
    template=prompt_template, input_variables=["context", "question"]
)
chain_type_kwargs = {"prompt": PROMPT}
memory = ConversationBufferWindowMemory(k=4, memory_key="chat_history", input_key="question", output_key='answer',
                                        return_messages=True)

retriever = vector_store.as_retriever()
# memory = VectorStoreRetrieverMemory(
#     memory_key="chat_history",
#     return_docs=False,
#     retriever=retriever,
#     return_messages=True,
# )
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever,
                                 chain_type_kwargs=chain_type_kwargs, return_source_documents=True)
                                 #memory=memory)
# qa = ConversationalRetrievalChain.from_llm(
#         llm=llm, retriever=retriever, memory=memory,
#         return_source_documents=True,
#         combine_docs_chain_kwargs=chain_type_kwargs)


# while True:
#     query = input("Question: ")
#     #docs = vector_store.similarity_search(query)
#     #print(wrap_text_preserve_newlines(str(docs[0].page_content)))
#     response = qa({"query": query})
#     print(f"Answer: {response['result']}")
#     print(f"Sources: {response['source_documents']}")

def chat(message, history):
    result = qa({"query": message})
    sources_list = []
    for item in result['source_documents']:
        if 'http' in item.metadata['source']:
            sources_list = sources_list + [item.metadata['source']]
    sources_list = list(set(sources_list))
    response = result['result']
    if len(sources_list) > 0:
        response = response + '\n\nSources:\n' + '\n'.join(sources_list)
    yield response
    # print(f"Answer: {response['result']}")
    # print(f"Sources: {response['source_documents']}")

gr.ChatInterface(chat, chatbot=gr.Chatbot(),
    textbox=gr.Textbox(placeholder="Ask me about SHERLOCK, MATRIX or WATSON!", container=False, scale=7),
    title="PlanetHunters informative chatbot",
    description="Ask anything related to SHERLOCK, MATRIX, WATSON and their related papers!",
    theme="soft",
    examples=["What is SHERLOCK?", "What is MATRIX?", "What are the foundational papers of SHERLOCK?",
              "How should I cite SHERLOCK?", "How should I cite MATRIX?"],
    cache_examples=False,
    retry_btn=None,
    undo_btn="Delete Previous",
    clear_btn="Clear").queue().launch()