File size: 9,715 Bytes
6868171 867a3e3 8c5d1ba 6868171 867a3e3 6868171 867a3e3 6868171 867a3e3 8c5d1ba 867a3e3 d97f90c 867a3e3 6868171 47fdc91 6868171 8a1cbe0 6868171 867a3e3 6868171 867a3e3 6868171 867a3e3 d82c6f1 e00e24d d82c6f1 867a3e3 d82c6f1 867a3e3 d82c6f1 867a3e3 d82c6f1 867a3e3 d82c6f1 867a3e3 e00e24d d82c6f1 271158e d82c6f1 867a3e3 e00e24d 84c9646 867a3e3 84c9646 d82c6f1 867a3e3 84c9646 d82c6f1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | import itertools
import os
import gradio as gr
import openai
from langchain.chains import RetrievalQA
from langchain.document_loaders.directory import DirectoryLoader
from langchain.memory import ConversationBufferWindowMemory
from langchain.text_splitter import CharacterTextSplitter
import textwrap
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Chroma
from langchain_community.document_loaders import UnstructuredURLLoader
from langchain_core.callbacks import StreamingStdOutCallbackHandler
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from pdfminer.high_level import extract_text
os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
openai.api_key = os.getenv('OPENAI_API_KEY')
#loader = DirectoryLoader('documents/')
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
# if pdf_doc not in txt_documents:
# print("Converting " + pdf_doc + ' pdf to txt')
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
# f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
# if previous_row is not None and previous_row['intent'] == row['intent']:
# intent_index = intent_index + 1
# else:
# intent_index = 0
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
# f.write(row['response'])
# previous_row = row
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
# converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
# if pdf_doc not in txt_documents:
# print("Converting " + pdf_doc + ' pdf to txt')
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
# f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
# if previous_row is not None and previous_row['intent'] == row['intent']:
# intent_index = intent_index + 1
# else:
# intent_index = 0
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
# f.write(row['response'])
# previous_row = row
loader = DirectoryLoader(txts_dir)
txt_documents = loader.load()
loader = UnstructuredURLLoader(urls=['https://sherlock-ph.readthedocs.io/en/latest/',
'https://tkmatrix.readthedocs.io/en/latest/',
'https://dearwatson.readthedocs.io/en/latest/',
'https://dearwatson.readthedocs.io/en/latest/installation.html',
'https://dearwatson.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
'https://sherlock-ph.readthedocs.io/en/latest/installation.html',
'https://sherlock-ph.readthedocs.io/en/latest/the-pipeline.html',
'https://sherlock-ph.readthedocs.io/en/latest/search-proc.html',
'https://sherlock-ph.readthedocs.io/en/latest/toi-178-programmatical.html',
'https://sherlock-ph.readthedocs.io/en/latest/README_270.html',
'https://sherlock-ph.readthedocs.io/en/latest/README_540.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-search.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-search.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting2.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-validation.html',
'https://github.com/franpoz/SHERLOCK',
'https://github.com/PlanetHunters/watson',
'https://github.com/PlanetHunters/tkmatrix',
'https://github.com/iaa-so-training/sherlock-tutorial/tree/main',
'https://github.com/franpoz/SHERLOCK/blob/master/sherlockpipe/properties.yaml'])
url_documents = loader.load()
def wrap_text_preserve_newlines(text, width=110):
# Split the input text into lines based on newline characters
lines = text.split('\n')
# Wrap each line individually
wrapped_lines = [textwrap.fill(line, width=width) for line in lines]
# Join the wrapped lines back together using newline characters
wrapped_text = '\n'.join(wrapped_lines)
return wrapped_text
max_length = 1000
text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10)
docs = text_splitter.split_documents(itertools.chain(txt_documents, url_documents))
model_name = "decapoda-research/llama-7b-hf"
model_name = "gpt-3.5-turbo"
#model_name = "gpt2"
#tokenizer = GPT2Tokenizer.from_pretrained(model_name)
#tokenizer = tiktoken.get_encoding("gpt2")
#assert tokenizer.decode(tokenizer.encode("hola mundo")) == "hola mundo"
# tokenizer._pad_token = tokenizer.eos_token
# tokenizer._pad_token_id = 50256
#model = GPT2LMHeadModel.from_pretrained(model_name)
embeddings = OpenAIEmbeddings()
vector_store = Chroma.from_documents(docs, embeddings)
model_kwargs = {"max_length": max_length, "pad_token_id": 50256, "pad_token": "[PAD]", "padding": 'longest'}
# llm = HuggingFaceHub(repo_id=model_name, model_kwargs={"temperature": 0.1, "max_length": 500,
# "pad_token_id": 50256, "pad_token": "[PAD]",
# "padding": 'longest' #for open end generation
# })
llm = ChatOpenAI(streaming=True, model_name=model_name, temperature=0.9)
prompt_template = """You are going to
receive a context with information about related works, papers and tools for the PhD of Martín Dévora Pajares
(exoplanets field), also appearing as M. Dévora-Pajares.
You can only answer things based on the question and a context to answer it.
Context: {context}
Question: {question}"""
#Memory: {chat_history}"""
PROMPT = PromptTemplate(
template=prompt_template, input_variables=["context", "question"]
)
chain_type_kwargs = {"prompt": PROMPT}
memory = ConversationBufferWindowMemory(k=4, memory_key="chat_history", input_key="question", output_key='answer',
return_messages=True)
retriever = vector_store.as_retriever()
# memory = VectorStoreRetrieverMemory(
# memory_key="chat_history",
# return_docs=False,
# retriever=retriever,
# return_messages=True,
# )
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever,
chain_type_kwargs=chain_type_kwargs, return_source_documents=True)
#memory=memory)
# qa = ConversationalRetrievalChain.from_llm(
# llm=llm, retriever=retriever, memory=memory,
# return_source_documents=True,
# combine_docs_chain_kwargs=chain_type_kwargs)
# while True:
# query = input("Question: ")
# #docs = vector_store.similarity_search(query)
# #print(wrap_text_preserve_newlines(str(docs[0].page_content)))
# response = qa({"query": query})
# print(f"Answer: {response['result']}")
# print(f"Sources: {response['source_documents']}")
def chat(message, history):
result = qa({"query": message})
sources_list = []
for item in result['source_documents']:
if 'http' in item.metadata['source']:
sources_list = sources_list + [item.metadata['source']]
sources_list = list(set(sources_list))
response = result['result']
if len(sources_list) > 0:
response = response + '\n\nSources:\n' + '\n'.join(sources_list)
yield response
# print(f"Answer: {response['result']}")
# print(f"Sources: {response['source_documents']}")
gr.ChatInterface(chat, chatbot=gr.Chatbot(),
textbox=gr.Textbox(placeholder="Ask me about SHERLOCK, MATRIX or WATSON!", container=False, scale=7),
title="PlanetHunters informative chatbot",
description="Ask anything related to SHERLOCK, MATRIX, WATSON and their related papers!",
theme="soft",
examples=["What is SHERLOCK?", "What is MATRIX?", "What are the foundational papers of SHERLOCK?",
"How should I cite SHERLOCK?", "How should I cite MATRIX?"],
cache_examples=False,
retry_btn=None,
undo_btn="Delete Previous",
clear_btn="Clear").queue().launch()
|