sherlock / app.py
mdevora's picture
Update app.py
47fdc91
Raw History Blame Contribute Delete
9.72 kB
import itertools
import os
import gradio as gr
import openai
from langchain.chains import RetrievalQA
from langchain.document_loaders.directory import DirectoryLoader
from langchain.memory import ConversationBufferWindowMemory
from langchain.text_splitter import CharacterTextSplitter
import textwrap
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Chroma
from langchain_community.document_loaders import UnstructuredURLLoader
from langchain_core.callbacks import StreamingStdOutCallbackHandler
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from pdfminer.high_level import extract_text
os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
openai.api_key = os.getenv('OPENAI_API_KEY')
#loader = DirectoryLoader('documents/')
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
# if pdf_doc not in txt_documents:
# print("Converting " + pdf_doc + ' pdf to txt')
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
# f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
# if previous_row is not None and previous_row['intent'] == row['intent']:
# intent_index = intent_index + 1
# else:
# intent_index = 0
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
# f.write(row['response'])
# previous_row = row
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
# converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
# if pdf_doc not in txt_documents:
# print("Converting " + pdf_doc + ' pdf to txt')
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
# f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
# if previous_row is not None and previous_row['intent'] == row['intent']:
# intent_index = intent_index + 1
# else:
# intent_index = 0
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
# f.write(row['response'])
# previous_row = row
loader = DirectoryLoader(txts_dir)
txt_documents = loader.load()
loader = UnstructuredURLLoader(urls=['https://sherlock-ph.readthedocs.io/en/latest/',
'https://tkmatrix.readthedocs.io/en/latest/',
'https://dearwatson.readthedocs.io/en/latest/',
'https://dearwatson.readthedocs.io/en/latest/installation.html',
'https://dearwatson.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
'https://sherlock-ph.readthedocs.io/en/latest/installation.html',
'https://sherlock-ph.readthedocs.io/en/latest/the-pipeline.html',
'https://sherlock-ph.readthedocs.io/en/latest/search-proc.html',
'https://sherlock-ph.readthedocs.io/en/latest/toi-178-programmatical.html',
'https://sherlock-ph.readthedocs.io/en/latest/README_270.html',
'https://sherlock-ph.readthedocs.io/en/latest/README_540.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-search.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-search.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting2.html',
'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-validation.html',
'https://github.com/franpoz/SHERLOCK',
'https://github.com/PlanetHunters/watson',
'https://github.com/PlanetHunters/tkmatrix',
'https://github.com/iaa-so-training/sherlock-tutorial/tree/main',
'https://github.com/franpoz/SHERLOCK/blob/master/sherlockpipe/properties.yaml'])
url_documents = loader.load()
def wrap_text_preserve_newlines(text, width=110):
# Split the input text into lines based on newline characters
lines = text.split('\n')
# Wrap each line individually
wrapped_lines = [textwrap.fill(line, width=width) for line in lines]
# Join the wrapped lines back together using newline characters
wrapped_text = '\n'.join(wrapped_lines)
return wrapped_text
max_length = 1000
text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10)
docs = text_splitter.split_documents(itertools.chain(txt_documents, url_documents))
model_name = "decapoda-research/llama-7b-hf"
model_name = "gpt-3.5-turbo"
#model_name = "gpt2"
#tokenizer = GPT2Tokenizer.from_pretrained(model_name)
#tokenizer = tiktoken.get_encoding("gpt2")
#assert tokenizer.decode(tokenizer.encode("hola mundo")) == "hola mundo"
# tokenizer._pad_token = tokenizer.eos_token
# tokenizer._pad_token_id = 50256
#model = GPT2LMHeadModel.from_pretrained(model_name)
embeddings = OpenAIEmbeddings()
vector_store = Chroma.from_documents(docs, embeddings)
model_kwargs = {"max_length": max_length, "pad_token_id": 50256, "pad_token": "[PAD]", "padding": 'longest'}
# llm = HuggingFaceHub(repo_id=model_name, model_kwargs={"temperature": 0.1, "max_length": 500,
# "pad_token_id": 50256, "pad_token": "[PAD]",
# "padding": 'longest' #for open end generation
# })
llm = ChatOpenAI(streaming=True, model_name=model_name, temperature=0.9)
prompt_template = """You are going to
receive a context with information about related works, papers and tools for the PhD of Martín Dévora Pajares
(exoplanets field), also appearing as M. Dévora-Pajares.
You can only answer things based on the question and a context to answer it.
Context: {context}
Question: {question}"""
#Memory: {chat_history}"""
PROMPT = PromptTemplate(
template=prompt_template, input_variables=["context", "question"]
)
chain_type_kwargs = {"prompt": PROMPT}
memory = ConversationBufferWindowMemory(k=4, memory_key="chat_history", input_key="question", output_key='answer',
return_messages=True)
retriever = vector_store.as_retriever()
# memory = VectorStoreRetrieverMemory(
# memory_key="chat_history",
# return_docs=False,
# retriever=retriever,
# return_messages=True,
# )
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever,
chain_type_kwargs=chain_type_kwargs, return_source_documents=True)
#memory=memory)
# qa = ConversationalRetrievalChain.from_llm(
# llm=llm, retriever=retriever, memory=memory,
# return_source_documents=True,
# combine_docs_chain_kwargs=chain_type_kwargs)
# while True:
# query = input("Question: ")
# #docs = vector_store.similarity_search(query)
# #print(wrap_text_preserve_newlines(str(docs[0].page_content)))
# response = qa({"query": query})
# print(f"Answer: {response['result']}")
# print(f"Sources: {response['source_documents']}")
def chat(message, history):
result = qa({"query": message})
sources_list = []
for item in result['source_documents']:
if 'http' in item.metadata['source']:
sources_list = sources_list + [item.metadata['source']]
sources_list = list(set(sources_list))
response = result['result']
if len(sources_list) > 0:
response = response + '\n\nSources:\n' + '\n'.join(sources_list)
yield response
# print(f"Answer: {response['result']}")
# print(f"Sources: {response['source_documents']}")
gr.ChatInterface(chat, chatbot=gr.Chatbot(),
textbox=gr.Textbox(placeholder="Ask me about SHERLOCK, MATRIX or WATSON!", container=False, scale=7),
title="PlanetHunters informative chatbot",
description="Ask anything related to SHERLOCK, MATRIX, WATSON and their related papers!",
theme="soft",
examples=["What is SHERLOCK?", "What is MATRIX?", "What are the foundational papers of SHERLOCK?",
"How should I cite SHERLOCK?", "How should I cite MATRIX?"],
cache_examples=False,
retry_btn=None,
undo_btn="Delete Previous",
clear_btn="Clear").queue().launch()