import itertools import os import gradio as gr import openai from langchain.chains import RetrievalQA from langchain.document_loaders.directory import DirectoryLoader from langchain.memory import ConversationBufferWindowMemory from langchain.text_splitter import CharacterTextSplitter import textwrap from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Chroma from langchain_community.document_loaders import UnstructuredURLLoader from langchain_core.callbacks import StreamingStdOutCallbackHandler from langchain.prompts import PromptTemplate from langchain.chat_models import ChatOpenAI from pdfminer.high_level import extract_text os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN') openai.api_key = os.getenv('OPENAI_API_KEY') #loader = DirectoryLoader('documents/') docs_dir = 'documents/' txts_dir = docs_dir + 'txt/' converted_txts_dir = docs_dir + 'converted_txt/' # pdfs_dir = docs_dir + 'pdf/' # txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))] # pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))] # for pdf_doc in pdf_documents: # if pdf_doc not in txt_documents: # print("Converting " + pdf_doc + ' pdf to txt') # text = extract_text(pdfs_dir + pdf_doc + '.pdf') # with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f: # f.write(text) # intents_df = pd.read_csv(docs_dir + 'intents.csv') # intents_df.sort_values(by=['intent', 'response'], inplace=True) # previous_row = None # for index, row in intents_df.iterrows(): # if previous_row is not None and previous_row['intent'] == row['intent']: # intent_index = intent_index + 1 # else: # intent_index = 0 # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f: # f.write(row['response']) # previous_row = row docs_dir = 'documents/' txts_dir = docs_dir + 'txt/' # converted_txts_dir = docs_dir + 'converted_txt/' # pdfs_dir = docs_dir + 'pdf/' # txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))] # pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))] # for pdf_doc in pdf_documents: # if pdf_doc not in txt_documents: # print("Converting " + pdf_doc + ' pdf to txt') # text = extract_text(pdfs_dir + pdf_doc + '.pdf') # with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f: # f.write(text) # intents_df = pd.read_csv(docs_dir + 'intents.csv') # intents_df.sort_values(by=['intent', 'response'], inplace=True) # previous_row = None # for index, row in intents_df.iterrows(): # if previous_row is not None and previous_row['intent'] == row['intent']: # intent_index = intent_index + 1 # else: # intent_index = 0 # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f: # f.write(row['response']) # previous_row = row loader = DirectoryLoader(txts_dir) txt_documents = loader.load() loader = UnstructuredURLLoader(urls=['https://sherlock-ph.readthedocs.io/en/latest/', 'https://tkmatrix.readthedocs.io/en/latest/', 'https://dearwatson.readthedocs.io/en/latest/', 'https://dearwatson.readthedocs.io/en/latest/installation.html', 'https://dearwatson.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html', 'https://sherlock-ph.readthedocs.io/en/latest/installation.html', 'https://sherlock-ph.readthedocs.io/en/latest/the-pipeline.html', 'https://sherlock-ph.readthedocs.io/en/latest/search-proc.html', 'https://sherlock-ph.readthedocs.io/en/latest/toi-178-programmatical.html', 'https://sherlock-ph.readthedocs.io/en/latest/README_270.html', 'https://sherlock-ph.readthedocs.io/en/latest/README_540.html', 'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-search.html', 'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html', 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-search.html', 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting.html', 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting2.html', 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-validation.html', 'https://github.com/franpoz/SHERLOCK', 'https://github.com/PlanetHunters/watson', 'https://github.com/PlanetHunters/tkmatrix', 'https://github.com/iaa-so-training/sherlock-tutorial/tree/main', 'https://github.com/franpoz/SHERLOCK/blob/master/sherlockpipe/properties.yaml']) url_documents = loader.load() def wrap_text_preserve_newlines(text, width=110): # Split the input text into lines based on newline characters lines = text.split('\n') # Wrap each line individually wrapped_lines = [textwrap.fill(line, width=width) for line in lines] # Join the wrapped lines back together using newline characters wrapped_text = '\n'.join(wrapped_lines) return wrapped_text max_length = 1000 text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10) docs = text_splitter.split_documents(itertools.chain(txt_documents, url_documents)) model_name = "decapoda-research/llama-7b-hf" model_name = "gpt-3.5-turbo" #model_name = "gpt2" #tokenizer = GPT2Tokenizer.from_pretrained(model_name) #tokenizer = tiktoken.get_encoding("gpt2") #assert tokenizer.decode(tokenizer.encode("hola mundo")) == "hola mundo" # tokenizer._pad_token = tokenizer.eos_token # tokenizer._pad_token_id = 50256 #model = GPT2LMHeadModel.from_pretrained(model_name) embeddings = OpenAIEmbeddings() vector_store = Chroma.from_documents(docs, embeddings) model_kwargs = {"max_length": max_length, "pad_token_id": 50256, "pad_token": "[PAD]", "padding": 'longest'} # llm = HuggingFaceHub(repo_id=model_name, model_kwargs={"temperature": 0.1, "max_length": 500, # "pad_token_id": 50256, "pad_token": "[PAD]", # "padding": 'longest' #for open end generation # }) llm = ChatOpenAI(streaming=True, model_name=model_name, temperature=0.9) prompt_template = """You are going to receive a context with information about related works, papers and tools for the PhD of Martín Dévora Pajares (exoplanets field), also appearing as M. Dévora-Pajares. You can only answer things based on the question and a context to answer it. Context: {context} Question: {question}""" #Memory: {chat_history}""" PROMPT = PromptTemplate( template=prompt_template, input_variables=["context", "question"] ) chain_type_kwargs = {"prompt": PROMPT} memory = ConversationBufferWindowMemory(k=4, memory_key="chat_history", input_key="question", output_key='answer', return_messages=True) retriever = vector_store.as_retriever() # memory = VectorStoreRetrieverMemory( # memory_key="chat_history", # return_docs=False, # retriever=retriever, # return_messages=True, # ) qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, chain_type_kwargs=chain_type_kwargs, return_source_documents=True) #memory=memory) # qa = ConversationalRetrievalChain.from_llm( # llm=llm, retriever=retriever, memory=memory, # return_source_documents=True, # combine_docs_chain_kwargs=chain_type_kwargs) # while True: # query = input("Question: ") # #docs = vector_store.similarity_search(query) # #print(wrap_text_preserve_newlines(str(docs[0].page_content))) # response = qa({"query": query}) # print(f"Answer: {response['result']}") # print(f"Sources: {response['source_documents']}") def chat(message, history): result = qa({"query": message}) sources_list = [] for item in result['source_documents']: if 'http' in item.metadata['source']: sources_list = sources_list + [item.metadata['source']] sources_list = list(set(sources_list)) response = result['result'] if len(sources_list) > 0: response = response + '\n\nSources:\n' + '\n'.join(sources_list) yield response # print(f"Answer: {response['result']}") # print(f"Sources: {response['source_documents']}") gr.ChatInterface(chat, chatbot=gr.Chatbot(), textbox=gr.Textbox(placeholder="Ask me about SHERLOCK, MATRIX or WATSON!", container=False, scale=7), title="PlanetHunters informative chatbot", description="Ask anything related to SHERLOCK, MATRIX, WATSON and their related papers!", theme="soft", examples=["What is SHERLOCK?", "What is MATRIX?", "What are the foundational papers of SHERLOCK?", "How should I cite SHERLOCK?", "How should I cite MATRIX?"], cache_examples=False, retry_btn=None, undo_btn="Delete Previous", clear_btn="Clear").queue().launch()