Download app.py from mdevora/sherlock: direct link, hf CLI and curl.
- Browser
- Download file 9.72 kB
-
https://huggingface.co/spaces/mdevora/sherlock/resolve/main/app.py
- Command line
-
hf download hf://spaces/mdevora/sherlock/app.py
-
curl -L -o app.py https://huggingface.co/spaces/mdevora/sherlock/resolve/main/app.py
9.72 kB
| import itertools | |
| import os | |
| import gradio as gr | |
| import openai | |
| from langchain.chains import RetrievalQA | |
| from langchain.document_loaders.directory import DirectoryLoader | |
| from langchain.memory import ConversationBufferWindowMemory | |
| from langchain.text_splitter import CharacterTextSplitter | |
| import textwrap | |
| from langchain.embeddings import OpenAIEmbeddings | |
| from langchain.vectorstores import Chroma | |
| from langchain_community.document_loaders import UnstructuredURLLoader | |
| from langchain_core.callbacks import StreamingStdOutCallbackHandler | |
| from langchain.prompts import PromptTemplate | |
| from langchain.chat_models import ChatOpenAI | |
| from pdfminer.high_level import extract_text | |
| os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN') | |
| openai.api_key = os.getenv('OPENAI_API_KEY') | |
| #loader = DirectoryLoader('documents/') | |
| docs_dir = 'documents/' | |
| txts_dir = docs_dir + 'txt/' | |
| converted_txts_dir = docs_dir + 'converted_txt/' | |
| # pdfs_dir = docs_dir + 'pdf/' | |
| # txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))] | |
| # pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))] | |
| # for pdf_doc in pdf_documents: | |
| # if pdf_doc not in txt_documents: | |
| # print("Converting " + pdf_doc + ' pdf to txt') | |
| # text = extract_text(pdfs_dir + pdf_doc + '.pdf') | |
| # with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f: | |
| # f.write(text) | |
| # intents_df = pd.read_csv(docs_dir + 'intents.csv') | |
| # intents_df.sort_values(by=['intent', 'response'], inplace=True) | |
| # previous_row = None | |
| # for index, row in intents_df.iterrows(): | |
| # if previous_row is not None and previous_row['intent'] == row['intent']: | |
| # intent_index = intent_index + 1 | |
| # else: | |
| # intent_index = 0 | |
| # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f: | |
| # f.write(row['response']) | |
| # previous_row = row | |
| docs_dir = 'documents/' | |
| txts_dir = docs_dir + 'txt/' | |
| # converted_txts_dir = docs_dir + 'converted_txt/' | |
| # pdfs_dir = docs_dir + 'pdf/' | |
| # txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))] | |
| # pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))] | |
| # for pdf_doc in pdf_documents: | |
| # if pdf_doc not in txt_documents: | |
| # print("Converting " + pdf_doc + ' pdf to txt') | |
| # text = extract_text(pdfs_dir + pdf_doc + '.pdf') | |
| # with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f: | |
| # f.write(text) | |
| # intents_df = pd.read_csv(docs_dir + 'intents.csv') | |
| # intents_df.sort_values(by=['intent', 'response'], inplace=True) | |
| # previous_row = None | |
| # for index, row in intents_df.iterrows(): | |
| # if previous_row is not None and previous_row['intent'] == row['intent']: | |
| # intent_index = intent_index + 1 | |
| # else: | |
| # intent_index = 0 | |
| # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f: | |
| # f.write(row['response']) | |
| # previous_row = row | |
| loader = DirectoryLoader(txts_dir) | |
| txt_documents = loader.load() | |
| loader = UnstructuredURLLoader(urls=['https://sherlock-ph.readthedocs.io/en/latest/', | |
| 'https://tkmatrix.readthedocs.io/en/latest/', | |
| 'https://dearwatson.readthedocs.io/en/latest/', | |
| 'https://dearwatson.readthedocs.io/en/latest/installation.html', | |
| 'https://dearwatson.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/installation.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/the-pipeline.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/search-proc.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/toi-178-programmatical.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/README_270.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/README_540.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-search.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-search.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting2.html', | |
| 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-validation.html', | |
| 'https://github.com/franpoz/SHERLOCK', | |
| 'https://github.com/PlanetHunters/watson', | |
| 'https://github.com/PlanetHunters/tkmatrix', | |
| 'https://github.com/iaa-so-training/sherlock-tutorial/tree/main', | |
| 'https://github.com/franpoz/SHERLOCK/blob/master/sherlockpipe/properties.yaml']) | |
| url_documents = loader.load() | |
| def wrap_text_preserve_newlines(text, width=110): | |
| # Split the input text into lines based on newline characters | |
| lines = text.split('\n') | |
| # Wrap each line individually | |
| wrapped_lines = [textwrap.fill(line, width=width) for line in lines] | |
| # Join the wrapped lines back together using newline characters | |
| wrapped_text = '\n'.join(wrapped_lines) | |
| return wrapped_text | |
| max_length = 1000 | |
| text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10) | |
| docs = text_splitter.split_documents(itertools.chain(txt_documents, url_documents)) | |
| model_name = "decapoda-research/llama-7b-hf" | |
| model_name = "gpt-3.5-turbo" | |
| #model_name = "gpt2" | |
| #tokenizer = GPT2Tokenizer.from_pretrained(model_name) | |
| #tokenizer = tiktoken.get_encoding("gpt2") | |
| #assert tokenizer.decode(tokenizer.encode("hola mundo")) == "hola mundo" | |
| # tokenizer._pad_token = tokenizer.eos_token | |
| # tokenizer._pad_token_id = 50256 | |
| #model = GPT2LMHeadModel.from_pretrained(model_name) | |
| embeddings = OpenAIEmbeddings() | |
| vector_store = Chroma.from_documents(docs, embeddings) | |
| model_kwargs = {"max_length": max_length, "pad_token_id": 50256, "pad_token": "[PAD]", "padding": 'longest'} | |
| # llm = HuggingFaceHub(repo_id=model_name, model_kwargs={"temperature": 0.1, "max_length": 500, | |
| # "pad_token_id": 50256, "pad_token": "[PAD]", | |
| # "padding": 'longest' #for open end generation | |
| # }) | |
| llm = ChatOpenAI(streaming=True, model_name=model_name, temperature=0.9) | |
| prompt_template = """You are going to | |
| receive a context with information about related works, papers and tools for the PhD of Martín Dévora Pajares | |
| (exoplanets field), also appearing as M. Dévora-Pajares. | |
| You can only answer things based on the question and a context to answer it. | |
| Context: {context} | |
| Question: {question}""" | |
| #Memory: {chat_history}""" | |
| PROMPT = PromptTemplate( | |
| template=prompt_template, input_variables=["context", "question"] | |
| ) | |
| chain_type_kwargs = {"prompt": PROMPT} | |
| memory = ConversationBufferWindowMemory(k=4, memory_key="chat_history", input_key="question", output_key='answer', | |
| return_messages=True) | |
| retriever = vector_store.as_retriever() | |
| # memory = VectorStoreRetrieverMemory( | |
| # memory_key="chat_history", | |
| # return_docs=False, | |
| # retriever=retriever, | |
| # return_messages=True, | |
| # ) | |
| qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, | |
| chain_type_kwargs=chain_type_kwargs, return_source_documents=True) | |
| #memory=memory) | |
| # qa = ConversationalRetrievalChain.from_llm( | |
| # llm=llm, retriever=retriever, memory=memory, | |
| # return_source_documents=True, | |
| # combine_docs_chain_kwargs=chain_type_kwargs) | |
| # while True: | |
| # query = input("Question: ") | |
| # #docs = vector_store.similarity_search(query) | |
| # #print(wrap_text_preserve_newlines(str(docs[0].page_content))) | |
| # response = qa({"query": query}) | |
| # print(f"Answer: {response['result']}") | |
| # print(f"Sources: {response['source_documents']}") | |
| def chat(message, history): | |
| result = qa({"query": message}) | |
| sources_list = [] | |
| for item in result['source_documents']: | |
| if 'http' in item.metadata['source']: | |
| sources_list = sources_list + [item.metadata['source']] | |
| sources_list = list(set(sources_list)) | |
| response = result['result'] | |
| if len(sources_list) > 0: | |
| response = response + '\n\nSources:\n' + '\n'.join(sources_list) | |
| yield response | |
| # print(f"Answer: {response['result']}") | |
| # print(f"Sources: {response['source_documents']}") | |
| gr.ChatInterface(chat, chatbot=gr.Chatbot(), | |
| textbox=gr.Textbox(placeholder="Ask me about SHERLOCK, MATRIX or WATSON!", container=False, scale=7), | |
| title="PlanetHunters informative chatbot", | |
| description="Ask anything related to SHERLOCK, MATRIX, WATSON and their related papers!", | |
| theme="soft", | |
| examples=["What is SHERLOCK?", "What is MATRIX?", "What are the foundational papers of SHERLOCK?", | |
| "How should I cite SHERLOCK?", "How should I cite MATRIX?"], | |
| cache_examples=False, | |
| retry_btn=None, | |
| undo_btn="Delete Previous", | |
| clear_btn="Clear").queue().launch() | |