mdevora commited on
Commit
6868171
·
1 Parent(s): 9541a7e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +54 -15
app.py CHANGED
@@ -1,22 +1,17 @@
 
1
  import os
2
- import pandas as pd
3
  import gradio as gr
4
  import openai
5
  from langchain.chains import RetrievalQA
6
- from langchain.document_loaders import TextLoader
7
  from langchain.document_loaders.directory import DirectoryLoader
8
- from langchain.text_splitter import CharacterTextSplitter, TokenTextSplitter
9
  import textwrap
10
- from langchain.embeddings import HuggingFaceEmbeddings, OpenAIEmbeddings
11
  from langchain.vectorstores import Chroma
12
- from langchain.chains.question_answering import load_qa_chain
13
- from langchain.llms import HuggingFaceHub, HuggingFacePipeline, OpenAI
14
  from langchain_core.callbacks import StreamingStdOutCallbackHandler
15
- from transformers import GPT2Tokenizer, GPT2LMHeadModel, pipeline, AutoTokenizer, LlamaTokenizer
16
- import tiktoken
17
  from langchain.prompts import PromptTemplate
18
  from langchain.chat_models import ChatOpenAI
19
- from pdfminer.high_level import extract_text
20
 
21
 
22
  os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
@@ -45,8 +40,53 @@ converted_txts_dir = docs_dir + 'converted_txt/'
45
  # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
46
  # f.write(row['response'])
47
  # previous_row = row
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  loader = DirectoryLoader(txts_dir)
49
- documents = loader.load()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  def wrap_text_preserve_newlines(text, width=110):
51
  # Split the input text into lines based on newline characters
52
  lines = text.split('\n')
@@ -58,9 +98,9 @@ def wrap_text_preserve_newlines(text, width=110):
58
 
59
  max_length = 1000
60
  text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10)
61
- docs = text_splitter.split_documents(documents)
62
 
63
- #model_name = "decapoda-research/llama-7b-hf"
64
  model_name = "gpt-3.5-turbo"
65
  #model_name = "gpt2"
66
  #tokenizer = GPT2Tokenizer.from_pretrained(model_name)
@@ -80,7 +120,6 @@ llm = ChatOpenAI(streaming=True, callbacks=[StreamingStdOutCallbackHandler()], m
80
  prompt_template = """I will ask about tools, techniques and discoveries about the exoplanets field. You are going to receive a context with information about
81
  related works for the PhD of Martín Dévora with his developments and referenced works. You can only answer things based on the context and if you find that the
82
  question can't be answered from the context, just respond that you don't know.
83
-
84
  Context: {context}
85
  Question: {question}"""
86
 
@@ -104,11 +143,11 @@ def chat(message, history):
104
  # print(f"Sources: {response['source_documents']}")
105
 
106
  gr.ChatInterface(chat, chatbot=gr.Chatbot(height=300),
107
- textbox=gr.Textbox(placeholder="Ask me!", container=False, scale=7),
108
  title="Ask",
109
  description="Ask",
110
  theme="soft",
111
- examples=["How are you?"],
112
  cache_examples=True,
113
  retry_btn=None,
114
  undo_btn="Delete Previous",
 
1
+ import itertools
2
  import os
 
3
  import gradio as gr
4
  import openai
5
  from langchain.chains import RetrievalQA
 
6
  from langchain.document_loaders.directory import DirectoryLoader
7
+ from langchain.text_splitter import CharacterTextSplitter
8
  import textwrap
9
+ from langchain.embeddings import OpenAIEmbeddings
10
  from langchain.vectorstores import Chroma
11
+ from langchain_community.document_loaders import UnstructuredURLLoader
 
12
  from langchain_core.callbacks import StreamingStdOutCallbackHandler
 
 
13
  from langchain.prompts import PromptTemplate
14
  from langchain.chat_models import ChatOpenAI
 
15
 
16
 
17
  os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
 
40
  # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
41
  # f.write(row['response'])
42
  # previous_row = row
43
+ docs_dir = 'documents/'
44
+ txts_dir = docs_dir + 'txt/'
45
+ # converted_txts_dir = docs_dir + 'converted_txt/'
46
+ # pdfs_dir = docs_dir + 'pdf/'
47
+ # txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
48
+ # pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
49
+ # for pdf_doc in pdf_documents:
50
+ # if pdf_doc not in txt_documents:
51
+ # print("Converting " + pdf_doc + ' pdf to txt')
52
+ # text = extract_text(pdfs_dir + pdf_doc + '.pdf')
53
+ # with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
54
+ # f.write(text)
55
+ # intents_df = pd.read_csv(docs_dir + 'intents.csv')
56
+ # intents_df.sort_values(by=['intent', 'response'], inplace=True)
57
+ # previous_row = None
58
+ # for index, row in intents_df.iterrows():
59
+ # if previous_row is not None and previous_row['intent'] == row['intent']:
60
+ # intent_index = intent_index + 1
61
+ # else:
62
+ # intent_index = 0
63
+ # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
64
+ # f.write(row['response'])
65
+ # previous_row = row
66
  loader = DirectoryLoader(txts_dir)
67
+ txt_documents = loader.load()
68
+ loader = UnstructuredURLLoader(urls=['https://sherlock-ph.readthedocs.io/en/latest/',
69
+ 'https://tkmatrix.readthedocs.io/en/latest/',
70
+ 'https://dearwatson.readthedocs.io/en/latest/',
71
+ 'https://dearwatson.readthedocs.io/en/latest/installation.html',
72
+ 'https://dearwatson.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
73
+ 'https://sherlock-ph.readthedocs.io/en/latest/installation.html',
74
+ 'https://sherlock-ph.readthedocs.io/en/latest/the-pipeline.html',
75
+ 'https://sherlock-ph.readthedocs.io/en/latest/search-proc.html',
76
+ 'https://sherlock-ph.readthedocs.io/en/latest/toi-178-programmatical.html',
77
+ 'https://sherlock-ph.readthedocs.io/en/latest/README_270.html',
78
+ 'https://sherlock-ph.readthedocs.io/en/latest/README_540.html',
79
+ 'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-search.html',
80
+ 'https://sherlock-ph.readthedocs.io/en/latest/TOI-175/TOI-175-vetting.html',
81
+ 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-search.html',
82
+ 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting.html',
83
+ 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-vetting2.html',
84
+ 'https://sherlock-ph.readthedocs.io/en/latest/TOI-1798/TOI-1798-validation.html',
85
+ 'https://github.com/franpoz/SHERLOCK',
86
+ 'https://github.com/PlanetHunters/watson',
87
+ 'https://github.com/PlanetHunters/tkmatrix',
88
+ 'https://github.com/iaa-so-training/sherlock-tutorial/tree/main'])
89
+ url_documents = loader.load()
90
  def wrap_text_preserve_newlines(text, width=110):
91
  # Split the input text into lines based on newline characters
92
  lines = text.split('\n')
 
98
 
99
  max_length = 1000
100
  text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10)
101
+ docs = text_splitter.split_documents(itertools.chain(txt_documents, url_documents))
102
 
103
+ model_name = "decapoda-research/llama-7b-hf"
104
  model_name = "gpt-3.5-turbo"
105
  #model_name = "gpt2"
106
  #tokenizer = GPT2Tokenizer.from_pretrained(model_name)
 
120
  prompt_template = """I will ask about tools, techniques and discoveries about the exoplanets field. You are going to receive a context with information about
121
  related works for the PhD of Martín Dévora with his developments and referenced works. You can only answer things based on the context and if you find that the
122
  question can't be answered from the context, just respond that you don't know.
 
123
  Context: {context}
124
  Question: {question}"""
125
 
 
143
  # print(f"Sources: {response['source_documents']}")
144
 
145
  gr.ChatInterface(chat, chatbot=gr.Chatbot(height=300),
146
+ textbox=gr.Textbox(placeholder="Ask me about Voutique!", container=False, scale=7),
147
  title="Ask",
148
  description="Ask",
149
  theme="soft",
150
+ examples=["What is SHERLOCK?", "What is MATRIX?", "What are the foundational papers of SHERLOCK?"],
151
  cache_examples=True,
152
  retry_btn=None,
153
  undo_btn="Delete Previous",