wholewhale commited on
Commit
a8e46d8
·
1 Parent(s): 1dcd36b
Files changed (1) hide show
  1. app.py +59 -37
app.py CHANGED
@@ -8,20 +8,16 @@ import gradio as gr
8
  import os
9
  from sklearn.neighbors import NearestNeighbors
10
 
11
- def sent_tokenize(text):
12
- # Simple sentence splitter based on regular expression.
13
- # This method is not as robust as nltk's sent_tokenize.
14
- sentences = re.split(r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?)\s', text)
15
- return sentences
16
-
17
  def download_pdf(url, output_path):
18
  urllib.request.urlretrieve(url, output_path)
19
 
 
20
  def preprocess(text):
21
  text = text.replace('\n', ' ')
22
  text = re.sub('\s+', ' ', text)
23
  return text
24
 
 
25
  def pdf_to_text(path, start_page=1, end_page=None):
26
  doc = fitz.open(path)
27
  total_pages = doc.page_count
@@ -39,30 +35,32 @@ def pdf_to_text(path, start_page=1, end_page=None):
39
  doc.close()
40
  return text_list
41
 
42
- def text_to_chunks(texts, num_sentences=5):
43
- chunks = []
44
-
45
- for text in texts:
46
- # Tokenize text into sentences
47
- sentences = sent_tokenize(text)
48
-
49
- # Calculate overlap
50
- overlap = int(0.2 * num_sentences)
51
-
52
- i = 0
53
- while i < len(sentences):
54
- chunk = sentences[i:i+num_sentences]
55
- chunks.append(' '.join(chunk))
56
- i += num_sentences - overlap
57
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  return chunks
59
 
 
60
  class SemanticSearch:
61
 
62
  def __init__(self):
63
  self.use = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')
64
  self.fitted = False
65
 
 
66
  def fit(self, data, batch=1000, n_neighbors=5):
67
  self.data = data
68
  self.embeddings = self.get_text_embedding(data, batch=batch)
@@ -71,6 +69,7 @@ class SemanticSearch:
71
  self.nn.fit(self.embeddings)
72
  self.fitted = True
73
 
 
74
  def __call__(self, text, return_data=True):
75
  inp_emb = self.use([text])
76
  neighbors = self.nn.kneighbors(inp_emb, return_distance=False)[0]
@@ -80,6 +79,7 @@ class SemanticSearch:
80
  else:
81
  return neighbors
82
 
 
83
  def get_text_embedding(self, texts, batch=1000):
84
  embeddings = []
85
  for i in range(0, len(texts), batch):
@@ -89,14 +89,17 @@ class SemanticSearch:
89
  embeddings = np.vstack(embeddings)
90
  return embeddings
91
 
 
 
92
  def load_recommender(path, start_page=1):
93
  global recommender
94
  texts = pdf_to_text(path, start_page=start_page)
95
- chunks = text_to_chunks(texts)
96
  recommender.fit(chunks)
97
  return 'Corpus Loaded.'
98
 
99
- def generate_text(prompt, engine="gpt-3.5-turbo-16k"):
 
100
  openai.api_key = os.getenv('YOUR_KEY_HERE')
101
  completions = openai.Completion.create(
102
  engine=engine,
@@ -106,50 +109,69 @@ def generate_text(prompt, engine="gpt-3.5-turbo-16k"):
106
  stop=None,
107
  temperature=0.7,
108
  )
109
- message = completions.choices[0].text.strip()
110
  return message
111
 
 
112
  def generate_answer(question):
113
  topn_chunks = recommender(question)
114
- prompt = 'search results:\n\n'
 
115
  for c in topn_chunks:
116
  prompt += c + '\n\n'
117
- prompt += ("Instructions: Compose a comprehensive reply to the query using the "
118
- "search results given. Cite each reference using [Page Number] notation. "
119
- "Only include information found in the results. The answer should be short and concise.\n\n")
 
 
 
 
 
 
 
120
  prompt += f"Query: {question}\nAnswer:"
121
- answer = generate_text(prompt)
122
  return answer
123
 
 
124
  def question_answer(url, file, question):
125
- if url.strip() == '' and file is None:
126
- return '[ERROR]: Both URL and PDF is empty. Provide at least one.'
127
- if url.strip() != '' and file is not None:
128
- return '[ERROR]: Both URL and PDF is provided. Please provide only one (either URL or PDF).'
 
 
129
  if url.strip() != '':
130
  glob_url = url
131
  download_pdf(glob_url, 'corpus.pdf')
132
  load_recommender('corpus.pdf')
 
133
  else:
134
  old_file_name = file.name
135
- file_name = old_file_name.replace(" ", "_")
 
136
  os.rename(old_file_name, file_name)
137
  load_recommender(file_name)
 
138
  if question.strip() == '':
139
  return '[ERROR]: Question field is empty'
 
140
  return generate_answer(question)
141
 
 
142
  recommender = SemanticSearch()
143
 
144
- title = 'PDF GPT'
145
- description = ("PDF GPT allows you to chat with your PDF file using Universal Sentence Encoder and Open AI. "
146
- "It provides coherent responses with page citations, helping you locate pertinent information quickly.")
147
 
148
  with gr.Blocks() as demo:
 
149
  gr.Markdown(f'<center><h1>{title}</h1></center>')
150
  gr.Markdown(description)
151
 
152
  with gr.Row():
 
153
  with gr.Group():
154
  url = gr.Textbox(label='Enter PDF URL here')
155
  gr.Markdown("<center><h4>OR<h4></center>")
 
8
  import os
9
  from sklearn.neighbors import NearestNeighbors
10
 
 
 
 
 
 
 
11
  def download_pdf(url, output_path):
12
  urllib.request.urlretrieve(url, output_path)
13
 
14
+
15
  def preprocess(text):
16
  text = text.replace('\n', ' ')
17
  text = re.sub('\s+', ' ', text)
18
  return text
19
 
20
+
21
  def pdf_to_text(path, start_page=1, end_page=None):
22
  doc = fitz.open(path)
23
  total_pages = doc.page_count
 
35
  doc.close()
36
  return text_list
37
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
 
39
+ def text_to_chunks(texts, word_length=150, start_page=1):
40
+ text_toks = [t.split(' ') for t in texts]
41
+ page_nums = []
42
+ chunks = []
43
+
44
+ for idx, words in enumerate(text_toks):
45
+ for i in range(0, len(words), word_length):
46
+ chunk = words[i:i+word_length]
47
+ if (i+word_length) > len(words) and (len(chunk) < word_length) and (
48
+ len(text_toks) != (idx+1)):
49
+ text_toks[idx+1] = chunk + text_toks[idx+1]
50
+ continue
51
+ chunk = ' '.join(chunk).strip()
52
+ chunk = f'[Page no. {idx+start_page}]' + ' ' + '"' + chunk + '"'
53
+ chunks.append(chunk)
54
  return chunks
55
 
56
+
57
  class SemanticSearch:
58
 
59
  def __init__(self):
60
  self.use = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')
61
  self.fitted = False
62
 
63
+
64
  def fit(self, data, batch=1000, n_neighbors=5):
65
  self.data = data
66
  self.embeddings = self.get_text_embedding(data, batch=batch)
 
69
  self.nn.fit(self.embeddings)
70
  self.fitted = True
71
 
72
+
73
  def __call__(self, text, return_data=True):
74
  inp_emb = self.use([text])
75
  neighbors = self.nn.kneighbors(inp_emb, return_distance=False)[0]
 
79
  else:
80
  return neighbors
81
 
82
+
83
  def get_text_embedding(self, texts, batch=1000):
84
  embeddings = []
85
  for i in range(0, len(texts), batch):
 
89
  embeddings = np.vstack(embeddings)
90
  return embeddings
91
 
92
+
93
+
94
  def load_recommender(path, start_page=1):
95
  global recommender
96
  texts = pdf_to_text(path, start_page=start_page)
97
+ chunks = text_to_chunks(texts, start_page=start_page)
98
  recommender.fit(chunks)
99
  return 'Corpus Loaded.'
100
 
101
+
102
+ def generate_text(prompt, engine="text-davinci-003"):
103
  openai.api_key = os.getenv('YOUR_KEY_HERE')
104
  completions = openai.Completion.create(
105
  engine=engine,
 
109
  stop=None,
110
  temperature=0.7,
111
  )
112
+ message = completions.choices[0].text
113
  return message
114
 
115
+
116
  def generate_answer(question):
117
  topn_chunks = recommender(question)
118
+ prompt = ""
119
+ prompt += 'search results:\n\n'
120
  for c in topn_chunks:
121
  prompt += c + '\n\n'
122
+
123
+ prompt += "Instructions: Compose a comprehensive reply to the query using the search results given. "\
124
+ "Cite each reference using [ Page Number] notation (every result has this number at the beginning). "\
125
+ "Citation should be done at the end of each sentence. If the search results mention multiple subjects "\
126
+ "with the same name, create separate answers for each. Only include information found in the results and "\
127
+ "don't add any additional information. Make sure the answer is correct and don't output false content. "\
128
+ "If the text does not relate to the query, simply state 'Found Nothing'. Ignore outlier "\
129
+ "search results which has nothing to do with the question. Only answer what is asked. The "\
130
+ "answer should be short and concise. \n\nQuery: {question}\nAnswer: "
131
+
132
  prompt += f"Query: {question}\nAnswer:"
133
+ answer = generate_text(prompt,"text-davinci-003")
134
  return answer
135
 
136
+
137
  def question_answer(url, file, question):
138
+ if url.strip() == '' and file == None:
139
+ return '[ERROR]: Both URL and PDF is empty. Provide atleast one.'
140
+
141
+ if url.strip() != '' and file != None:
142
+ return '[ERROR]: Both URL and PDF is provided. Please provide only one (eiter URL or PDF).'
143
+
144
  if url.strip() != '':
145
  glob_url = url
146
  download_pdf(glob_url, 'corpus.pdf')
147
  load_recommender('corpus.pdf')
148
+
149
  else:
150
  old_file_name = file.name
151
+ file_name = file.name
152
+ file_name = file_name[:-12] + file_name[-4:]
153
  os.rename(old_file_name, file_name)
154
  load_recommender(file_name)
155
+
156
  if question.strip() == '':
157
  return '[ERROR]: Question field is empty'
158
+
159
  return generate_answer(question)
160
 
161
+
162
  recommender = SemanticSearch()
163
 
164
+ title = 'CauseWriterPDF'
165
+ description = """ PDF GPT allows you to chat with your PDF file using Universal Sentence Encoder and Open AI. The returned response can even cite the page number in square brackets([]) where the information is located, adding credibility to the responses and helping to locate pertinent information quickly."""
166
+
167
 
168
  with gr.Blocks() as demo:
169
+
170
  gr.Markdown(f'<center><h1>{title}</h1></center>')
171
  gr.Markdown(description)
172
 
173
  with gr.Row():
174
+
175
  with gr.Group():
176
  url = gr.Textbox(label='Enter PDF URL here')
177
  gr.Markdown("<center><h4>OR<h4></center>")