dipenvekaria commited on
Commit
fc6e95c
·
verified ·
1 Parent(s): e0026c9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +75 -7
app.py CHANGED
@@ -87,11 +87,27 @@ class PdfProcessor:
87
  summary = '. '.join(sentences[:2]) if len(sentences) >= 2 else text[:100]
88
  return summary if summary.endswith('.') else summary + '.'
89
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
  def process_and_store_pdf(self, file_obj):
91
  self._ensure_collection_exists()
92
  text = self._extract_text(file_obj.name)
93
  filename = os.path.basename(file_obj.name)
94
  chunks = self._chunk_text(text)
 
95
  points = []
96
  batch_size = 100
97
  seen_chunks = set()
@@ -116,7 +132,8 @@ class PdfProcessor:
116
  "chunk_id": i + j,
117
  "summary": summary,
118
  "original_file_name": filename,
119
- "conversion_date": datetime.now().isoformat()
 
120
  }
121
  )
122
  )
@@ -127,6 +144,26 @@ class PdfProcessor:
127
 
128
  return filename
129
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  def get_processed_files(self):
131
  self._ensure_collection_exists()
132
  processed_files = set()
@@ -222,7 +259,7 @@ Answer: """
222
  final_answer = f"{answer} [Source: {tool_output['source']}]"
223
  return {"answer": final_answer, "chunks": tool_output["chunks"]}
224
 
225
- def delete_file(selected_files, processor):
226
  if selected_files and len(selected_files) > 0:
227
  selected_file = selected_files[0]
228
  processor.delete_file_points(selected_file)
@@ -230,6 +267,24 @@ def delete_file(selected_files, processor):
230
  return f"Deleted: {selected_file}", updated_choices
231
  return "No file selected", sorted(processor.get_processed_files())
232
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
  processor = PdfProcessor()
234
 
235
  def chat_function(message, history):
@@ -336,14 +391,21 @@ with gr.Blocks(theme=modern_theme, css=custom_css) as demo:
336
  with gr.Accordion("Manage Documents", open=False):
337
  files_selection = gr.CheckboxGroup(
338
  choices=sorted(processor.get_processed_files()),
339
- label="Document Selection (Select one to delete)",
340
  interactive=True,
341
  elem_classes="info-box checkbox-group"
342
  )
343
- delete_btn = gr.Button(
344
- "Delete Selected Document",
345
- size="sm"
346
- )
 
 
 
 
 
 
 
347
  chunks_display = gr.Textbox(
348
  value="Query results will appear here",
349
  label="Retrieved Chunks",
@@ -386,6 +448,12 @@ with gr.Blocks(theme=modern_theme, css=custom_css) as demo:
386
  outputs=[files_selection]
387
  )
388
 
 
 
 
 
 
 
389
  delete_btn.click(
390
  delete_file,
391
  inputs=[files_selection],
 
87
  summary = '. '.join(sentences[:2]) if len(sentences) >= 2 else text[:100]
88
  return summary if summary.endswith('.') else summary + '.'
89
 
90
+ def _generate_full_summary(self, text: str) -> str:
91
+ """Generate a 5-point summary for the entire document."""
92
+ llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
93
+ prompt = PromptTemplate(
94
+ input_variables=["text"],
95
+ template="""Generate a concise 5-point summary of the following text in bullet point format:
96
+
97
+ {text}
98
+
99
+ Summary:"""
100
+ )
101
+ chain = prompt | llm
102
+ summary = chain.invoke({"text": text[:10000]}).content.strip() # Limit text to avoid token limits
103
+ return summary
104
+
105
  def process_and_store_pdf(self, file_obj):
106
  self._ensure_collection_exists()
107
  text = self._extract_text(file_obj.name)
108
  filename = os.path.basename(file_obj.name)
109
  chunks = self._chunk_text(text)
110
+ full_summary = self._generate_full_summary(text)
111
  points = []
112
  batch_size = 100
113
  seen_chunks = set()
 
132
  "chunk_id": i + j,
133
  "summary": summary,
134
  "original_file_name": filename,
135
+ "conversion_date": datetime.now().isoformat(),
136
+ "full_summary": full_summary
137
  }
138
  )
139
  )
 
144
 
145
  return filename
146
 
147
+ def get_document_summary(self, filename: str) -> str:
148
+ """Retrieve the full summary for a specific document."""
149
+ scroll_response = self.qdrant_client.scroll(
150
+ collection_name=self.collection_name,
151
+ scroll_filter=models.Filter(
152
+ must=[
153
+ models.FieldCondition(
154
+ key="original_file_name",
155
+ match=models.MatchValue(value=filename)
156
+ )
157
+ ]
158
+ ),
159
+ limit=1,
160
+ with_payload=True
161
+ )
162
+ points, _ = scroll_response
163
+ if points and "full_summary" in points[0].payload:
164
+ return points[0].payload["full_summary"]
165
+ return "No summary available."
166
+
167
  def get_processed_files(self):
168
  self._ensure_collection_exists()
169
  processed_files = set()
 
259
  final_answer = f"{answer} [Source: {tool_output['source']}]"
260
  return {"answer": final_answer, "chunks": tool_output["chunks"]}
261
 
262
+ def delete_file(selected_files):
263
  if selected_files and len(selected_files) > 0:
264
  selected_file = selected_files[0]
265
  processor.delete_file_points(selected_file)
 
267
  return f"Deleted: {selected_file}", updated_choices
268
  return "No file selected", sorted(processor.get_processed_files())
269
 
270
+ def summarize_document(selected_files, history):
271
+ if not selected_files or len(selected_files) == 0:
272
+ return history, "No file selected for summary"
273
+
274
+ selected_file = selected_files[0]
275
+ summary = processor.get_document_summary(selected_file)
276
+
277
+ summary_message = f"""5 Point Summary of {selected_file}
278
+
279
+ {summary}
280
+
281
+ If you need specific information, type your question in the box below and click Send."""
282
+
283
+ new_history = history + [
284
+ {"role": "assistant", "content": summary_message}
285
+ ]
286
+ return new_history, f"Summarized: {selected_file}"
287
+
288
  processor = PdfProcessor()
289
 
290
  def chat_function(message, history):
 
391
  with gr.Accordion("Manage Documents", open=False):
392
  files_selection = gr.CheckboxGroup(
393
  choices=sorted(processor.get_processed_files()),
394
+ label="Document Selection (Select one)",
395
  interactive=True,
396
  elem_classes="info-box checkbox-group"
397
  )
398
+ with gr.Row():
399
+ summarize_btn = gr.Button(
400
+ "Summarize Document",
401
+ size="sm",
402
+ scale=1
403
+ )
404
+ delete_btn = gr.Button(
405
+ "Delete Document",
406
+ size="sm",
407
+ scale=1
408
+ )
409
  chunks_display = gr.Textbox(
410
  value="Query results will appear here",
411
  label="Retrieved Chunks",
 
448
  outputs=[files_selection]
449
  )
450
 
451
+ summarize_btn.click(
452
+ summarize_document,
453
+ inputs=[files_selection, chatbot],
454
+ outputs=[chatbot, upload_status]
455
+ )
456
+
457
  delete_btn.click(
458
  delete_file,
459
  inputs=[files_selection],