dipenvekaria commited on
Commit
bfa6208
·
verified ·
1 Parent(s): cf8a3db

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -478
app.py DELETED
@@ -1,478 +0,0 @@
1
- import os
2
- import uuid
3
- from datetime import datetime
4
- import gradio as gr
5
- from dotenv import load_dotenv
6
- import fitz # Import PyMuPDF as fitz
7
- print(f"PyMuPDF version: {fitz.__version__}") # Debug2
8
- from langchain_openai import OpenAIEmbeddings, ChatOpenAI
9
- from langchain.prompts import PromptTemplate
10
- from qdrant_client import QdrantClient
11
- from qdrant_client.http import models
12
- from sklearn.metrics.pairwise import cosine_similarity
13
- import re
14
- import time
15
-
16
- load_dotenv()
17
-
18
- class PdfProcessor:
19
- def __init__(self):
20
- self.embeddings = OpenAIEmbeddings(model="text-embedding-3-small", openai_api_key=os.getenv("OPENAI_API_KEY"))
21
- self.qdrant_client = QdrantClient(
22
- url=os.getenv("QDRANT_URL"),
23
- api_key=os.getenv("QDRANT_API_KEY"),
24
- timeout=60
25
- )
26
- self.collection_name = "document_embeddings"
27
- self._ensure_collection_exists()
28
-
29
- def _ensure_collection_exists(self):
30
- """Ensure the Qdrant collection exists, creating it if necessary."""
31
- if not self.qdrant_client.collection_exists(collection_name=self.collection_name):
32
- self.qdrant_client.create_collection(
33
- collection_name=self.collection_name,
34
- vectors_config=models.VectorParams(size=1536, distance=models.Distance.COSINE)
35
- )
36
- print(f"Created collection: {self.collection_name}") # Debug
37
-
38
- def _extract_text(self, file_path):
39
- """Extract text from a PDF using PyMuPDF."""
40
- text = ""
41
- with fitz.open(file_path) as pdf:
42
- for page in pdf:
43
- text += page.get_text("text") or "" # Extract text from each page
44
- return text
45
-
46
- def _chunk_text(self, text, max_chunk_size=500, min_chunk_size=100, similarity_threshold=0.5):
47
- """Performs semantic chunking using embeddings and cosine similarity."""
48
- sentences = re.split(r'(?<=[.!?])\s+', text)
49
- sentence_embeddings = self.embeddings.embed_documents(sentences)
50
- chunks, chunk, chunk_embedding = [], [], []
51
- chunk_token_length = 0
52
-
53
- for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
54
- if not chunk:
55
- chunk.append(sentence)
56
- chunk_embedding.append(embedding)
57
- chunk_token_length = len(sentence)
58
- continue
59
-
60
- similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
61
- if chunk_token_length + len(sentence) > max_chunk_size:
62
- if chunk_token_length >= min_chunk_size:
63
- chunks.append(" ".join(chunk))
64
- chunk = [sentence]
65
- chunk_embedding = [embedding]
66
- chunk_token_length = len(sentence)
67
- elif similarity > similarity_threshold:
68
- chunk.append(sentence)
69
- chunk_embedding.append(embedding)
70
- chunk_token_length += len(sentence)
71
- else:
72
- if chunk_token_length >= min_chunk_size:
73
- chunks.append(" ".join(chunk))
74
- chunk = [sentence]
75
- chunk_embedding = [embedding]
76
- chunk_token_length = len(sentence)
77
-
78
- if chunk and (chunk_token_length >= min_chunk_size or len(chunks) == 0):
79
- chunks.append(" ".join(chunk))
80
- unique_chunks = list(dict.fromkeys(chunks))
81
- return unique_chunks
82
-
83
- def _generate_summary(self, text: str) -> str:
84
- sentences = [s.strip() for s in text.split('.') if s.strip()]
85
- summary = '. '.join(sentences[:2]) if len(sentences) >= 2 else text[:100]
86
- return summary if summary.endswith('.') else summary + '.'
87
-
88
- def _generate_full_summary(self, text: str) -> str:
89
- """Generate a 5-point summary for the entire document."""
90
- llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
91
- prompt = PromptTemplate(
92
- input_variables=["text"],
93
- template="""Generate a concise 5-point summary of the following text in bullet point format:
94
-
95
- {text}
96
-
97
- Summary:"""
98
- )
99
- chain = prompt | llm
100
- summary = chain.invoke({"text": text[:10000]}).content.strip() # Limit text to avoid token limits
101
- return summary
102
-
103
- def process_and_store_pdf(self, file_obj):
104
- self._ensure_collection_exists()
105
- text = self._extract_text(file_obj.name)
106
- filename = os.path.basename(file_obj.name)
107
- chunks = self._chunk_text(text)
108
- full_summary = self._generate_full_summary(text)
109
- points = []
110
- batch_size = 100
111
- seen_chunks = set()
112
-
113
- for i in range(0, len(chunks), batch_size):
114
- batch_chunks = chunks[i:i + batch_size]
115
- embeddings = self.embeddings.embed_documents(batch_chunks)
116
-
117
- for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
118
- if chunk in seen_chunks:
119
- continue
120
- seen_chunks.add(chunk)
121
- point_id = str(uuid.uuid4())
122
- summary = self._generate_summary(chunk)
123
- points.append(
124
- models.PointStruct(
125
- id=point_id,
126
- vector=embedding,
127
- payload={
128
- "text": chunk,
129
- "source_file": filename,
130
- "chunk_id": i + j,
131
- "summary": summary,
132
- "original_file_name": filename,
133
- "conversion_date": datetime.now().isoformat(),
134
- "full_summary": full_summary
135
- }
136
- )
137
- )
138
-
139
- if points:
140
- self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
141
- points = []
142
-
143
- return filename
144
-
145
- def get_document_summary(self, filename: str) -> str:
146
- """Retrieve the full summary for a specific document."""
147
- scroll_response = self.qdrant_client.scroll(
148
- collection_name=self.collection_name,
149
- scroll_filter=models.Filter(
150
- must=[
151
- models.FieldCondition(
152
- key="original_file_name",
153
- match=models.MatchValue(value=filename)
154
- )
155
- ]
156
- ),
157
- limit=1,
158
- with_payload=True
159
- )
160
- points, _ = scroll_response
161
- if points and "full_summary" in points[0].payload:
162
- return points[0].payload["full_summary"]
163
- return "No summary available."
164
-
165
- def get_processed_files(self):
166
- processed_files=()
167
- self._ensure_collection_exists()
168
- processed_files = set()
169
-
170
- scroll_response = self.qdrant_client.scroll(
171
- collection_name=self.collection_name,
172
- limit=100,
173
- with_payload=True
174
- )
175
- points, next_offset = scroll_response
176
- while points:
177
- for point in points:
178
- original_file_name = point.payload.get("original_file_name")
179
- if original_file_name:
180
- processed_files.add(original_file_name)
181
- if next_offset is None:
182
- break
183
- scroll_response = self.qdrant_client.scroll(
184
- collection_name=self.collection_name,
185
- limit=100,
186
- offset=next_offset,
187
- with_payload=True
188
- )
189
- points, next_offset = scroll_response
190
- return processed_files
191
-
192
- def delete_file_points(self, filename: str):
193
- self._ensure_collection_exists()
194
- filter_condition = models.Filter(
195
- must=[
196
- models.FieldCondition(
197
- key="original_file_name",
198
- match=models.MatchValue(value=filename)
199
- )
200
- ]
201
- )
202
- self.qdrant_client.delete(
203
- collection_name=self.collection_name,
204
- points_selector=models.FilterSelector(filter=filter_condition)
205
- )
206
- time.sleep(0.5) # Small delay to ensure Qdrant processes the deletion
207
- return f"Deleted points for file: {filename}"
208
-
209
- def query_vector_db(query: str) -> dict:
210
- qdrant_client = QdrantClient(
211
- url=os.getenv("QDRANT_URL"),
212
- api_key=os.getenv("QDRANT_API_KEY")
213
- )
214
- if not qdrant_client.collection_exists(collection_name="document_embeddings"):
215
- qdrant_client.create_collection(
216
- collection_name="document_embeddings",
217
- vectors_config=models.VectorParams(size=1536, distance=models.Distance.COSINE)
218
- )
219
- print("Created collection 'document_embeddings' in query_vector_db")
220
- embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
221
- query_embedding = embeddings.embed_query(query)
222
- query_response = qdrant_client.query_points(
223
- collection_name="document_embeddings",
224
- query=query_embedding,
225
- limit=5,
226
- with_payload=True
227
- )
228
- search_results = query_response.points
229
- if not search_results:
230
- return {"context": "No relevant information found.", "source": "None", "chunks": []}
231
- context = "\n\n".join(
232
- [result.payload.get("text", "No text available") for result in search_results]
233
- )
234
- chunks = [result.payload.get("text", "No text available") for result in search_results]
235
- source = search_results[0].payload.get("original_file_name", "Unknown")
236
- return {"context": context, "source": source, "chunks": chunks}
237
-
238
- def classify_relevant_file(question, processor):
239
- processed_files = processor.get_processed_files()
240
- return list(processed_files)[0] if processed_files else None
241
-
242
- def query_documents(question, processor):
243
- relevant_file = classify_relevant_file(question, processor)
244
- tool_output = query_vector_db(query=question)
245
- answer_prompt = PromptTemplate(
246
- input_variables=["question", "context"],
247
- template="""Based solely on the provided context, provide a concise, accurate answer to the question. Do not use external knowledge.
248
-
249
- Question: {question}
250
- Context: {context}
251
-
252
- Answer: """
253
- )
254
- llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
255
- chain = answer_prompt | llm
256
- answer = chain.invoke({"question": question, "context": tool_output["context"]}).content.strip()
257
- if "context does not" in answer:
258
- final_answer = answer
259
- else:
260
- final_answer = f"{answer} [Source: {tool_output['source']}]"
261
- return {"answer": final_answer, "chunks": tool_output["chunks"]}
262
-
263
- def delete_file(selected_files):
264
- if selected_files and len(selected_files) > 0:
265
- selected_file = selected_files[0]
266
- processor.delete_file_points(selected_file)
267
- updated_choices = sorted(processor.get_processed_files())
268
- return f"Deleted: {selected_file}", updated_choices
269
- return "No file selected", sorted(processor.get_processed_files())
270
-
271
- def summarize_document(selected_files, history):
272
- if not selected_files or len(selected_files) == 0:
273
- return history, "No file selected for summary"
274
-
275
- selected_file = selected_files[0]
276
- summary = processor.get_document_summary(selected_file)
277
-
278
- summary_message = f"""5 Point Summary of {selected_file}
279
-
280
- {summary}
281
-
282
- If you need specific information, type your question in the box below and click Send."""
283
-
284
- new_history = history + [
285
- {"role": "assistant", "content": summary_message}
286
- ]
287
- return new_history, f"Summarized: {selected_file}"
288
-
289
- processor = PdfProcessor()
290
-
291
- def chat_function(message, history):
292
- response = query_documents(message, processor)
293
- answer = response["answer"]
294
- chunks = "\n\n".join([f"Chunk {i+1}: {chunk}" for i, chunk in enumerate(response["chunks"])])
295
- new_history = history + [
296
- {"role": "user", "content": message},
297
- {"role": "assistant", "content": answer}
298
- ]
299
- return new_history, "", chunks
300
-
301
- def upload_file(file):
302
- if file is None:
303
- return "Please upload a PDF file"
304
- filename = processor.process_and_store_pdf(file)
305
- return f"Processed: {filename}"
306
-
307
- modern_theme = gr.themes.Monochrome(
308
- primary_hue="indigo",
309
- secondary_hue="gray",
310
- neutral_hue="slate",
311
- radius_size="lg",
312
- text_size="md",
313
- )
314
-
315
- custom_css = """
316
- @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap');
317
- * { font-family: 'Inter', sans-serif !important; }
318
- .container { max-width: 1200px; margin: 0 auto; }
319
- .header { text-align: center; padding: 1rem 0; }
320
- .chatbot .message { border-radius: 10px; padding: 10px; margin: 5px 0; }
321
- .user-message { background-color: #E94560 !important; color: white; }
322
- .assistant-message { background-color: #0F3460 !important; color: white; }
323
- .message-input { background-color: #0F3460; color: white; border: 1px solid #E94560; border-radius: 8px; }
324
- .message-input textarea { height: 60px !important; }
325
- .send-btn { background-color: #E94560; color: white; border: none; height: 60px !important; display: flex; align-items: center; justify-content: center; font-size: 18px !important; font-weight: 500; }
326
- .send-btn:hover { background-color: #FF6B6B; }
327
- .file-upload { background-color: #16213E; border: 1px dashed #E94560; border-radius: 8px; height: 200px; }
328
- .status-text { color: #E94560; font-size: 14px; text-align: center; }
329
- .info-box { background-color: #16213E; color: white; border: 1px solid #0F3460; border-radius: 8px; }
330
- .gr-button { transition: all 0.3s ease; }
331
- .gr-accordion { background-color: #16213E; border: 1px solid #0F3460; }
332
- .square-column { min-width: 300px; max-width: 400px; }
333
- .checkbox-group { max-height: 150px; overflow-y: auto; }
334
- .gr-accordion > div > button { font-size: 18px !important; font-weight: 500; }
335
- .header-text { color: white; font-size: 16px; margin-top: -10px; }
336
- .warning-text { color: #E94560; font-size: 16px; margin-top: -10px; }
337
- """
338
-
339
- with gr.Blocks(theme=modern_theme, css=custom_css) as demo:
340
- gr.Markdown(
341
- """
342
- # Zaplytic
343
- <p style='color: #E94560; font-size: 16px; margin-top: -10px;'>Upload your document to store it in a vector database and chat with it to get instant answers.</p>
344
- <p style='color: red; font-size: 16px; margin-top: -10px;'><b style='color: red;'>Do not upload personal or private data</b></p>
345
- """,
346
- elem_classes="header"
347
- )
348
-
349
- with gr.Row(elem_classes="container"):
350
- with gr.Column(scale=7):
351
- chatbot = gr.Chatbot(
352
- label=None,
353
- type="messages",
354
- height=500,
355
- elem_classes="info-box"
356
- )
357
- with gr.Row():
358
- msg = gr.Textbox(
359
- placeholder="Ask anything...",
360
- show_label=False,
361
- container=False,
362
- elem_classes="message-input",
363
- lines=3,
364
- scale=8
365
- )
366
- send_btn = gr.Button(
367
- "Send",
368
- size="sm",
369
- elem_classes="send-btn",
370
- scale=2
371
- )
372
-
373
- with gr.Column(scale=3):
374
- with gr.Accordion("File Upload", open=False):
375
- file_upload = gr.File(
376
- label="Drop PDF here",
377
- file_types=[".pdf"],
378
- elem_classes="file-upload",
379
- height=200
380
- )
381
- upload_btn = gr.Button(
382
- "Upload PDF",
383
- size="sm"
384
- )
385
- upload_status = gr.Textbox(
386
- value="Upload a PDF to begin",
387
- show_label=False,
388
- interactive=False,
389
- elem_classes="status-text"
390
- )
391
-
392
- with gr.Accordion("Manage Documents", open=False):
393
- files_selection = gr.CheckboxGroup(
394
- choices=sorted(processor.get_processed_files()),
395
- label="Document Selection (Select one)",
396
- interactive=True,
397
- elem_classes="info-box checkbox-group"
398
- )
399
- with gr.Row():
400
- summarize_btn = gr.Button(
401
- "Summarize Document",
402
- size="sm",
403
- scale=1
404
- )
405
- delete_btn = gr.Button(
406
- "Delete Document",
407
- size="sm",
408
- scale=1
409
- )
410
- chunks_display = gr.Textbox(
411
- value="Query results will appear here",
412
- label="Retrieved Chunks",
413
- interactive=False,
414
- lines=5,
415
- max_lines=5,
416
- elem_classes="info-box"
417
- )
418
-
419
- # Add footer with link to your website
420
- gr.Markdown(
421
- """
422
- ---
423
- Interested in more of my thoughts and projects? [Click here](https://dipenvekaria.com) to visit my website!
424
- """,
425
- elem_classes="footer"
426
- )
427
-
428
- def update_selection(selected_files):
429
- if len(selected_files) > 1:
430
- return [selected_files[-1]]
431
- return selected_files
432
-
433
- send_btn.click(
434
- chat_function,
435
- inputs=[msg, chatbot],
436
- outputs=[chatbot, msg, chunks_display]
437
- )
438
- msg.submit(
439
- chat_function,
440
- inputs=[msg, chatbot],
441
- outputs=[chatbot, msg, chunks_display]
442
- )
443
- upload_btn.click(
444
- upload_file,
445
- inputs=[file_upload],
446
- outputs=[upload_status]
447
- ).then(
448
- lambda: gr.update(choices=sorted(processor.get_processed_files())),
449
- outputs=[files_selection]
450
- ).then(
451
- lambda: [], # Clear selection after upload
452
- outputs=[files_selection]
453
- )
454
-
455
- files_selection.change(
456
- update_selection,
457
- inputs=[files_selection],
458
- outputs=[files_selection]
459
- )
460
-
461
- summarize_btn.click(
462
- summarize_document,
463
- inputs=[files_selection, chatbot],
464
- outputs=[chatbot, upload_status]
465
- )
466
-
467
- delete_btn.click(
468
- delete_file,
469
- inputs=[files_selection],
470
- outputs=[upload_status, files_selection]
471
- ).then(
472
- lambda: [], # Clear selection after deletion
473
- outputs=[files_selection]
474
- )
475
-
476
- if __name__ == "__main__":
477
- print(f"===== Application Startup at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} =====")
478
- demo.launch(server_name="0.0.0.0", server_port=7860, show_api=False)