dipenvekaria commited on
Commit
c5aa90d
·
verified ·
1 Parent(s): ca01fcb

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +30 -23
app.py CHANGED
@@ -5,7 +5,7 @@ from datetime import datetime
5
  from typing import List
6
  import gradio as gr
7
  from dotenv import load_dotenv
8
- import pdfplumber
9
  from langchain_openai import OpenAIEmbeddings, ChatOpenAI
10
  from langchain.prompts import PromptTemplate
11
  from qdrant_client import QdrantClient
@@ -25,7 +25,7 @@ class PdfProcessor:
25
  timeout=60
26
  )
27
  self.collection_name = "document_embeddings"
28
- self._ensure_collection_exists() # Changed from _setup_collection
29
 
30
  def _ensure_collection_exists(self):
31
  """Ensure the Qdrant collection exists, creating it if necessary."""
@@ -36,13 +36,14 @@ class PdfProcessor:
36
  )
37
 
38
  def _extract_text(self, file_path):
39
- text = ""
40
- with pdfplumber.open(file_path) as pdf:
41
- for page in pdf.pages:
42
- text += page.extract_text() or ""
43
- return text
 
44
 
45
- def _chunk_text(self, text, max_chunk_size=1000, min_chunk_size=300, similarity_threshold=0.7):
46
  """Performs semantic chunking using embeddings and cosine similarity."""
47
 
48
  sentences = re.split(r'(?<=[.!?])\s+', text) # Split text into sentences
@@ -61,21 +62,28 @@ class PdfProcessor:
61
  # Compute similarity to previous sentence
62
  similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
63
 
64
- # Merge if similarity is high & under max token limit
65
- if similarity > similarity_threshold and chunk_token_length + len(sentence) <= max_chunk_size:
 
 
 
 
 
 
 
66
  chunk.append(sentence)
67
  chunk_embedding.append(embedding)
68
  chunk_token_length += len(sentence)
69
  else:
70
  # Finalize current chunk and start a new one
71
- if chunk_token_length >= min_chunk_size: # Ensure minimum chunk size
72
  chunks.append(" ".join(chunk))
73
-
74
  chunk = [sentence]
75
  chunk_embedding = [embedding]
76
  chunk_token_length = len(sentence)
77
 
78
- if chunk and chunk_token_length >= min_chunk_size: # Add the last chunk if it meets min size
 
79
  chunks.append(" ".join(chunk))
80
 
81
  # Deduplicate chunks
@@ -88,22 +96,21 @@ class PdfProcessor:
88
  return summary if summary.endswith('.') else summary + '.'
89
 
90
  def process_and_store_pdf(self, file_obj):
91
- self._ensure_collection_exists() # Ensure collection exists before upsert
92
  text = self._extract_text(file_obj.name)
93
  filename = os.path.basename(file_obj.name)
94
  chunks = self._chunk_text(text)
95
  points = []
96
- batch_size = 100 # Adjust based on OpenAI token limits
97
 
98
- # Use a set to track unique chunk texts within this method (optional redundancy)
99
  seen_chunks = set()
100
 
101
  for i in range(0, len(chunks), batch_size):
102
  batch_chunks = chunks[i:i + batch_size]
103
- embeddings = self.embeddings.embed_documents(batch_chunks) # Batch embedding
104
 
105
  for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
106
- if chunk in seen_chunks: # Skip if chunk is a duplicate
107
  continue
108
  seen_chunks.add(chunk)
109
  point_id = str(uuid.uuid4())
@@ -125,12 +132,12 @@ class PdfProcessor:
125
 
126
  if points:
127
  self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
128
- points = [] # Clear points after upsert
129
 
130
  return filename
131
 
132
  def get_processed_files(self):
133
- self._ensure_collection_exists() # Ensure collection exists before scroll
134
  processed_files = set()
135
  scroll_response = self.qdrant_client.scroll(
136
  collection_name=self.collection_name,
@@ -155,7 +162,7 @@ class PdfProcessor:
155
  return processed_files
156
 
157
  def delete_file_points(self, filename: str):
158
- self._ensure_collection_exists() # Ensure collection exists before delete
159
  filter_condition = models.Filter(
160
  must=[
161
  models.FieldCondition(
@@ -177,7 +184,7 @@ def query_vector_db(query: str) -> dict:
177
  url=os.getenv("QDRANT_URL"),
178
  api_key=os.getenv("QDRANT_API_KEY")
179
  )
180
- qdrant_client.ensure_collection_exists(collection_name="document_embeddings") # Ensure collection exists
181
  embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
182
  query_embedding = embeddings.embed_query(query)
183
  query_response = qdrant_client.query_points(
@@ -190,7 +197,7 @@ def query_vector_db(query: str) -> dict:
190
  if not search_results:
191
  return {"context": "No relevant information found.", "source": "None", "chunks": []}
192
  context = "\n\n".join(
193
- [result.payload.get("text", "No text available") for result in search_results] # Use full text
194
  )
195
  chunks = [result.payload.get("text", "No text available") for result in search_results]
196
  source = search_results[0].payload.get("original_file_name", "Unknown")
 
5
  from typing import List
6
  import gradio as gr
7
  from dotenv import load_dotenv
8
+ from docling.document import DoclingDocument # Import Docling
9
  from langchain_openai import OpenAIEmbeddings, ChatOpenAI
10
  from langchain.prompts import PromptTemplate
11
  from qdrant_client import QdrantClient
 
25
  timeout=60
26
  )
27
  self.collection_name = "document_embeddings"
28
+ self._ensure_collection_exists()
29
 
30
  def _ensure_collection_exists(self):
31
  """Ensure the Qdrant collection exists, creating it if necessary."""
 
36
  )
37
 
38
  def _extract_text(self, file_path):
39
+ """Extract text from a PDF using Docling."""
40
+ # Load and parse the PDF with Docling
41
+ doc = DoclingDocument.from_file(file_path)
42
+ # Extract text from the document (assuming Docling provides a text representation)
43
+ text = doc.text # Adjust based on actual Docling API; this might need tweaking
44
+ return text if text else ""
45
 
46
+ def _chunk_text(self, text, max_chunk_size=500, min_chunk_size=100, similarity_threshold=0.5):
47
  """Performs semantic chunking using embeddings and cosine similarity."""
48
 
49
  sentences = re.split(r'(?<=[.!?])\s+', text) # Split text into sentences
 
62
  # Compute similarity to previous sentence
63
  similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
64
 
65
+ # If adding the sentence exceeds max_chunk_size, finalize the current chunk
66
+ if chunk_token_length + len(sentence) > max_chunk_size:
67
+ if chunk_token_length >= min_chunk_size:
68
+ chunks.append(" ".join(chunk))
69
+ chunk = [sentence]
70
+ chunk_embedding = [embedding]
71
+ chunk_token_length = len(sentence)
72
+ # Merge if similarity is high and under max token limit
73
+ elif similarity > similarity_threshold:
74
  chunk.append(sentence)
75
  chunk_embedding.append(embedding)
76
  chunk_token_length += len(sentence)
77
  else:
78
  # Finalize current chunk and start a new one
79
+ if chunk_token_length >= min_chunk_size:
80
  chunks.append(" ".join(chunk))
 
81
  chunk = [sentence]
82
  chunk_embedding = [embedding]
83
  chunk_token_length = len(sentence)
84
 
85
+ # Add the last chunk if it meets min size or is non-empty
86
+ if chunk and (chunk_token_length >= min_chunk_size or len(chunks) == 0):
87
  chunks.append(" ".join(chunk))
88
 
89
  # Deduplicate chunks
 
96
  return summary if summary.endswith('.') else summary + '.'
97
 
98
  def process_and_store_pdf(self, file_obj):
99
+ self._ensure_collection_exists()
100
  text = self._extract_text(file_obj.name)
101
  filename = os.path.basename(file_obj.name)
102
  chunks = self._chunk_text(text)
103
  points = []
104
+ batch_size = 100
105
 
 
106
  seen_chunks = set()
107
 
108
  for i in range(0, len(chunks), batch_size):
109
  batch_chunks = chunks[i:i + batch_size]
110
+ embeddings = self.embeddings.embed_documents(batch_chunks)
111
 
112
  for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
113
+ if chunk in seen_chunks:
114
  continue
115
  seen_chunks.add(chunk)
116
  point_id = str(uuid.uuid4())
 
132
 
133
  if points:
134
  self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
135
+ points = []
136
 
137
  return filename
138
 
139
  def get_processed_files(self):
140
+ self._ensure_collection_exists()
141
  processed_files = set()
142
  scroll_response = self.qdrant_client.scroll(
143
  collection_name=self.collection_name,
 
162
  return processed_files
163
 
164
  def delete_file_points(self, filename: str):
165
+ self._ensure_collection_exists()
166
  filter_condition = models.Filter(
167
  must=[
168
  models.FieldCondition(
 
184
  url=os.getenv("QDRANT_URL"),
185
  api_key=os.getenv("QDRANT_API_KEY")
186
  )
187
+ qdrant_client.ensure_collection_exists(collection_name="document_embeddings")
188
  embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
189
  query_embedding = embeddings.embed_query(query)
190
  query_response = qdrant_client.query_points(
 
197
  if not search_results:
198
  return {"context": "No relevant information found.", "source": "None", "chunks": []}
199
  context = "\n\n".join(
200
+ [result.payload.get("text", "No text available") for result in search_results]
201
  )
202
  chunks = [result.payload.get("text", "No text available") for result in search_results]
203
  source = search_results[0].payload.get("original_file_name", "Unknown")