dipenvekaria commited on
Commit
2ddcc25
·
verified ·
1 Parent(s): d5c3acb

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +14 -6
app.py CHANGED
@@ -74,10 +74,12 @@ class PdfProcessor:
74
  chunk_embedding = [embedding]
75
  chunk_token_length = len(sentence)
76
 
77
- if chunk:
78
- chunks.append(" ".join(chunk)) # Add the last chunk
79
 
80
- return chunks
 
 
81
 
82
  def _generate_summary(self, text: str) -> str:
83
  sentences = [s.strip() for s in text.split('.') if s.strip()]
@@ -91,11 +93,17 @@ class PdfProcessor:
91
  points = []
92
  batch_size = 100 # Adjust based on OpenAI token limits
93
 
 
 
 
94
  for i in range(0, len(chunks), batch_size):
95
  batch_chunks = chunks[i:i + batch_size]
96
  embeddings = self.embeddings.embed_documents(batch_chunks) # Batch embedding
97
 
98
  for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
 
 
 
99
  point_id = str(uuid.uuid4())
100
  summary = self._generate_summary(chunk)
101
  points.append(
@@ -115,7 +123,7 @@ class PdfProcessor:
115
 
116
  if points:
117
  self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
118
- points = []
119
 
120
  return filename
121
 
@@ -176,9 +184,8 @@ def query_vector_db(query: str) -> dict:
176
  search_results = query_response.points
177
  if not search_results:
178
  return {"context": "No relevant information found.", "source": "None", "chunks": []}
179
- # Use full chunk text instead of summaries
180
  context = "\n\n".join(
181
- [result.payload.get("text", "No text available") for result in search_results]
182
  )
183
  chunks = [result.payload.get("text", "No text available") for result in search_results]
184
  source = search_results[0].payload.get("original_file_name", "Unknown")
@@ -363,6 +370,7 @@ with gr.Blocks(theme=modern_theme, css=custom_css) as demo:
363
  outputs=[upload_status]
364
  ).then(
365
  lambda: gr.update(choices=sorted(processor.get_processed_files())),
 
366
  outputs=[files_selection]
367
  )
368
 
 
74
  chunk_embedding = [embedding]
75
  chunk_token_length = len(sentence)
76
 
77
+ if chunk and chunk_token_length >= min_chunk_size: # Add the last chunk if it meets min size
78
+ chunks.append(" ".join(chunk))
79
 
80
+ # Deduplicate chunks
81
+ unique_chunks = list(dict.fromkeys(chunks)) # Preserve order, remove duplicates
82
+ return unique_chunks
83
 
84
  def _generate_summary(self, text: str) -> str:
85
  sentences = [s.strip() for s in text.split('.') if s.strip()]
 
93
  points = []
94
  batch_size = 100 # Adjust based on OpenAI token limits
95
 
96
+ # Use a set to track unique chunk texts within this method (optional redundancy)
97
+ seen_chunks = set()
98
+
99
  for i in range(0, len(chunks), batch_size):
100
  batch_chunks = chunks[i:i + batch_size]
101
  embeddings = self.embeddings.embed_documents(batch_chunks) # Batch embedding
102
 
103
  for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
104
+ if chunk in seen_chunks: # Skip if chunk is a duplicate
105
+ continue
106
+ seen_chunks.add(chunk)
107
  point_id = str(uuid.uuid4())
108
  summary = self._generate_summary(chunk)
109
  points.append(
 
123
 
124
  if points:
125
  self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
126
+ points = [] # Clear points after upsert
127
 
128
  return filename
129
 
 
184
  search_results = query_response.points
185
  if not search_results:
186
  return {"context": "No relevant information found.", "source": "None", "chunks": []}
 
187
  context = "\n\n".join(
188
+ [result.payload.get("text", "No text available") for result in search_results] # Use full text
189
  )
190
  chunks = [result.payload.get("text", "No text available") for result in search_results]
191
  source = search_results[0].payload.get("original_file_name", "Unknown")
 
370
  outputs=[upload_status]
371
  ).then(
372
  lambda: gr.update(choices=sorted(processor.get_processed_files())),
373
+ :sorted(processor.get_processed_files())),
374
  outputs=[files_selection]
375
  )
376