Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -74,10 +74,12 @@ class PdfProcessor:
|
|
| 74 |
chunk_embedding = [embedding]
|
| 75 |
chunk_token_length = len(sentence)
|
| 76 |
|
| 77 |
-
if chunk:
|
| 78 |
-
chunks.append(" ".join(chunk))
|
| 79 |
|
| 80 |
-
|
|
|
|
|
|
|
| 81 |
|
| 82 |
def _generate_summary(self, text: str) -> str:
|
| 83 |
sentences = [s.strip() for s in text.split('.') if s.strip()]
|
|
@@ -91,11 +93,17 @@ class PdfProcessor:
|
|
| 91 |
points = []
|
| 92 |
batch_size = 100 # Adjust based on OpenAI token limits
|
| 93 |
|
|
|
|
|
|
|
|
|
|
| 94 |
for i in range(0, len(chunks), batch_size):
|
| 95 |
batch_chunks = chunks[i:i + batch_size]
|
| 96 |
embeddings = self.embeddings.embed_documents(batch_chunks) # Batch embedding
|
| 97 |
|
| 98 |
for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
|
|
|
|
|
|
|
|
|
|
| 99 |
point_id = str(uuid.uuid4())
|
| 100 |
summary = self._generate_summary(chunk)
|
| 101 |
points.append(
|
|
@@ -115,7 +123,7 @@ class PdfProcessor:
|
|
| 115 |
|
| 116 |
if points:
|
| 117 |
self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
|
| 118 |
-
points = []
|
| 119 |
|
| 120 |
return filename
|
| 121 |
|
|
@@ -176,9 +184,8 @@ def query_vector_db(query: str) -> dict:
|
|
| 176 |
search_results = query_response.points
|
| 177 |
if not search_results:
|
| 178 |
return {"context": "No relevant information found.", "source": "None", "chunks": []}
|
| 179 |
-
# Use full chunk text instead of summaries
|
| 180 |
context = "\n\n".join(
|
| 181 |
-
[result.payload.get("text", "No text available") for result in search_results]
|
| 182 |
)
|
| 183 |
chunks = [result.payload.get("text", "No text available") for result in search_results]
|
| 184 |
source = search_results[0].payload.get("original_file_name", "Unknown")
|
|
@@ -363,6 +370,7 @@ with gr.Blocks(theme=modern_theme, css=custom_css) as demo:
|
|
| 363 |
outputs=[upload_status]
|
| 364 |
).then(
|
| 365 |
lambda: gr.update(choices=sorted(processor.get_processed_files())),
|
|
|
|
| 366 |
outputs=[files_selection]
|
| 367 |
)
|
| 368 |
|
|
|
|
| 74 |
chunk_embedding = [embedding]
|
| 75 |
chunk_token_length = len(sentence)
|
| 76 |
|
| 77 |
+
if chunk and chunk_token_length >= min_chunk_size: # Add the last chunk if it meets min size
|
| 78 |
+
chunks.append(" ".join(chunk))
|
| 79 |
|
| 80 |
+
# Deduplicate chunks
|
| 81 |
+
unique_chunks = list(dict.fromkeys(chunks)) # Preserve order, remove duplicates
|
| 82 |
+
return unique_chunks
|
| 83 |
|
| 84 |
def _generate_summary(self, text: str) -> str:
|
| 85 |
sentences = [s.strip() for s in text.split('.') if s.strip()]
|
|
|
|
| 93 |
points = []
|
| 94 |
batch_size = 100 # Adjust based on OpenAI token limits
|
| 95 |
|
| 96 |
+
# Use a set to track unique chunk texts within this method (optional redundancy)
|
| 97 |
+
seen_chunks = set()
|
| 98 |
+
|
| 99 |
for i in range(0, len(chunks), batch_size):
|
| 100 |
batch_chunks = chunks[i:i + batch_size]
|
| 101 |
embeddings = self.embeddings.embed_documents(batch_chunks) # Batch embedding
|
| 102 |
|
| 103 |
for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
|
| 104 |
+
if chunk in seen_chunks: # Skip if chunk is a duplicate
|
| 105 |
+
continue
|
| 106 |
+
seen_chunks.add(chunk)
|
| 107 |
point_id = str(uuid.uuid4())
|
| 108 |
summary = self._generate_summary(chunk)
|
| 109 |
points.append(
|
|
|
|
| 123 |
|
| 124 |
if points:
|
| 125 |
self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
|
| 126 |
+
points = [] # Clear points after upsert
|
| 127 |
|
| 128 |
return filename
|
| 129 |
|
|
|
|
| 184 |
search_results = query_response.points
|
| 185 |
if not search_results:
|
| 186 |
return {"context": "No relevant information found.", "source": "None", "chunks": []}
|
|
|
|
| 187 |
context = "\n\n".join(
|
| 188 |
+
[result.payload.get("text", "No text available") for result in search_results] # Use full text
|
| 189 |
)
|
| 190 |
chunks = [result.payload.get("text", "No text available") for result in search_results]
|
| 191 |
source = search_results[0].payload.get("original_file_name", "Unknown")
|
|
|
|
| 370 |
outputs=[upload_status]
|
| 371 |
).then(
|
| 372 |
lambda: gr.update(choices=sorted(processor.get_processed_files())),
|
| 373 |
+
:sorted(processor.get_processed_files())),
|
| 374 |
outputs=[files_selection]
|
| 375 |
)
|
| 376 |
|