Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -5,7 +5,7 @@ from datetime import datetime
|
|
| 5 |
from typing import List
|
| 6 |
import gradio as gr
|
| 7 |
from dotenv import load_dotenv
|
| 8 |
-
import
|
| 9 |
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
|
| 10 |
from langchain.prompts import PromptTemplate
|
| 11 |
from qdrant_client import QdrantClient
|
|
@@ -25,7 +25,7 @@ class PdfProcessor:
|
|
| 25 |
timeout=60
|
| 26 |
)
|
| 27 |
self.collection_name = "document_embeddings"
|
| 28 |
-
self._ensure_collection_exists()
|
| 29 |
|
| 30 |
def _ensure_collection_exists(self):
|
| 31 |
"""Ensure the Qdrant collection exists, creating it if necessary."""
|
|
@@ -36,13 +36,14 @@ class PdfProcessor:
|
|
| 36 |
)
|
| 37 |
|
| 38 |
def _extract_text(self, file_path):
|
| 39 |
-
text
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
|
|
|
| 44 |
|
| 45 |
-
def _chunk_text(self, text, max_chunk_size=
|
| 46 |
"""Performs semantic chunking using embeddings and cosine similarity."""
|
| 47 |
|
| 48 |
sentences = re.split(r'(?<=[.!?])\s+', text) # Split text into sentences
|
|
@@ -61,21 +62,28 @@ class PdfProcessor:
|
|
| 61 |
# Compute similarity to previous sentence
|
| 62 |
similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
|
| 63 |
|
| 64 |
-
#
|
| 65 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
chunk.append(sentence)
|
| 67 |
chunk_embedding.append(embedding)
|
| 68 |
chunk_token_length += len(sentence)
|
| 69 |
else:
|
| 70 |
# Finalize current chunk and start a new one
|
| 71 |
-
if chunk_token_length >= min_chunk_size:
|
| 72 |
chunks.append(" ".join(chunk))
|
| 73 |
-
|
| 74 |
chunk = [sentence]
|
| 75 |
chunk_embedding = [embedding]
|
| 76 |
chunk_token_length = len(sentence)
|
| 77 |
|
| 78 |
-
|
|
|
|
| 79 |
chunks.append(" ".join(chunk))
|
| 80 |
|
| 81 |
# Deduplicate chunks
|
|
@@ -88,22 +96,21 @@ class PdfProcessor:
|
|
| 88 |
return summary if summary.endswith('.') else summary + '.'
|
| 89 |
|
| 90 |
def process_and_store_pdf(self, file_obj):
|
| 91 |
-
self._ensure_collection_exists()
|
| 92 |
text = self._extract_text(file_obj.name)
|
| 93 |
filename = os.path.basename(file_obj.name)
|
| 94 |
chunks = self._chunk_text(text)
|
| 95 |
points = []
|
| 96 |
-
batch_size = 100
|
| 97 |
|
| 98 |
-
# Use a set to track unique chunk texts within this method (optional redundancy)
|
| 99 |
seen_chunks = set()
|
| 100 |
|
| 101 |
for i in range(0, len(chunks), batch_size):
|
| 102 |
batch_chunks = chunks[i:i + batch_size]
|
| 103 |
-
embeddings = self.embeddings.embed_documents(batch_chunks)
|
| 104 |
|
| 105 |
for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
|
| 106 |
-
if chunk in seen_chunks:
|
| 107 |
continue
|
| 108 |
seen_chunks.add(chunk)
|
| 109 |
point_id = str(uuid.uuid4())
|
|
@@ -125,12 +132,12 @@ class PdfProcessor:
|
|
| 125 |
|
| 126 |
if points:
|
| 127 |
self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
|
| 128 |
-
points = []
|
| 129 |
|
| 130 |
return filename
|
| 131 |
|
| 132 |
def get_processed_files(self):
|
| 133 |
-
self._ensure_collection_exists()
|
| 134 |
processed_files = set()
|
| 135 |
scroll_response = self.qdrant_client.scroll(
|
| 136 |
collection_name=self.collection_name,
|
|
@@ -155,7 +162,7 @@ class PdfProcessor:
|
|
| 155 |
return processed_files
|
| 156 |
|
| 157 |
def delete_file_points(self, filename: str):
|
| 158 |
-
self._ensure_collection_exists()
|
| 159 |
filter_condition = models.Filter(
|
| 160 |
must=[
|
| 161 |
models.FieldCondition(
|
|
@@ -177,7 +184,7 @@ def query_vector_db(query: str) -> dict:
|
|
| 177 |
url=os.getenv("QDRANT_URL"),
|
| 178 |
api_key=os.getenv("QDRANT_API_KEY")
|
| 179 |
)
|
| 180 |
-
qdrant_client.ensure_collection_exists(collection_name="document_embeddings")
|
| 181 |
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
| 182 |
query_embedding = embeddings.embed_query(query)
|
| 183 |
query_response = qdrant_client.query_points(
|
|
@@ -190,7 +197,7 @@ def query_vector_db(query: str) -> dict:
|
|
| 190 |
if not search_results:
|
| 191 |
return {"context": "No relevant information found.", "source": "None", "chunks": []}
|
| 192 |
context = "\n\n".join(
|
| 193 |
-
[result.payload.get("text", "No text available") for result in search_results]
|
| 194 |
)
|
| 195 |
chunks = [result.payload.get("text", "No text available") for result in search_results]
|
| 196 |
source = search_results[0].payload.get("original_file_name", "Unknown")
|
|
|
|
| 5 |
from typing import List
|
| 6 |
import gradio as gr
|
| 7 |
from dotenv import load_dotenv
|
| 8 |
+
from docling.document import DoclingDocument # Import Docling
|
| 9 |
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
|
| 10 |
from langchain.prompts import PromptTemplate
|
| 11 |
from qdrant_client import QdrantClient
|
|
|
|
| 25 |
timeout=60
|
| 26 |
)
|
| 27 |
self.collection_name = "document_embeddings"
|
| 28 |
+
self._ensure_collection_exists()
|
| 29 |
|
| 30 |
def _ensure_collection_exists(self):
|
| 31 |
"""Ensure the Qdrant collection exists, creating it if necessary."""
|
|
|
|
| 36 |
)
|
| 37 |
|
| 38 |
def _extract_text(self, file_path):
|
| 39 |
+
"""Extract text from a PDF using Docling."""
|
| 40 |
+
# Load and parse the PDF with Docling
|
| 41 |
+
doc = DoclingDocument.from_file(file_path)
|
| 42 |
+
# Extract text from the document (assuming Docling provides a text representation)
|
| 43 |
+
text = doc.text # Adjust based on actual Docling API; this might need tweaking
|
| 44 |
+
return text if text else ""
|
| 45 |
|
| 46 |
+
def _chunk_text(self, text, max_chunk_size=500, min_chunk_size=100, similarity_threshold=0.5):
|
| 47 |
"""Performs semantic chunking using embeddings and cosine similarity."""
|
| 48 |
|
| 49 |
sentences = re.split(r'(?<=[.!?])\s+', text) # Split text into sentences
|
|
|
|
| 62 |
# Compute similarity to previous sentence
|
| 63 |
similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
|
| 64 |
|
| 65 |
+
# If adding the sentence exceeds max_chunk_size, finalize the current chunk
|
| 66 |
+
if chunk_token_length + len(sentence) > max_chunk_size:
|
| 67 |
+
if chunk_token_length >= min_chunk_size:
|
| 68 |
+
chunks.append(" ".join(chunk))
|
| 69 |
+
chunk = [sentence]
|
| 70 |
+
chunk_embedding = [embedding]
|
| 71 |
+
chunk_token_length = len(sentence)
|
| 72 |
+
# Merge if similarity is high and under max token limit
|
| 73 |
+
elif similarity > similarity_threshold:
|
| 74 |
chunk.append(sentence)
|
| 75 |
chunk_embedding.append(embedding)
|
| 76 |
chunk_token_length += len(sentence)
|
| 77 |
else:
|
| 78 |
# Finalize current chunk and start a new one
|
| 79 |
+
if chunk_token_length >= min_chunk_size:
|
| 80 |
chunks.append(" ".join(chunk))
|
|
|
|
| 81 |
chunk = [sentence]
|
| 82 |
chunk_embedding = [embedding]
|
| 83 |
chunk_token_length = len(sentence)
|
| 84 |
|
| 85 |
+
# Add the last chunk if it meets min size or is non-empty
|
| 86 |
+
if chunk and (chunk_token_length >= min_chunk_size or len(chunks) == 0):
|
| 87 |
chunks.append(" ".join(chunk))
|
| 88 |
|
| 89 |
# Deduplicate chunks
|
|
|
|
| 96 |
return summary if summary.endswith('.') else summary + '.'
|
| 97 |
|
| 98 |
def process_and_store_pdf(self, file_obj):
|
| 99 |
+
self._ensure_collection_exists()
|
| 100 |
text = self._extract_text(file_obj.name)
|
| 101 |
filename = os.path.basename(file_obj.name)
|
| 102 |
chunks = self._chunk_text(text)
|
| 103 |
points = []
|
| 104 |
+
batch_size = 100
|
| 105 |
|
|
|
|
| 106 |
seen_chunks = set()
|
| 107 |
|
| 108 |
for i in range(0, len(chunks), batch_size):
|
| 109 |
batch_chunks = chunks[i:i + batch_size]
|
| 110 |
+
embeddings = self.embeddings.embed_documents(batch_chunks)
|
| 111 |
|
| 112 |
for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
|
| 113 |
+
if chunk in seen_chunks:
|
| 114 |
continue
|
| 115 |
seen_chunks.add(chunk)
|
| 116 |
point_id = str(uuid.uuid4())
|
|
|
|
| 132 |
|
| 133 |
if points:
|
| 134 |
self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
|
| 135 |
+
points = []
|
| 136 |
|
| 137 |
return filename
|
| 138 |
|
| 139 |
def get_processed_files(self):
|
| 140 |
+
self._ensure_collection_exists()
|
| 141 |
processed_files = set()
|
| 142 |
scroll_response = self.qdrant_client.scroll(
|
| 143 |
collection_name=self.collection_name,
|
|
|
|
| 162 |
return processed_files
|
| 163 |
|
| 164 |
def delete_file_points(self, filename: str):
|
| 165 |
+
self._ensure_collection_exists()
|
| 166 |
filter_condition = models.Filter(
|
| 167 |
must=[
|
| 168 |
models.FieldCondition(
|
|
|
|
| 184 |
url=os.getenv("QDRANT_URL"),
|
| 185 |
api_key=os.getenv("QDRANT_API_KEY")
|
| 186 |
)
|
| 187 |
+
qdrant_client.ensure_collection_exists(collection_name="document_embeddings")
|
| 188 |
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
| 189 |
query_embedding = embeddings.embed_query(query)
|
| 190 |
query_response = qdrant_client.query_points(
|
|
|
|
| 197 |
if not search_results:
|
| 198 |
return {"context": "No relevant information found.", "source": "None", "chunks": []}
|
| 199 |
context = "\n\n".join(
|
| 200 |
+
[result.payload.get("text", "No text available") for result in search_results]
|
| 201 |
)
|
| 202 |
chunks = [result.payload.get("text", "No text available") for result in search_results]
|
| 203 |
source = search_results[0].payload.get("original_file_name", "Unknown")
|