Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -10,6 +10,9 @@ from langchain_openai import OpenAIEmbeddings, ChatOpenAI
|
|
| 10 |
from langchain.prompts import PromptTemplate
|
| 11 |
from qdrant_client import QdrantClient
|
| 12 |
from qdrant_client.http import models
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
load_dotenv()
|
| 15 |
|
|
@@ -38,21 +41,42 @@ class PdfProcessor:
|
|
| 38 |
text += page.extract_text() or ""
|
| 39 |
return text
|
| 40 |
|
| 41 |
-
def _chunk_text(self, text,
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
return chunks
|
| 57 |
|
| 58 |
def _generate_summary(self, text: str) -> str:
|
|
|
|
| 10 |
from langchain.prompts import PromptTemplate
|
| 11 |
from qdrant_client import QdrantClient
|
| 12 |
from qdrant_client.http import models
|
| 13 |
+
from sklearn.metrics.pairwise import cosine_similarity
|
| 14 |
+
import numpy as np
|
| 15 |
+
import re
|
| 16 |
|
| 17 |
load_dotenv()
|
| 18 |
|
|
|
|
| 41 |
text += page.extract_text() or ""
|
| 42 |
return text
|
| 43 |
|
| 44 |
+
def _chunk_text(self, text, max_chunk_size=1000, min_chunk_size=300, similarity_threshold=0.7):
|
| 45 |
+
"""Performs semantic chunking using embeddings and cosine similarity."""
|
| 46 |
+
|
| 47 |
+
sentences = re.split(r'(?<=[.!?])\s+', text) # Split text into sentences
|
| 48 |
+
sentence_embeddings = self.embeddings.embed_documents(sentences) # Generate embeddings
|
| 49 |
+
|
| 50 |
+
chunks, chunk, chunk_embedding = [], [], []
|
| 51 |
+
chunk_token_length = 0
|
| 52 |
+
|
| 53 |
+
for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
|
| 54 |
+
if not chunk:
|
| 55 |
+
chunk.append(sentence)
|
| 56 |
+
chunk_embedding.append(embedding)
|
| 57 |
+
chunk_token_length = len(sentence)
|
| 58 |
+
continue
|
| 59 |
+
|
| 60 |
+
# Compute similarity to previous sentence
|
| 61 |
+
similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
|
| 62 |
+
|
| 63 |
+
# Merge if similarity is high & under max token limit
|
| 64 |
+
if similarity > similarity_threshold and chunk_token_length + len(sentence) <= max_chunk_size:
|
| 65 |
+
chunk.append(sentence)
|
| 66 |
+
chunk_embedding.append(embedding)
|
| 67 |
+
chunk_token_length += len(sentence)
|
| 68 |
+
else:
|
| 69 |
+
# Finalize current chunk and start a new one
|
| 70 |
+
if chunk_token_length >= min_chunk_size: # Ensure minimum chunk size
|
| 71 |
+
chunks.append(" ".join(chunk))
|
| 72 |
+
|
| 73 |
+
chunk = [sentence]
|
| 74 |
+
chunk_embedding = [embedding]
|
| 75 |
+
chunk_token_length = len(sentence)
|
| 76 |
+
|
| 77 |
+
if chunk:
|
| 78 |
+
chunks.append(" ".join(chunk)) # Add the last chunk
|
| 79 |
+
|
| 80 |
return chunks
|
| 81 |
|
| 82 |
def _generate_summary(self, text: str) -> str:
|