dipenvekaria commited on
Commit
d79a838
·
verified ·
1 Parent(s): e508e34

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +39 -15
app.py CHANGED
@@ -10,6 +10,9 @@ from langchain_openai import OpenAIEmbeddings, ChatOpenAI
10
  from langchain.prompts import PromptTemplate
11
  from qdrant_client import QdrantClient
12
  from qdrant_client.http import models
 
 
 
13
 
14
  load_dotenv()
15
 
@@ -38,21 +41,42 @@ class PdfProcessor:
38
  text += page.extract_text() or ""
39
  return text
40
 
41
- def _chunk_text(self, text, chunk_size=1000, chunk_overlap=200):
42
- chunks = []
43
- start = 0
44
- text_length = len(text)
45
- while start < text_length:
46
- end = min(start + chunk_size, text_length)
47
- if end < text_length:
48
- last_period = text.rfind('.', start, end)
49
- last_newline = text.rfind('\n', start, end)
50
- split_point = max(last_period, last_newline) if last_period > start or last_newline > start else end
51
- end = split_point if split_point > start else end
52
- chunk = text[start:end].strip()
53
- if chunk:
54
- chunks.append(chunk)
55
- start = end - chunk_overlap if end - chunk_overlap > start else end
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  return chunks
57
 
58
  def _generate_summary(self, text: str) -> str:
 
10
  from langchain.prompts import PromptTemplate
11
  from qdrant_client import QdrantClient
12
  from qdrant_client.http import models
13
+ from sklearn.metrics.pairwise import cosine_similarity
14
+ import numpy as np
15
+ import re
16
 
17
  load_dotenv()
18
 
 
41
  text += page.extract_text() or ""
42
  return text
43
 
44
+ def _chunk_text(self, text, max_chunk_size=1000, min_chunk_size=300, similarity_threshold=0.7):
45
+ """Performs semantic chunking using embeddings and cosine similarity."""
46
+
47
+ sentences = re.split(r'(?<=[.!?])\s+', text) # Split text into sentences
48
+ sentence_embeddings = self.embeddings.embed_documents(sentences) # Generate embeddings
49
+
50
+ chunks, chunk, chunk_embedding = [], [], []
51
+ chunk_token_length = 0
52
+
53
+ for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
54
+ if not chunk:
55
+ chunk.append(sentence)
56
+ chunk_embedding.append(embedding)
57
+ chunk_token_length = len(sentence)
58
+ continue
59
+
60
+ # Compute similarity to previous sentence
61
+ similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
62
+
63
+ # Merge if similarity is high & under max token limit
64
+ if similarity > similarity_threshold and chunk_token_length + len(sentence) <= max_chunk_size:
65
+ chunk.append(sentence)
66
+ chunk_embedding.append(embedding)
67
+ chunk_token_length += len(sentence)
68
+ else:
69
+ # Finalize current chunk and start a new one
70
+ if chunk_token_length >= min_chunk_size: # Ensure minimum chunk size
71
+ chunks.append(" ".join(chunk))
72
+
73
+ chunk = [sentence]
74
+ chunk_embedding = [embedding]
75
+ chunk_token_length = len(sentence)
76
+
77
+ if chunk:
78
+ chunks.append(" ".join(chunk)) # Add the last chunk
79
+
80
  return chunks
81
 
82
  def _generate_summary(self, text: str) -> str: