GraphRAG / vector_processor.py
dipenvekaria's picture
Upload 6 files
7b43396 verified
Raw History Blame
7.98 kB
import os
import uuid
from datetime import datetime
from dotenv import load_dotenv
import re
import time
from openai import OpenAI
from qdrant_client import QdrantClient
from qdrant_client.http import models
from sklearn.metrics.pairwise import cosine_similarity
from utils import extract_pdf_text
load_dotenv()
class VectorDBProcessor:
def __init__(self):
self.openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
self.qdrant_client = QdrantClient(
url=os.getenv("QDRANT_URL"),
api_key=os.getenv("QDRANT_API_KEY"),
timeout=60
)
self.collection_name = "pdf_text_vectors"
self._ensure_collection_exists()
def _ensure_collection_exists(self):
if not self.qdrant_client.collection_exists(collection_name=self.collection_name):
self.qdrant_client.create_collection(
collection_name=self.collection_name,
vectors_config=models.VectorParams(size=1536, distance=models.Distance.COSINE)
)
print(f"Created collection: {self.collection_name}")
def _chunk_text(self, text, max_chunk_size=500, min_chunk_size=100, similarity_threshold=0.5):
# Split text into sentences and filter out empty strings
sentences = [s.strip() for s in re.split(r'(?<=[.!?])\s+', text) if s.strip()]
if not sentences:
return [] # Return empty list if no valid sentences
# Log the sentences for debugging
print(f"Sentences to embed: {sentences}")
# Embed sentences
sentence_embeddings = self._embed_documents(sentences)
if len(sentences) != len(sentence_embeddings):
raise ValueError("Mismatch between number of sentences and embeddings")
chunks, chunk, chunk_embedding = [], [], []
chunk_token_length = 0
for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
if not chunk:
chunk.append(sentence)
chunk_embedding.append(embedding)
chunk_token_length = len(sentence)
continue
similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
if chunk_token_length + len(sentence) > max_chunk_size:
if chunk_token_length >= min_chunk_size:
chunks.append(" ".join(chunk))
chunk = [sentence]
chunk_embedding = [embedding]
chunk_token_length = len(sentence)
elif similarity > similarity_threshold:
chunk.append(sentence)
chunk_embedding.append(embedding)
chunk_token_length += len(sentence)
else:
if chunk_token_length >= min_chunk_size:
chunks.append(" ".join(chunk))
chunk = [sentence]
chunk_embedding = [embedding]
chunk_token_length = len(sentence)
if chunk and (chunk_token_length >= min_chunk_size or len(chunks) == 0):
chunks.append(" ".join(chunk))
return list(dict.fromkeys(chunks))
def _embed_documents(self, texts):
response = self.openai_client.embeddings.create(
model="text-embedding-3-small",
input=texts
)
return [item.embedding for item in response.data]
def _embed_query(self, text):
response = self.openai_client.embeddings.create(
model="text-embedding-3-small",
input=[text]
)
return response.data[0].embedding
def process_and_store_pdf(self, file_obj):
self._ensure_collection_exists()
text = extract_pdf_text(file_obj.name)
filename = os.path.basename(file_obj.name)
chunks = self._chunk_text(text)
points = []
batch_size = 100
seen_chunks = set()
for i in range(0, len(chunks), batch_size):
batch_chunks = chunks[i:i + batch_size]
embeddings = self._embed_documents(batch_chunks)
for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
if chunk in seen_chunks:
continue
seen_chunks.add(chunk)
point_id = str(uuid.uuid4())
points.append(
models.PointStruct(
id=point_id,
vector=embedding,
payload={
"text": chunk,
"source_file": filename,
"chunk_id": i + j,
"original_file_name": filename,
"conversion_date": datetime.now().isoformat()
}
)
)
if points:
self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
points = []
return filename
def query(self, question):
query_embedding = self._embed_query(question)
query_response = self.qdrant_client.query_points(
collection_name=self.collection_name,
query=query_embedding,
limit=5,
with_payload=True
)
search_results = query_response.points
if not search_results:
return {"answer": "No relevant information found.", "chunks": []}
context = "\n\n".join([result.payload.get("text", "No text available") for result in search_results])
chunks = [result.payload.get("text", "No text available") for result in search_results]
source = search_results[0].payload.get("original_file_name", "Unknown")
prompt = f"""Based solely on the provided context, provide a concise, accurate answer to the question. Do not use external knowledge.
Question: {question}
Context: {context}
Answer: """
response = self.openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[
{"role": "system", "content": "You are a helpful assistant that answers questions based on provided context."},
{"role": "user", "content": prompt}
],
temperature=0,
max_tokens=100
)
answer = response.choices[0].message.content.strip()
if "context does not" in answer:
final_answer = answer
else:
final_answer = f"{answer} [Source: {source}]"
return {"answer": final_answer, "chunks": chunks}
def get_processed_files(self):
processed_files = set()
scroll_response = self.qdrant_client.scroll(
collection_name=self.collection_name,
limit=100,
with_payload=True
)
points, next_offset = scroll_response
while points:
for point in points:
original_file_name = point.payload.get("original_file_name")
if original_file_name:
processed_files.add(original_file_name)
if next_offset is None:
break
scroll_response = self.qdrant_client.scroll(
collection_name=self.collection_name,
limit=100,
offset=next_offset,
with_payload=True
)
points, next_offset = scroll_response
return processed_files
def delete_file_points(self, filename):
filter_condition = models.Filter(
must=[
models.FieldCondition(
key="original_file_name",
match=models.MatchValue(value=filename)
)
]
)
self.qdrant_client.delete(
collection_name=self.collection_name,
points_selector=models.FilterSelector(filter=filter_condition)
)
time.sleep(0.5)
return f"Deleted points for file: {filename}"