File size: 1,411 Bytes
9cc7f8d 399b102 c9dbaae 9cc7f8d 77d7fca 9cc7f8d c9dbaae 399b102 77d7fca c9dbaae bb05158 399b102 77d7fca bb05158 399b102 bb05158 399b102 bb05158 3301ece 399b102 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | import uuid
import os
from dotenv import load_dotenv
from src.ingestion import ingestion_and_chunking
from langchain_qdrant import QdrantVectorStore, RetrievalMode, FastEmbedSparse
from langchain_huggingface import HuggingFaceEmbeddings
load_dotenv()
qdrant_api_key = os.getenv("QDRANT_API_KEY")
qdrant_url = os.getenv("QDRANT_URL")
dense_embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
def upload_file(file_bytes: bytes, filename: str, user_id: str, collection_name: str = "pdf_rag"):
docs = ingestion_and_chunking(file_bytes, filename)
file_id = str(uuid.uuid4())
for doc in docs:
doc.metadata["user_id"] = user_id
doc.metadata["file_id"] = file_id
vector_store = QdrantVectorStore.from_documents(
docs,
embedding=dense_embeddings,
sparse_embedding=sparse_embeddings,
url=qdrant_url,
api_key=qdrant_api_key,
collection_name=collection_name,
retrieval_mode=RetrievalMode.HYBRID,
vector_name="dense",
sparse_vector_name="sparse",
)
try:
vector_store.client.create_payload_index(
collection_name=collection_name,
field_name="metadata.user_id",
field_schema="keyword",
)
except Exception:
print("Failed")
return vector_store |