replyflow-rag / rag /embedder.py
Qurrah's picture
REFINE: update exception handling to use ValueError instead of custom exceptions
6fab394
Raw History Blame Contribute Delete
2.39 kB
import logging
from sentence_transformers import SentenceTransformer
from core.config import settings
from repositories.embedding import (
delete_chunks_for_source as delete_chunks_for_source_record,
upsert_embedding_records,
)
logger = logging.getLogger(__name__)
_model = None
def get_embedding_model():
global _model
if _model is None:
_model = SentenceTransformer(settings.embedding_model)
return _model
def get_embedding(text: str) -> list[float]:
return get_embedding_model().encode(text).tolist()
def store_chunks(chunks: list[dict]):
if not chunks:
logger.warning("store_chunks called with empty list, skipping")
return
records = []
for chunk in chunks:
try:
embedding = get_embedding(chunk["content"])
records.append({
"workspace_id": chunk["workspace_id"],
"source_type": chunk["source_type"],
"source_id": chunk["source_id"],
"chunk_index": chunk["chunk_index"],
"content": chunk["content"],
"embedding": embedding,
"metadata": chunk["metadata"],
"document_id": chunk.get("document_id"), # None for non-BYOK chunks
})
except Exception as e:
logger.error(f"Failed to embed chunk {chunk.get('chunk_index')} of source {chunk.get('source_id')}: {e}")
continue
if not records:
logger.warning("No records to store after embedding — all chunks failed")
return
batch_size = 50
total = len(records)
stored = 0
for i in range(0, total, batch_size):
batch = records[i:i + batch_size]
try:
upsert_embedding_records(batch)
stored += len(batch)
logger.debug(f"Stored batch {i // batch_size + 1} ({len(batch)} chunks)")
except Exception as e:
logger.error(f"Failed to store batch {i // batch_size + 1}: {e}")
continue
logger.info(f"Stored {stored}/{total} chunks successfully")
def delete_chunks_for_source(source_id: str):
try:
delete_chunks_for_source_record(source_id)
logger.debug(f"Deleted existing chunks for source_id={source_id}")
except Exception as e:
logger.error(f"Failed to delete chunks for source_id={source_id}: {e}")
raise