import logging from sentence_transformers import SentenceTransformer from core.config import settings from repositories.embedding import ( delete_chunks_for_source as delete_chunks_for_source_record, upsert_embedding_records, ) logger = logging.getLogger(__name__) _model = None def get_embedding_model(): global _model if _model is None: _model = SentenceTransformer(settings.embedding_model) return _model def get_embedding(text: str) -> list[float]: return get_embedding_model().encode(text).tolist() def store_chunks(chunks: list[dict]): if not chunks: logger.warning("store_chunks called with empty list, skipping") return records = [] for chunk in chunks: try: embedding = get_embedding(chunk["content"]) records.append({ "workspace_id": chunk["workspace_id"], "source_type": chunk["source_type"], "source_id": chunk["source_id"], "chunk_index": chunk["chunk_index"], "content": chunk["content"], "embedding": embedding, "metadata": chunk["metadata"], "document_id": chunk.get("document_id"), # None for non-BYOK chunks }) except Exception as e: logger.error(f"Failed to embed chunk {chunk.get('chunk_index')} of source {chunk.get('source_id')}: {e}") continue if not records: logger.warning("No records to store after embedding — all chunks failed") return batch_size = 50 total = len(records) stored = 0 for i in range(0, total, batch_size): batch = records[i:i + batch_size] try: upsert_embedding_records(batch) stored += len(batch) logger.debug(f"Stored batch {i // batch_size + 1} ({len(batch)} chunks)") except Exception as e: logger.error(f"Failed to store batch {i // batch_size + 1}: {e}") continue logger.info(f"Stored {stored}/{total} chunks successfully") def delete_chunks_for_source(source_id: str): try: delete_chunks_for_source_record(source_id) logger.debug(f"Deleted existing chunks for source_id={source_id}") except Exception as e: logger.error(f"Failed to delete chunks for source_id={source_id}: {e}") raise