Spaces:
Sleeping
Sleeping
Download rag/embedder.py from Qurrah/replyflow-rag: direct link, hf CLI and curl.
- Browser
- Download file 2.39 kB
-
https://huggingface.co/spaces/Qurrah/replyflow-rag/resolve/main/rag/embedder.py
- Command line
-
hf download hf://spaces/Qurrah/replyflow-rag/rag/embedder.py
-
curl -L -o embedder.py https://huggingface.co/spaces/Qurrah/replyflow-rag/resolve/main/rag/embedder.py
2.39 kB
| import logging | |
| from sentence_transformers import SentenceTransformer | |
| from core.config import settings | |
| from repositories.embedding import ( | |
| delete_chunks_for_source as delete_chunks_for_source_record, | |
| upsert_embedding_records, | |
| ) | |
| logger = logging.getLogger(__name__) | |
| _model = None | |
| def get_embedding_model(): | |
| global _model | |
| if _model is None: | |
| _model = SentenceTransformer(settings.embedding_model) | |
| return _model | |
| def get_embedding(text: str) -> list[float]: | |
| return get_embedding_model().encode(text).tolist() | |
| def store_chunks(chunks: list[dict]): | |
| if not chunks: | |
| logger.warning("store_chunks called with empty list, skipping") | |
| return | |
| records = [] | |
| for chunk in chunks: | |
| try: | |
| embedding = get_embedding(chunk["content"]) | |
| records.append({ | |
| "workspace_id": chunk["workspace_id"], | |
| "source_type": chunk["source_type"], | |
| "source_id": chunk["source_id"], | |
| "chunk_index": chunk["chunk_index"], | |
| "content": chunk["content"], | |
| "embedding": embedding, | |
| "metadata": chunk["metadata"], | |
| "document_id": chunk.get("document_id"), # None for non-BYOK chunks | |
| }) | |
| except Exception as e: | |
| logger.error(f"Failed to embed chunk {chunk.get('chunk_index')} of source {chunk.get('source_id')}: {e}") | |
| continue | |
| if not records: | |
| logger.warning("No records to store after embedding — all chunks failed") | |
| return | |
| batch_size = 50 | |
| total = len(records) | |
| stored = 0 | |
| for i in range(0, total, batch_size): | |
| batch = records[i:i + batch_size] | |
| try: | |
| upsert_embedding_records(batch) | |
| stored += len(batch) | |
| logger.debug(f"Stored batch {i // batch_size + 1} ({len(batch)} chunks)") | |
| except Exception as e: | |
| logger.error(f"Failed to store batch {i // batch_size + 1}: {e}") | |
| continue | |
| logger.info(f"Stored {stored}/{total} chunks successfully") | |
| def delete_chunks_for_source(source_id: str): | |
| try: | |
| delete_chunks_for_source_record(source_id) | |
| logger.debug(f"Deleted existing chunks for source_id={source_id}") | |
| except Exception as e: | |
| logger.error(f"Failed to delete chunks for source_id={source_id}: {e}") | |
| raise | |