Spaces:
Sleeping
Sleeping
File size: 2,385 Bytes
c1ef8cb 6fab394 c1ef8cb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | import logging
from sentence_transformers import SentenceTransformer
from core.config import settings
from repositories.embedding import (
delete_chunks_for_source as delete_chunks_for_source_record,
upsert_embedding_records,
)
logger = logging.getLogger(__name__)
_model = None
def get_embedding_model():
global _model
if _model is None:
_model = SentenceTransformer(settings.embedding_model)
return _model
def get_embedding(text: str) -> list[float]:
return get_embedding_model().encode(text).tolist()
def store_chunks(chunks: list[dict]):
if not chunks:
logger.warning("store_chunks called with empty list, skipping")
return
records = []
for chunk in chunks:
try:
embedding = get_embedding(chunk["content"])
records.append({
"workspace_id": chunk["workspace_id"],
"source_type": chunk["source_type"],
"source_id": chunk["source_id"],
"chunk_index": chunk["chunk_index"],
"content": chunk["content"],
"embedding": embedding,
"metadata": chunk["metadata"],
"document_id": chunk.get("document_id"), # None for non-BYOK chunks
})
except Exception as e:
logger.error(f"Failed to embed chunk {chunk.get('chunk_index')} of source {chunk.get('source_id')}: {e}")
continue
if not records:
logger.warning("No records to store after embedding — all chunks failed")
return
batch_size = 50
total = len(records)
stored = 0
for i in range(0, total, batch_size):
batch = records[i:i + batch_size]
try:
upsert_embedding_records(batch)
stored += len(batch)
logger.debug(f"Stored batch {i // batch_size + 1} ({len(batch)} chunks)")
except Exception as e:
logger.error(f"Failed to store batch {i // batch_size + 1}: {e}")
continue
logger.info(f"Stored {stored}/{total} chunks successfully")
def delete_chunks_for_source(source_id: str):
try:
delete_chunks_for_source_record(source_id)
logger.debug(f"Deleted existing chunks for source_id={source_id}")
except Exception as e:
logger.error(f"Failed to delete chunks for source_id={source_id}: {e}")
raise
|