Spaces:
Sleeping
Sleeping
Download src/store.py from Jky71/doculearn-rag: direct link, hf CLI and curl.
- Browser
- Download file 6.75 kB
-
https://huggingface.co/spaces/Jky71/doculearn-rag/resolve/main/src/store.py
- Command line
-
hf download hf://spaces/Jky71/doculearn-rag/src/store.py
-
curl -L -o store.py https://huggingface.co/spaces/Jky71/doculearn-rag/resolve/main/src/store.py
6.75 kB
| """Embeddings, Qdrant client, collection setup, and vector store.""" | |
| from __future__ import annotations | |
| import atexit | |
| from collections.abc import Iterator | |
| from functools import lru_cache | |
| from langchain_qdrant import QdrantVectorStore | |
| from qdrant_client import QdrantClient | |
| from qdrant_client.http import models as qmodels | |
| from src.config import settings | |
| from src.embeddings import get_embeddings | |
| _SCROLL_PAGE_SIZE = 256 | |
| INDEXED_PAYLOAD_FIELDS = { | |
| "metadata.document_id": qmodels.PayloadSchemaType.KEYWORD, | |
| "metadata.filename": qmodels.PayloadSchemaType.KEYWORD, | |
| "metadata.page": qmodels.PayloadSchemaType.INTEGER, | |
| } | |
| def _old_document_versions_filter( | |
| filename: str, | |
| keep_document_id: str, | |
| ) -> qmodels.Filter: | |
| """Select older vector versions for one uploaded filename.""" | |
| return qmodels.Filter( | |
| must=[ | |
| qmodels.FieldCondition( | |
| key="metadata.filename", | |
| match=qmodels.MatchValue(value=filename), | |
| ) | |
| ], | |
| must_not=[ | |
| qmodels.FieldCondition( | |
| key="metadata.document_id", | |
| match=qmodels.MatchValue(value=keep_document_id), | |
| ) | |
| ], | |
| ) | |
| def delete_old_document_versions( | |
| filename: str, | |
| keep_document_id: str, | |
| collection_name: str | None = None, | |
| ) -> None: | |
| """Delete stale vectors for a filename after its new version is indexed.""" | |
| name = collection_name or settings.qdrant_collection | |
| get_client().delete( | |
| collection_name=name, | |
| points_selector=qmodels.FilterSelector( | |
| filter=_old_document_versions_filter(filename, keep_document_id) | |
| ), | |
| wait=True, | |
| ) | |
| def delete_document_by_filename(filename: str, collection_name: str | None = None) -> None: | |
| """Delete all vectors for a given filename from Qdrant.""" | |
| name = collection_name or settings.qdrant_collection | |
| get_client().delete( | |
| collection_name=name, | |
| points_selector=qmodels.FilterSelector( | |
| filter=qmodels.Filter( | |
| must=[ | |
| qmodels.FieldCondition( | |
| key="metadata.filename", | |
| match=qmodels.MatchValue(value=filename), | |
| ) | |
| ], | |
| ) | |
| ), | |
| wait=True, | |
| ) | |
| def close_client() -> None: | |
| if get_client.cache_info().currsize == 0: | |
| return | |
| client = get_client() | |
| client.close() | |
| get_client.cache_clear() | |
| atexit.register(close_client) | |
| def get_client() -> QdrantClient: | |
| """Return a cached local Qdrant client backed by on-disk storage.""" | |
| settings.storage_dir.mkdir(parents=True, exist_ok=True) | |
| lock = settings.storage_dir / ".lock" | |
| if lock.exists(): | |
| lock.unlink() | |
| return QdrantClient(path=str(settings.storage_dir)) | |
| def ensure_collection(recreate: bool = False, collection_name: str | None = None) -> None: | |
| """Create the collection and payload indexes if they do not exist.""" | |
| client = get_client() | |
| name = collection_name or settings.qdrant_collection | |
| exists = client.collection_exists(name) | |
| if exists and recreate: | |
| client.delete_collection(name) | |
| exists = False | |
| if not exists: | |
| dim = len(get_embeddings().embed_query("dimension probe")) | |
| client.create_collection( | |
| collection_name=name, | |
| vectors_config=qmodels.VectorParams( | |
| size=dim, | |
| distance=qmodels.Distance.COSINE, | |
| ), | |
| ) | |
| payload_schema = client.get_collection(name).payload_schema or {} | |
| for field_name, field_schema in INDEXED_PAYLOAD_FIELDS.items(): | |
| existing = payload_schema.get(field_name) | |
| if existing is None: | |
| client.create_payload_index( | |
| collection_name=name, | |
| field_name=field_name, | |
| field_schema=field_schema, | |
| ) | |
| continue | |
| existing_schema = getattr(existing, "data_type", None) | |
| if existing_schema != field_schema: | |
| raise ValueError( | |
| f"Payload index for '{field_name}' has schema " | |
| f"{existing_schema!r}, expected {field_schema!r}." | |
| ) | |
| def scroll_all( | |
| collection_name: str, | |
| scroll_filter: qmodels.Filter | None = None, | |
| with_payload: bool | list[str] = True, | |
| limit: int = _SCROLL_PAGE_SIZE, | |
| ) -> Iterator[list[qmodels.Record]]: | |
| """Yield pages of Qdrant points (no vectors) until the collection is exhausted.""" | |
| client = get_client() | |
| offset = None | |
| while True: | |
| try: | |
| points, next_offset = client.scroll( | |
| collection_name=collection_name, | |
| scroll_filter=scroll_filter, | |
| limit=limit, | |
| offset=offset, | |
| with_payload=with_payload, | |
| with_vectors=False, | |
| ) | |
| except ValueError as exc: | |
| # Local Qdrant raises ValueError when collection doesn't exist yet. | |
| if "not found" in str(exc).lower(): | |
| return | |
| raise | |
| yield points | |
| if next_offset is None: | |
| break | |
| offset = next_offset | |
| def get_vector_store(collection_name: str | None = None) -> QdrantVectorStore: | |
| return QdrantVectorStore( | |
| client=get_client(), | |
| collection_name=collection_name or settings.qdrant_collection, | |
| embedding=get_embeddings(), | |
| ) | |
| def list_documents() -> list[dict[str, object]]: | |
| """List indexed documents with filename, document_id, pages, and chunk counts. | |
| Returns one entry per filename matching the API `DocumentInfo` shape. | |
| """ | |
| pages_map: dict[str, set[int]] = {} | |
| doc_id_map: dict[str, str] = {} | |
| count_map: dict[str, int] = {} | |
| for batch in scroll_all(settings.qdrant_collection, with_payload=["metadata"]): | |
| for point in batch: | |
| meta = (point.payload or {}).get("metadata") or {} | |
| filename = meta.get("filename") | |
| document_id = meta.get("document_id") | |
| pg = meta.get("page") | |
| if not filename or not document_id or not isinstance(pg, int): | |
| continue | |
| fn = str(filename) | |
| doc_id_map.setdefault(fn, str(document_id)) | |
| pages_map.setdefault(fn, set()).add(pg) | |
| count_map[fn] = count_map.get(fn, 0) + 1 | |
| return sorted( | |
| [ | |
| { | |
| "filename": fn, | |
| "document_id": doc_id_map[fn], | |
| "pages": sorted(pages_map[fn]), | |
| "page_count": len(pages_map[fn]), | |
| "chunk_count": count_map[fn], | |
| } | |
| for fn in doc_id_map | |
| ], | |
| key=lambda d: str(d["filename"]), | |
| ) | |