Spaces:
Sleeping
Sleeping
Download services/document_service.py from Qurrah/replyflow-rag: direct link, hf CLI and curl.
- Browser
- Download file 4.28 kB
-
https://huggingface.co/spaces/Qurrah/replyflow-rag/resolve/main/services/document_service.py
- Command line
-
hf download hf://spaces/Qurrah/replyflow-rag/services/document_service.py
-
curl -L -o document_service.py https://huggingface.co/spaces/Qurrah/replyflow-rag/resolve/main/services/document_service.py
4.28 kB
| import logging | |
| from core.exceptions import DocumentNotFoundError | |
| from repositories.document import ( | |
| delete_document_status, | |
| get_document_status as get_document_status_record, | |
| upsert_document_status, | |
| ) | |
| from repositories.embedding import delete_chunks_for_document | |
| from repositories.social_account import list_channel_identifiers_for_workspace | |
| from repositories.workspace import get_workspace_id_by_slug | |
| from rag.chunker import prepare_policy_chunks | |
| from rag.embedder import delete_chunks_for_source, store_chunks | |
| from services.cache_service import clear_instant_reply_cache, clear_instant_reply_cache_for_identifier | |
| from services.document_parser_service import SUPPORTED_MIME, parse_file | |
| from services.file_download_service import download_file | |
| logger = logging.getLogger(__name__) | |
| def validate_supported_mime(mime: str) -> None: | |
| if mime not in SUPPORTED_MIME: | |
| raise ValueError(f"Unsupported mime type. Supported: {SUPPORTED_MIME}") | |
| def resolve_workspace_id(workspace_slug: str) -> str: | |
| return get_workspace_id_by_slug(workspace_slug) | |
| def update_status( | |
| document_id: str, | |
| workspace_id: str, | |
| status: str, | |
| error: str = None, | |
| chunks: int = 0, | |
| ) -> None: | |
| upsert_document_status( | |
| document_id=document_id, | |
| workspace_id=workspace_id, | |
| status=status, | |
| error=error, | |
| chunks=chunks, | |
| ) | |
| def ensure_workspace_exists(workspace_slug: str) -> str: | |
| return get_workspace_id_by_slug(workspace_slug) | |
| def queue_document_ingest(background_tasks, **kwargs) -> None: | |
| background_tasks.add_task(process_document, **kwargs) | |
| def process_document( | |
| workspace_slug: str, | |
| document_id: str, | |
| source_url: str, | |
| mime: str, | |
| title: str, | |
| ) -> None: | |
| workspace_id = None | |
| try: | |
| validate_supported_mime(mime) | |
| workspace_id = resolve_workspace_id(workspace_slug) | |
| update_status(document_id, workspace_id, status="ingesting") | |
| logger.info(f"Processing document | id={document_id} | mime={mime} | workspace={workspace_slug}") | |
| file_bytes = download_file(source_url) | |
| logger.info(f"Downloaded {len(file_bytes) / 1024:.1f}KB for document={document_id}") | |
| text = parse_file(file_bytes, mime) | |
| if not text.strip(): | |
| raise ValueError("Could not extract any text - file may be empty or image-based") | |
| doc = { | |
| "id": document_id, | |
| "workspace_id": workspace_id, | |
| "type": "document", | |
| "title": title, | |
| "content": text, | |
| } | |
| delete_chunks_for_source(document_id) | |
| chunks = prepare_policy_chunks(doc) | |
| for chunk in chunks: | |
| chunk["document_id"] = document_id | |
| store_chunks(chunks) | |
| clear_document_product_result_cache(workspace_id) | |
| update_status(document_id, workspace_id, status="ready", chunks=len(chunks)) | |
| logger.info(f"Document embedded | id={document_id} | chunks={len(chunks)}") | |
| except Exception as e: | |
| logger.error(f"Document ingestion failed | id={document_id} | error={e}", exc_info=True) | |
| if workspace_id: | |
| update_status(document_id, workspace_id, status="failed", error=str(e)) | |
| def clear_document_product_result_cache(workspace_id: str) -> None: | |
| cleared = 0 | |
| for identifier in list_channel_identifiers_for_workspace(workspace_id): | |
| cleared += clear_instant_reply_cache_for_identifier(identifier) | |
| logger.info(f"Cache cleared for workspace={workspace_id} after document ingest | keys={cleared}") | |
| def get_document_status(document_id: str, workspace_slug: str) -> dict: | |
| workspace_id = resolve_workspace_id(workspace_slug) | |
| result = get_document_status_record(document_id, workspace_id) | |
| if not result: | |
| raise DocumentNotFoundError("Document not found") | |
| return { | |
| "status": result["status"], | |
| "error": result.get("error"), | |
| "chunks": result.get("chunks", 0), | |
| } | |
| def delete_document(document_id: str, workspace_slug: str) -> None: | |
| resolve_workspace_id(workspace_slug) | |
| delete_chunks_for_document(document_id) | |
| delete_document_status(document_id) | |
| clear_instant_reply_cache() | |
| logger.info(f"Document deleted | id={document_id}") | |
| get_workspace_id = resolve_workspace_id | |