import logging from core.exceptions import DocumentNotFoundError from repositories.document import ( delete_document_status, get_document_status as get_document_status_record, upsert_document_status, ) from repositories.embedding import delete_chunks_for_document from repositories.social_account import list_channel_identifiers_for_workspace from repositories.workspace import get_workspace_id_by_slug from rag.chunker import prepare_policy_chunks from rag.embedder import delete_chunks_for_source, store_chunks from services.cache_service import clear_instant_reply_cache, clear_instant_reply_cache_for_identifier from services.document_parser_service import SUPPORTED_MIME, parse_file from services.file_download_service import download_file logger = logging.getLogger(__name__) def validate_supported_mime(mime: str) -> None: if mime not in SUPPORTED_MIME: raise ValueError(f"Unsupported mime type. Supported: {SUPPORTED_MIME}") def resolve_workspace_id(workspace_slug: str) -> str: return get_workspace_id_by_slug(workspace_slug) def update_status( document_id: str, workspace_id: str, status: str, error: str = None, chunks: int = 0, ) -> None: upsert_document_status( document_id=document_id, workspace_id=workspace_id, status=status, error=error, chunks=chunks, ) def ensure_workspace_exists(workspace_slug: str) -> str: return get_workspace_id_by_slug(workspace_slug) def queue_document_ingest(background_tasks, **kwargs) -> None: background_tasks.add_task(process_document, **kwargs) def process_document( workspace_slug: str, document_id: str, source_url: str, mime: str, title: str, ) -> None: workspace_id = None try: validate_supported_mime(mime) workspace_id = resolve_workspace_id(workspace_slug) update_status(document_id, workspace_id, status="ingesting") logger.info(f"Processing document | id={document_id} | mime={mime} | workspace={workspace_slug}") file_bytes = download_file(source_url) logger.info(f"Downloaded {len(file_bytes) / 1024:.1f}KB for document={document_id}") text = parse_file(file_bytes, mime) if not text.strip(): raise ValueError("Could not extract any text - file may be empty or image-based") doc = { "id": document_id, "workspace_id": workspace_id, "type": "document", "title": title, "content": text, } delete_chunks_for_source(document_id) chunks = prepare_policy_chunks(doc) for chunk in chunks: chunk["document_id"] = document_id store_chunks(chunks) clear_document_product_result_cache(workspace_id) update_status(document_id, workspace_id, status="ready", chunks=len(chunks)) logger.info(f"Document embedded | id={document_id} | chunks={len(chunks)}") except Exception as e: logger.error(f"Document ingestion failed | id={document_id} | error={e}", exc_info=True) if workspace_id: update_status(document_id, workspace_id, status="failed", error=str(e)) def clear_document_product_result_cache(workspace_id: str) -> None: cleared = 0 for identifier in list_channel_identifiers_for_workspace(workspace_id): cleared += clear_instant_reply_cache_for_identifier(identifier) logger.info(f"Cache cleared for workspace={workspace_id} after document ingest | keys={cleared}") def get_document_status(document_id: str, workspace_slug: str) -> dict: workspace_id = resolve_workspace_id(workspace_slug) result = get_document_status_record(document_id, workspace_id) if not result: raise DocumentNotFoundError("Document not found") return { "status": result["status"], "error": result.get("error"), "chunks": result.get("chunks", 0), } def delete_document(document_id: str, workspace_slug: str) -> None: resolve_workspace_id(workspace_slug) delete_chunks_for_document(document_id) delete_document_status(document_id) clear_instant_reply_cache() logger.info(f"Document deleted | id={document_id}") get_workspace_id = resolve_workspace_id