Spaces:
Sleeping
Sleeping
File size: 4,283 Bytes
1c0a11d 4b128b9 8ec9671 1c0a11d 8ec9671 1c0a11d b6714d8 1c0a11d 6fab394 1c0a11d 6fab394 1c0a11d d6abbac 4b128b9 1c0a11d 6fab394 1c0a11d b6714d8 1c0a11d b6714d8 1c0a11d b6714d8 1c0a11d 4b128b9 1c0a11d b6714d8 1c0a11d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | import logging
from core.exceptions import DocumentNotFoundError
from repositories.document import (
delete_document_status,
get_document_status as get_document_status_record,
upsert_document_status,
)
from repositories.embedding import delete_chunks_for_document
from repositories.social_account import list_channel_identifiers_for_workspace
from repositories.workspace import get_workspace_id_by_slug
from rag.chunker import prepare_policy_chunks
from rag.embedder import delete_chunks_for_source, store_chunks
from services.cache_service import clear_instant_reply_cache, clear_instant_reply_cache_for_identifier
from services.document_parser_service import SUPPORTED_MIME, parse_file
from services.file_download_service import download_file
logger = logging.getLogger(__name__)
def validate_supported_mime(mime: str) -> None:
if mime not in SUPPORTED_MIME:
raise ValueError(f"Unsupported mime type. Supported: {SUPPORTED_MIME}")
def resolve_workspace_id(workspace_slug: str) -> str:
return get_workspace_id_by_slug(workspace_slug)
def update_status(
document_id: str,
workspace_id: str,
status: str,
error: str = None,
chunks: int = 0,
) -> None:
upsert_document_status(
document_id=document_id,
workspace_id=workspace_id,
status=status,
error=error,
chunks=chunks,
)
def ensure_workspace_exists(workspace_slug: str) -> str:
return get_workspace_id_by_slug(workspace_slug)
def queue_document_ingest(background_tasks, **kwargs) -> None:
background_tasks.add_task(process_document, **kwargs)
def process_document(
workspace_slug: str,
document_id: str,
source_url: str,
mime: str,
title: str,
) -> None:
workspace_id = None
try:
validate_supported_mime(mime)
workspace_id = resolve_workspace_id(workspace_slug)
update_status(document_id, workspace_id, status="ingesting")
logger.info(f"Processing document | id={document_id} | mime={mime} | workspace={workspace_slug}")
file_bytes = download_file(source_url)
logger.info(f"Downloaded {len(file_bytes) / 1024:.1f}KB for document={document_id}")
text = parse_file(file_bytes, mime)
if not text.strip():
raise ValueError("Could not extract any text - file may be empty or image-based")
doc = {
"id": document_id,
"workspace_id": workspace_id,
"type": "document",
"title": title,
"content": text,
}
delete_chunks_for_source(document_id)
chunks = prepare_policy_chunks(doc)
for chunk in chunks:
chunk["document_id"] = document_id
store_chunks(chunks)
clear_document_product_result_cache(workspace_id)
update_status(document_id, workspace_id, status="ready", chunks=len(chunks))
logger.info(f"Document embedded | id={document_id} | chunks={len(chunks)}")
except Exception as e:
logger.error(f"Document ingestion failed | id={document_id} | error={e}", exc_info=True)
if workspace_id:
update_status(document_id, workspace_id, status="failed", error=str(e))
def clear_document_product_result_cache(workspace_id: str) -> None:
cleared = 0
for identifier in list_channel_identifiers_for_workspace(workspace_id):
cleared += clear_instant_reply_cache_for_identifier(identifier)
logger.info(f"Cache cleared for workspace={workspace_id} after document ingest | keys={cleared}")
def get_document_status(document_id: str, workspace_slug: str) -> dict:
workspace_id = resolve_workspace_id(workspace_slug)
result = get_document_status_record(document_id, workspace_id)
if not result:
raise DocumentNotFoundError("Document not found")
return {
"status": result["status"],
"error": result.get("error"),
"chunks": result.get("chunks", 0),
}
def delete_document(document_id: str, workspace_slug: str) -> None:
resolve_workspace_id(workspace_slug)
delete_chunks_for_document(document_id)
delete_document_status(document_id)
clear_instant_reply_cache()
logger.info(f"Document deleted | id={document_id}")
get_workspace_id = resolve_workspace_id
|