File size: 4,283 Bytes
1c0a11d
 
4b128b9
8ec9671
1c0a11d
 
 
 
8ec9671
 
 
1c0a11d
 
b6714d8
1c0a11d
 
 
 
 
 
 
 
 
6fab394
1c0a11d
 
 
6fab394
1c0a11d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d6abbac
4b128b9
1c0a11d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6fab394
1c0a11d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b6714d8
1c0a11d
 
 
 
 
 
 
 
 
 
b6714d8
1c0a11d
 
b6714d8
1c0a11d
 
 
 
 
 
 
 
 
4b128b9
1c0a11d
 
 
 
 
 
 
 
 
 
 
 
 
 
b6714d8
1c0a11d
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
import logging

from core.exceptions import DocumentNotFoundError
from repositories.document import (
    delete_document_status,
    get_document_status as get_document_status_record,
    upsert_document_status,
)
from repositories.embedding import delete_chunks_for_document
from repositories.social_account import list_channel_identifiers_for_workspace
from repositories.workspace import get_workspace_id_by_slug
from rag.chunker import prepare_policy_chunks
from rag.embedder import delete_chunks_for_source, store_chunks
from services.cache_service import clear_instant_reply_cache, clear_instant_reply_cache_for_identifier
from services.document_parser_service import SUPPORTED_MIME, parse_file
from services.file_download_service import download_file


logger = logging.getLogger(__name__)


def validate_supported_mime(mime: str) -> None:
    if mime not in SUPPORTED_MIME:
        raise ValueError(f"Unsupported mime type. Supported: {SUPPORTED_MIME}")


def resolve_workspace_id(workspace_slug: str) -> str:
    return get_workspace_id_by_slug(workspace_slug)


def update_status(
    document_id: str,
    workspace_id: str,
    status: str,
    error: str = None,
    chunks: int = 0,
) -> None:
    upsert_document_status(
        document_id=document_id,
        workspace_id=workspace_id,
        status=status,
        error=error,
        chunks=chunks,
    )
def ensure_workspace_exists(workspace_slug: str) -> str:
    return get_workspace_id_by_slug(workspace_slug)

def queue_document_ingest(background_tasks, **kwargs) -> None:
    background_tasks.add_task(process_document, **kwargs)


def process_document(
    workspace_slug: str,
    document_id: str,
    source_url: str,
    mime: str,
    title: str,
) -> None:
    workspace_id = None
    try:
        validate_supported_mime(mime)
        workspace_id = resolve_workspace_id(workspace_slug)
        update_status(document_id, workspace_id, status="ingesting")

        logger.info(f"Processing document | id={document_id} | mime={mime} | workspace={workspace_slug}")

        file_bytes = download_file(source_url)
        logger.info(f"Downloaded {len(file_bytes) / 1024:.1f}KB for document={document_id}")

        text = parse_file(file_bytes, mime)
        if not text.strip():
            raise ValueError("Could not extract any text - file may be empty or image-based")

        doc = {
            "id": document_id,
            "workspace_id": workspace_id,
            "type": "document",
            "title": title,
            "content": text,
        }

        delete_chunks_for_source(document_id)

        chunks = prepare_policy_chunks(doc)
        for chunk in chunks:
            chunk["document_id"] = document_id

        store_chunks(chunks)
        clear_document_product_result_cache(workspace_id)

        update_status(document_id, workspace_id, status="ready", chunks=len(chunks))
        logger.info(f"Document embedded | id={document_id} | chunks={len(chunks)}")

    except Exception as e:
        logger.error(f"Document ingestion failed | id={document_id} | error={e}", exc_info=True)
        if workspace_id:
            update_status(document_id, workspace_id, status="failed", error=str(e))


def clear_document_product_result_cache(workspace_id: str) -> None:
    cleared = 0
    for identifier in list_channel_identifiers_for_workspace(workspace_id):
        cleared += clear_instant_reply_cache_for_identifier(identifier)

    logger.info(f"Cache cleared for workspace={workspace_id} after document ingest | keys={cleared}")


def get_document_status(document_id: str, workspace_slug: str) -> dict:
    workspace_id = resolve_workspace_id(workspace_slug)
    result = get_document_status_record(document_id, workspace_id)

    if not result:
        raise DocumentNotFoundError("Document not found")

    return {
        "status": result["status"],
        "error": result.get("error"),
        "chunks": result.get("chunks", 0),
    }


def delete_document(document_id: str, workspace_slug: str) -> None:
    resolve_workspace_id(workspace_slug)

    delete_chunks_for_document(document_id)
    delete_document_status(document_id)

    clear_instant_reply_cache()
    logger.info(f"Document deleted | id={document_id}")


get_workspace_id = resolve_workspace_id