File size: 6,545 Bytes
8602315
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
# tasks.py
import os
import io
import requests
import pandas as pd
from docx import Document as DocxDocument
from pypdf import PdfReader
import redis

from celery_worker import celery_app
from langchain_community.document_loaders import WebBaseLoader
from langchain_openai import OpenAIEmbeddings
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain.docstore.document import Document
from langchain_community.vectorstores import Chroma

EMBEDDING_MODEL = "text-embedding-3-small"

# Resource limits tuned for HF Basic CPU
MAX_SOURCE_CHARACTERS = 150_000
MAX_CHUNKS = 500
MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024  # 5 MB per file
PDF_MAX_PAGES = 50
CSV_MAX_ROWS = 1000

# --- FIX: THIS IS THE CORRECTED URL ---
REDIS_URL = "rediss://default:AR_OAAImcDFjYjFjODg2MDdlYTI0NmY1OTc3MzMxNGM3MzU5MDgwMnAxODE0Mg@guiding-firefly-8142.upstash.io:6379?ssl_cert_reqs=CERT_NONE"

# This now connects to your remote Upstash database
redis_client = redis.from_url(REDIS_URL, decode_responses=True)

def get_directory_size_bytes(path):
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(path):
        for f in filenames:
            fp = os.path.join(dirpath, f)
            if not os.path.islink(fp):
                total_size += os.path.getsize(fp)
    return total_size

def format_disk_size(size_in_bytes):
    size_in_kb = round(size_in_bytes / 1024, 2)
    return f"{size_in_kb} KB"

def _safe_truncate_text(text: str, limit: int) -> str:
    if len(text) <= limit:
        return text
    return text[:limit]

def _download_with_limit(url: str, max_bytes: int) -> bytes:
    with requests.get(url, stream=True, timeout=20) as r:
        r.raise_for_status()
        content_length = r.headers.get("Content-Length")
        if content_length is not None and int(content_length) > max_bytes:
            raise ValueError(f"File too large (>{max_bytes} bytes): {url}")
        data = bytearray()
        for chunk in r.iter_content(chunk_size=64 * 1024):
            if chunk:
                data.extend(chunk)
                if len(data) > max_bytes:
                    raise ValueError(f"Downloaded content exceeded limit (>{max_bytes} bytes): {url}")
        return bytes(data)

def _load_documents_from_data(training_data: dict) -> list[Document]:
    docs = []
    
    all_raw_text = []
    if website_content := training_data.get('website'):
        all_raw_text.append(website_content)
    if text := training_data.get('text'):
        all_raw_text.append(text)

    if all_raw_text:
        combined_text = _safe_truncate_text("\n\n".join(all_raw_text), MAX_SOURCE_CHARACTERS)
        print(f"Loading a combined document from 'website' and 'text' parameters...")
        docs.append(Document(page_content=combined_text))

    if faqs := training_data.get('faqs'):
        faq_content = "\n\n".join([f"Question: {item['question']}\nAnswer: {item['answer']}" for item in faqs])
        faq_content = _safe_truncate_text(faq_content, MAX_SOURCE_CHARACTERS)
        docs.append(Document(page_content=faq_content))

    if files := training_data.get('files'):
        for file_info in files:
            if url := file_info.get('url'):
                try:
                    file_content = _download_with_limit(url, MAX_DOWNLOAD_BYTES)
                    file_text = ""
                    if url.lower().endswith('.pdf'):
                        reader = PdfReader(io.BytesIO(file_content))
                        pages = reader.pages[:PDF_MAX_PAGES]
                        file_text = "".join(page.extract_text() or "" for page in pages)
                    elif url.lower().endswith('.docx'):
                        doc = DocxDocument(io.BytesIO(file_content))
                        file_text = "\n".join(para.text for para in doc.paragraphs)
                    elif url.lower().endswith('.csv'):
                        file_text = pd.read_csv(io.BytesIO(file_content), nrows=CSV_MAX_ROWS).to_string(max_rows=CSV_MAX_ROWS)
                    else:
                        file_text = file_content.decode('utf-8', errors='ignore')
                    if file_text:
                        file_text = _safe_truncate_text(file_text, MAX_SOURCE_CHARACTERS)
                        docs.append(Document(page_content=file_text, metadata={"source": url}))
                except Exception as e:
                    print(f"Error processing file {url}: {e}")
    return docs

@celery_app.task(name='tasks.process_and_embed_data', soft_time_limit=540, time_limit=600)
def process_and_embed_data(data: dict):
    chatbot_uuid = data.get('chatbot_uuid')
    openai_api_key = data.get('openai_api_key')
    if not openai_api_key:
        raise ValueError("OpenAI API key is required for embedding.")
    
    training_data = data.get('trainingData', {})
    
    documents = _load_documents_from_data(training_data)
    if not documents:
        return {"status": "Complete", "message": "No content found...", "chunks_processed": 0, "source_characters": 0, "rag_disk_size": "0.00 KB"}
    
    # Truncate overall source size
    cumulative = 0
    truncated_docs: list[Document] = []
    for d in documents:
        remaining = MAX_SOURCE_CHARACTERS - cumulative
        if remaining <= 0:
            break
        content = d.page_content[:remaining]
        truncated_docs.append(Document(page_content=content, metadata=d.metadata))
        cumulative += len(content)
    documents = truncated_docs

    source_characters = sum(len(doc.page_content) for doc in documents)
    
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
    chunks = text_splitter.split_documents(documents)

    # Limit number of chunks
    if len(chunks) > MAX_CHUNKS:
        chunks = chunks[:MAX_CHUNKS]
    
    embeddings = OpenAIEmbeddings(model=EMBEDDING_MODEL, openai_api_key=openai_api_key)
    
    persist_directory = f"/data/chroma_db_store/{chatbot_uuid}"
    
    Chroma.from_documents(documents=chunks, embedding=embeddings, collection_name=str(chatbot_uuid), persist_directory=persist_directory)
    
    redis_client.set(f"chatbot_key:{chatbot_uuid}", openai_api_key)
    print(f"Saved OpenAI API key to Redis for chatbot {chatbot_uuid}")
    
    rag_disk_size_bytes = get_directory_size_bytes(persist_directory)
    rag_disk_size_formatted = format_disk_size(rag_disk_size_bytes)
    
    return {
        "status": "Complete",
        "chunks_processed": len(chunks),
        "source_characters": source_characters,
        "rag_disk_size": rag_disk_size_formatted
    }