# tasks.py import os import io import requests import pandas as pd from docx import Document as DocxDocument from pypdf import PdfReader import redis from celery_worker import celery_app from langchain_community.document_loaders import WebBaseLoader from langchain_openai import OpenAIEmbeddings from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain.docstore.document import Document from langchain_community.vectorstores import Chroma EMBEDDING_MODEL = "text-embedding-3-small" # Resource limits tuned for HF Basic CPU MAX_SOURCE_CHARACTERS = 150_000 MAX_CHUNKS = 500 MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024 # 5 MB per file PDF_MAX_PAGES = 50 CSV_MAX_ROWS = 1000 # --- FIX: THIS IS THE CORRECTED URL --- REDIS_URL = "rediss://default:AR_OAAImcDFjYjFjODg2MDdlYTI0NmY1OTc3MzMxNGM3MzU5MDgwMnAxODE0Mg@guiding-firefly-8142.upstash.io:6379?ssl_cert_reqs=CERT_NONE" # This now connects to your remote Upstash database redis_client = redis.from_url(REDIS_URL, decode_responses=True) def get_directory_size_bytes(path): total_size = 0 for dirpath, dirnames, filenames in os.walk(path): for f in filenames: fp = os.path.join(dirpath, f) if not os.path.islink(fp): total_size += os.path.getsize(fp) return total_size def format_disk_size(size_in_bytes): size_in_kb = round(size_in_bytes / 1024, 2) return f"{size_in_kb} KB" def _safe_truncate_text(text: str, limit: int) -> str: if len(text) <= limit: return text return text[:limit] def _download_with_limit(url: str, max_bytes: int) -> bytes: with requests.get(url, stream=True, timeout=20) as r: r.raise_for_status() content_length = r.headers.get("Content-Length") if content_length is not None and int(content_length) > max_bytes: raise ValueError(f"File too large (>{max_bytes} bytes): {url}") data = bytearray() for chunk in r.iter_content(chunk_size=64 * 1024): if chunk: data.extend(chunk) if len(data) > max_bytes: raise ValueError(f"Downloaded content exceeded limit (>{max_bytes} bytes): {url}") return bytes(data) def _load_documents_from_data(training_data: dict) -> list[Document]: docs = [] all_raw_text = [] if website_content := training_data.get('website'): all_raw_text.append(website_content) if text := training_data.get('text'): all_raw_text.append(text) if all_raw_text: combined_text = _safe_truncate_text("\n\n".join(all_raw_text), MAX_SOURCE_CHARACTERS) print(f"Loading a combined document from 'website' and 'text' parameters...") docs.append(Document(page_content=combined_text)) if faqs := training_data.get('faqs'): faq_content = "\n\n".join([f"Question: {item['question']}\nAnswer: {item['answer']}" for item in faqs]) faq_content = _safe_truncate_text(faq_content, MAX_SOURCE_CHARACTERS) docs.append(Document(page_content=faq_content)) if files := training_data.get('files'): for file_info in files: if url := file_info.get('url'): try: file_content = _download_with_limit(url, MAX_DOWNLOAD_BYTES) file_text = "" if url.lower().endswith('.pdf'): reader = PdfReader(io.BytesIO(file_content)) pages = reader.pages[:PDF_MAX_PAGES] file_text = "".join(page.extract_text() or "" for page in pages) elif url.lower().endswith('.docx'): doc = DocxDocument(io.BytesIO(file_content)) file_text = "\n".join(para.text for para in doc.paragraphs) elif url.lower().endswith('.csv'): file_text = pd.read_csv(io.BytesIO(file_content), nrows=CSV_MAX_ROWS).to_string(max_rows=CSV_MAX_ROWS) else: file_text = file_content.decode('utf-8', errors='ignore') if file_text: file_text = _safe_truncate_text(file_text, MAX_SOURCE_CHARACTERS) docs.append(Document(page_content=file_text, metadata={"source": url})) except Exception as e: print(f"Error processing file {url}: {e}") return docs @celery_app.task(name='tasks.process_and_embed_data', soft_time_limit=540, time_limit=600) def process_and_embed_data(data: dict): chatbot_uuid = data.get('chatbot_uuid') openai_api_key = data.get('openai_api_key') if not openai_api_key: raise ValueError("OpenAI API key is required for embedding.") training_data = data.get('trainingData', {}) documents = _load_documents_from_data(training_data) if not documents: return {"status": "Complete", "message": "No content found...", "chunks_processed": 0, "source_characters": 0, "rag_disk_size": "0.00 KB"} # Truncate overall source size cumulative = 0 truncated_docs: list[Document] = [] for d in documents: remaining = MAX_SOURCE_CHARACTERS - cumulative if remaining <= 0: break content = d.page_content[:remaining] truncated_docs.append(Document(page_content=content, metadata=d.metadata)) cumulative += len(content) documents = truncated_docs source_characters = sum(len(doc.page_content) for doc in documents) text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) chunks = text_splitter.split_documents(documents) # Limit number of chunks if len(chunks) > MAX_CHUNKS: chunks = chunks[:MAX_CHUNKS] embeddings = OpenAIEmbeddings(model=EMBEDDING_MODEL, openai_api_key=openai_api_key) persist_directory = f"/data/chroma_db_store/{chatbot_uuid}" Chroma.from_documents(documents=chunks, embedding=embeddings, collection_name=str(chatbot_uuid), persist_directory=persist_directory) redis_client.set(f"chatbot_key:{chatbot_uuid}", openai_api_key) print(f"Saved OpenAI API key to Redis for chatbot {chatbot_uuid}") rag_disk_size_bytes = get_directory_size_bytes(persist_directory) rag_disk_size_formatted = format_disk_size(rag_disk_size_bytes) return { "status": "Complete", "chunks_processed": len(chunks), "source_characters": source_characters, "rag_disk_size": rag_disk_size_formatted }