testchatbot / tasks.py
mujeebanwar's picture
Upload 7 files
8602315 verified
Raw History Blame Contribute Delete
6.55 kB
# tasks.py
import os
import io
import requests
import pandas as pd
from docx import Document as DocxDocument
from pypdf import PdfReader
import redis
from celery_worker import celery_app
from langchain_community.document_loaders import WebBaseLoader
from langchain_openai import OpenAIEmbeddings
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain.docstore.document import Document
from langchain_community.vectorstores import Chroma
EMBEDDING_MODEL = "text-embedding-3-small"
# Resource limits tuned for HF Basic CPU
MAX_SOURCE_CHARACTERS = 150_000
MAX_CHUNKS = 500
MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024 # 5 MB per file
PDF_MAX_PAGES = 50
CSV_MAX_ROWS = 1000
# --- FIX: THIS IS THE CORRECTED URL ---
REDIS_URL = "rediss://default:AR_OAAImcDFjYjFjODg2MDdlYTI0NmY1OTc3MzMxNGM3MzU5MDgwMnAxODE0Mg@guiding-firefly-8142.upstash.io:6379?ssl_cert_reqs=CERT_NONE"
# This now connects to your remote Upstash database
redis_client = redis.from_url(REDIS_URL, decode_responses=True)
def get_directory_size_bytes(path):
total_size = 0
for dirpath, dirnames, filenames in os.walk(path):
for f in filenames:
fp = os.path.join(dirpath, f)
if not os.path.islink(fp):
total_size += os.path.getsize(fp)
return total_size
def format_disk_size(size_in_bytes):
size_in_kb = round(size_in_bytes / 1024, 2)
return f"{size_in_kb} KB"
def _safe_truncate_text(text: str, limit: int) -> str:
if len(text) <= limit:
return text
return text[:limit]
def _download_with_limit(url: str, max_bytes: int) -> bytes:
with requests.get(url, stream=True, timeout=20) as r:
r.raise_for_status()
content_length = r.headers.get("Content-Length")
if content_length is not None and int(content_length) > max_bytes:
raise ValueError(f"File too large (>{max_bytes} bytes): {url}")
data = bytearray()
for chunk in r.iter_content(chunk_size=64 * 1024):
if chunk:
data.extend(chunk)
if len(data) > max_bytes:
raise ValueError(f"Downloaded content exceeded limit (>{max_bytes} bytes): {url}")
return bytes(data)
def _load_documents_from_data(training_data: dict) -> list[Document]:
docs = []
all_raw_text = []
if website_content := training_data.get('website'):
all_raw_text.append(website_content)
if text := training_data.get('text'):
all_raw_text.append(text)
if all_raw_text:
combined_text = _safe_truncate_text("\n\n".join(all_raw_text), MAX_SOURCE_CHARACTERS)
print(f"Loading a combined document from 'website' and 'text' parameters...")
docs.append(Document(page_content=combined_text))
if faqs := training_data.get('faqs'):
faq_content = "\n\n".join([f"Question: {item['question']}\nAnswer: {item['answer']}" for item in faqs])
faq_content = _safe_truncate_text(faq_content, MAX_SOURCE_CHARACTERS)
docs.append(Document(page_content=faq_content))
if files := training_data.get('files'):
for file_info in files:
if url := file_info.get('url'):
try:
file_content = _download_with_limit(url, MAX_DOWNLOAD_BYTES)
file_text = ""
if url.lower().endswith('.pdf'):
reader = PdfReader(io.BytesIO(file_content))
pages = reader.pages[:PDF_MAX_PAGES]
file_text = "".join(page.extract_text() or "" for page in pages)
elif url.lower().endswith('.docx'):
doc = DocxDocument(io.BytesIO(file_content))
file_text = "\n".join(para.text for para in doc.paragraphs)
elif url.lower().endswith('.csv'):
file_text = pd.read_csv(io.BytesIO(file_content), nrows=CSV_MAX_ROWS).to_string(max_rows=CSV_MAX_ROWS)
else:
file_text = file_content.decode('utf-8', errors='ignore')
if file_text:
file_text = _safe_truncate_text(file_text, MAX_SOURCE_CHARACTERS)
docs.append(Document(page_content=file_text, metadata={"source": url}))
except Exception as e:
print(f"Error processing file {url}: {e}")
return docs
@celery_app.task(name='tasks.process_and_embed_data', soft_time_limit=540, time_limit=600)
def process_and_embed_data(data: dict):
chatbot_uuid = data.get('chatbot_uuid')
openai_api_key = data.get('openai_api_key')
if not openai_api_key:
raise ValueError("OpenAI API key is required for embedding.")
training_data = data.get('trainingData', {})
documents = _load_documents_from_data(training_data)
if not documents:
return {"status": "Complete", "message": "No content found...", "chunks_processed": 0, "source_characters": 0, "rag_disk_size": "0.00 KB"}
# Truncate overall source size
cumulative = 0
truncated_docs: list[Document] = []
for d in documents:
remaining = MAX_SOURCE_CHARACTERS - cumulative
if remaining <= 0:
break
content = d.page_content[:remaining]
truncated_docs.append(Document(page_content=content, metadata=d.metadata))
cumulative += len(content)
documents = truncated_docs
source_characters = sum(len(doc.page_content) for doc in documents)
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
chunks = text_splitter.split_documents(documents)
# Limit number of chunks
if len(chunks) > MAX_CHUNKS:
chunks = chunks[:MAX_CHUNKS]
embeddings = OpenAIEmbeddings(model=EMBEDDING_MODEL, openai_api_key=openai_api_key)
persist_directory = f"/data/chroma_db_store/{chatbot_uuid}"
Chroma.from_documents(documents=chunks, embedding=embeddings, collection_name=str(chatbot_uuid), persist_directory=persist_directory)
redis_client.set(f"chatbot_key:{chatbot_uuid}", openai_api_key)
print(f"Saved OpenAI API key to Redis for chatbot {chatbot_uuid}")
rag_disk_size_bytes = get_directory_size_bytes(persist_directory)
rag_disk_size_formatted = format_disk_size(rag_disk_size_bytes)
return {
"status": "Complete",
"chunks_processed": len(chunks),
"source_characters": source_characters,
"rag_disk_size": rag_disk_size_formatted
}