File size: 6,545 Bytes
8602315 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | # tasks.py
import os
import io
import requests
import pandas as pd
from docx import Document as DocxDocument
from pypdf import PdfReader
import redis
from celery_worker import celery_app
from langchain_community.document_loaders import WebBaseLoader
from langchain_openai import OpenAIEmbeddings
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain.docstore.document import Document
from langchain_community.vectorstores import Chroma
EMBEDDING_MODEL = "text-embedding-3-small"
# Resource limits tuned for HF Basic CPU
MAX_SOURCE_CHARACTERS = 150_000
MAX_CHUNKS = 500
MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024 # 5 MB per file
PDF_MAX_PAGES = 50
CSV_MAX_ROWS = 1000
# --- FIX: THIS IS THE CORRECTED URL ---
REDIS_URL = "rediss://default:AR_OAAImcDFjYjFjODg2MDdlYTI0NmY1OTc3MzMxNGM3MzU5MDgwMnAxODE0Mg@guiding-firefly-8142.upstash.io:6379?ssl_cert_reqs=CERT_NONE"
# This now connects to your remote Upstash database
redis_client = redis.from_url(REDIS_URL, decode_responses=True)
def get_directory_size_bytes(path):
total_size = 0
for dirpath, dirnames, filenames in os.walk(path):
for f in filenames:
fp = os.path.join(dirpath, f)
if not os.path.islink(fp):
total_size += os.path.getsize(fp)
return total_size
def format_disk_size(size_in_bytes):
size_in_kb = round(size_in_bytes / 1024, 2)
return f"{size_in_kb} KB"
def _safe_truncate_text(text: str, limit: int) -> str:
if len(text) <= limit:
return text
return text[:limit]
def _download_with_limit(url: str, max_bytes: int) -> bytes:
with requests.get(url, stream=True, timeout=20) as r:
r.raise_for_status()
content_length = r.headers.get("Content-Length")
if content_length is not None and int(content_length) > max_bytes:
raise ValueError(f"File too large (>{max_bytes} bytes): {url}")
data = bytearray()
for chunk in r.iter_content(chunk_size=64 * 1024):
if chunk:
data.extend(chunk)
if len(data) > max_bytes:
raise ValueError(f"Downloaded content exceeded limit (>{max_bytes} bytes): {url}")
return bytes(data)
def _load_documents_from_data(training_data: dict) -> list[Document]:
docs = []
all_raw_text = []
if website_content := training_data.get('website'):
all_raw_text.append(website_content)
if text := training_data.get('text'):
all_raw_text.append(text)
if all_raw_text:
combined_text = _safe_truncate_text("\n\n".join(all_raw_text), MAX_SOURCE_CHARACTERS)
print(f"Loading a combined document from 'website' and 'text' parameters...")
docs.append(Document(page_content=combined_text))
if faqs := training_data.get('faqs'):
faq_content = "\n\n".join([f"Question: {item['question']}\nAnswer: {item['answer']}" for item in faqs])
faq_content = _safe_truncate_text(faq_content, MAX_SOURCE_CHARACTERS)
docs.append(Document(page_content=faq_content))
if files := training_data.get('files'):
for file_info in files:
if url := file_info.get('url'):
try:
file_content = _download_with_limit(url, MAX_DOWNLOAD_BYTES)
file_text = ""
if url.lower().endswith('.pdf'):
reader = PdfReader(io.BytesIO(file_content))
pages = reader.pages[:PDF_MAX_PAGES]
file_text = "".join(page.extract_text() or "" for page in pages)
elif url.lower().endswith('.docx'):
doc = DocxDocument(io.BytesIO(file_content))
file_text = "\n".join(para.text for para in doc.paragraphs)
elif url.lower().endswith('.csv'):
file_text = pd.read_csv(io.BytesIO(file_content), nrows=CSV_MAX_ROWS).to_string(max_rows=CSV_MAX_ROWS)
else:
file_text = file_content.decode('utf-8', errors='ignore')
if file_text:
file_text = _safe_truncate_text(file_text, MAX_SOURCE_CHARACTERS)
docs.append(Document(page_content=file_text, metadata={"source": url}))
except Exception as e:
print(f"Error processing file {url}: {e}")
return docs
@celery_app.task(name='tasks.process_and_embed_data', soft_time_limit=540, time_limit=600)
def process_and_embed_data(data: dict):
chatbot_uuid = data.get('chatbot_uuid')
openai_api_key = data.get('openai_api_key')
if not openai_api_key:
raise ValueError("OpenAI API key is required for embedding.")
training_data = data.get('trainingData', {})
documents = _load_documents_from_data(training_data)
if not documents:
return {"status": "Complete", "message": "No content found...", "chunks_processed": 0, "source_characters": 0, "rag_disk_size": "0.00 KB"}
# Truncate overall source size
cumulative = 0
truncated_docs: list[Document] = []
for d in documents:
remaining = MAX_SOURCE_CHARACTERS - cumulative
if remaining <= 0:
break
content = d.page_content[:remaining]
truncated_docs.append(Document(page_content=content, metadata=d.metadata))
cumulative += len(content)
documents = truncated_docs
source_characters = sum(len(doc.page_content) for doc in documents)
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
chunks = text_splitter.split_documents(documents)
# Limit number of chunks
if len(chunks) > MAX_CHUNKS:
chunks = chunks[:MAX_CHUNKS]
embeddings = OpenAIEmbeddings(model=EMBEDDING_MODEL, openai_api_key=openai_api_key)
persist_directory = f"/data/chroma_db_store/{chatbot_uuid}"
Chroma.from_documents(documents=chunks, embedding=embeddings, collection_name=str(chatbot_uuid), persist_directory=persist_directory)
redis_client.set(f"chatbot_key:{chatbot_uuid}", openai_api_key)
print(f"Saved OpenAI API key to Redis for chatbot {chatbot_uuid}")
rag_disk_size_bytes = get_directory_size_bytes(persist_directory)
rag_disk_size_formatted = format_disk_size(rag_disk_size_bytes)
return {
"status": "Complete",
"chunks_processed": len(chunks),
"source_characters": source_characters,
"rag_disk_size": rag_disk_size_formatted
} |