Download tasks.py from Smartlyqofficial/testchatbot: direct link, hf CLI and curl.
- Browser
- Download file 6.55 kB
-
https://huggingface.co/spaces/Smartlyqofficial/testchatbot/resolve/main/tasks.py
- Command line
-
hf download hf://spaces/Smartlyqofficial/testchatbot/tasks.py
-
curl -L -o tasks.py https://huggingface.co/spaces/Smartlyqofficial/testchatbot/resolve/main/tasks.py
6.55 kB
| # tasks.py | |
| import os | |
| import io | |
| import requests | |
| import pandas as pd | |
| from docx import Document as DocxDocument | |
| from pypdf import PdfReader | |
| import redis | |
| from celery_worker import celery_app | |
| from langchain_community.document_loaders import WebBaseLoader | |
| from langchain_openai import OpenAIEmbeddings | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from langchain.docstore.document import Document | |
| from langchain_community.vectorstores import Chroma | |
| EMBEDDING_MODEL = "text-embedding-3-small" | |
| # Resource limits tuned for HF Basic CPU | |
| MAX_SOURCE_CHARACTERS = 150_000 | |
| MAX_CHUNKS = 500 | |
| MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024 # 5 MB per file | |
| PDF_MAX_PAGES = 50 | |
| CSV_MAX_ROWS = 1000 | |
| # --- FIX: THIS IS THE CORRECTED URL --- | |
| REDIS_URL = "rediss://default:AR_OAAImcDFjYjFjODg2MDdlYTI0NmY1OTc3MzMxNGM3MzU5MDgwMnAxODE0Mg@guiding-firefly-8142.upstash.io:6379?ssl_cert_reqs=CERT_NONE" | |
| # This now connects to your remote Upstash database | |
| redis_client = redis.from_url(REDIS_URL, decode_responses=True) | |
| def get_directory_size_bytes(path): | |
| total_size = 0 | |
| for dirpath, dirnames, filenames in os.walk(path): | |
| for f in filenames: | |
| fp = os.path.join(dirpath, f) | |
| if not os.path.islink(fp): | |
| total_size += os.path.getsize(fp) | |
| return total_size | |
| def format_disk_size(size_in_bytes): | |
| size_in_kb = round(size_in_bytes / 1024, 2) | |
| return f"{size_in_kb} KB" | |
| def _safe_truncate_text(text: str, limit: int) -> str: | |
| if len(text) <= limit: | |
| return text | |
| return text[:limit] | |
| def _download_with_limit(url: str, max_bytes: int) -> bytes: | |
| with requests.get(url, stream=True, timeout=20) as r: | |
| r.raise_for_status() | |
| content_length = r.headers.get("Content-Length") | |
| if content_length is not None and int(content_length) > max_bytes: | |
| raise ValueError(f"File too large (>{max_bytes} bytes): {url}") | |
| data = bytearray() | |
| for chunk in r.iter_content(chunk_size=64 * 1024): | |
| if chunk: | |
| data.extend(chunk) | |
| if len(data) > max_bytes: | |
| raise ValueError(f"Downloaded content exceeded limit (>{max_bytes} bytes): {url}") | |
| return bytes(data) | |
| def _load_documents_from_data(training_data: dict) -> list[Document]: | |
| docs = [] | |
| all_raw_text = [] | |
| if website_content := training_data.get('website'): | |
| all_raw_text.append(website_content) | |
| if text := training_data.get('text'): | |
| all_raw_text.append(text) | |
| if all_raw_text: | |
| combined_text = _safe_truncate_text("\n\n".join(all_raw_text), MAX_SOURCE_CHARACTERS) | |
| print(f"Loading a combined document from 'website' and 'text' parameters...") | |
| docs.append(Document(page_content=combined_text)) | |
| if faqs := training_data.get('faqs'): | |
| faq_content = "\n\n".join([f"Question: {item['question']}\nAnswer: {item['answer']}" for item in faqs]) | |
| faq_content = _safe_truncate_text(faq_content, MAX_SOURCE_CHARACTERS) | |
| docs.append(Document(page_content=faq_content)) | |
| if files := training_data.get('files'): | |
| for file_info in files: | |
| if url := file_info.get('url'): | |
| try: | |
| file_content = _download_with_limit(url, MAX_DOWNLOAD_BYTES) | |
| file_text = "" | |
| if url.lower().endswith('.pdf'): | |
| reader = PdfReader(io.BytesIO(file_content)) | |
| pages = reader.pages[:PDF_MAX_PAGES] | |
| file_text = "".join(page.extract_text() or "" for page in pages) | |
| elif url.lower().endswith('.docx'): | |
| doc = DocxDocument(io.BytesIO(file_content)) | |
| file_text = "\n".join(para.text for para in doc.paragraphs) | |
| elif url.lower().endswith('.csv'): | |
| file_text = pd.read_csv(io.BytesIO(file_content), nrows=CSV_MAX_ROWS).to_string(max_rows=CSV_MAX_ROWS) | |
| else: | |
| file_text = file_content.decode('utf-8', errors='ignore') | |
| if file_text: | |
| file_text = _safe_truncate_text(file_text, MAX_SOURCE_CHARACTERS) | |
| docs.append(Document(page_content=file_text, metadata={"source": url})) | |
| except Exception as e: | |
| print(f"Error processing file {url}: {e}") | |
| return docs | |
| def process_and_embed_data(data: dict): | |
| chatbot_uuid = data.get('chatbot_uuid') | |
| openai_api_key = data.get('openai_api_key') | |
| if not openai_api_key: | |
| raise ValueError("OpenAI API key is required for embedding.") | |
| training_data = data.get('trainingData', {}) | |
| documents = _load_documents_from_data(training_data) | |
| if not documents: | |
| return {"status": "Complete", "message": "No content found...", "chunks_processed": 0, "source_characters": 0, "rag_disk_size": "0.00 KB"} | |
| # Truncate overall source size | |
| cumulative = 0 | |
| truncated_docs: list[Document] = [] | |
| for d in documents: | |
| remaining = MAX_SOURCE_CHARACTERS - cumulative | |
| if remaining <= 0: | |
| break | |
| content = d.page_content[:remaining] | |
| truncated_docs.append(Document(page_content=content, metadata=d.metadata)) | |
| cumulative += len(content) | |
| documents = truncated_docs | |
| source_characters = sum(len(doc.page_content) for doc in documents) | |
| text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) | |
| chunks = text_splitter.split_documents(documents) | |
| # Limit number of chunks | |
| if len(chunks) > MAX_CHUNKS: | |
| chunks = chunks[:MAX_CHUNKS] | |
| embeddings = OpenAIEmbeddings(model=EMBEDDING_MODEL, openai_api_key=openai_api_key) | |
| persist_directory = f"/data/chroma_db_store/{chatbot_uuid}" | |
| Chroma.from_documents(documents=chunks, embedding=embeddings, collection_name=str(chatbot_uuid), persist_directory=persist_directory) | |
| redis_client.set(f"chatbot_key:{chatbot_uuid}", openai_api_key) | |
| print(f"Saved OpenAI API key to Redis for chatbot {chatbot_uuid}") | |
| rag_disk_size_bytes = get_directory_size_bytes(persist_directory) | |
| rag_disk_size_formatted = format_disk_size(rag_disk_size_bytes) | |
| return { | |
| "status": "Complete", | |
| "chunks_processed": len(chunks), | |
| "source_characters": source_characters, | |
| "rag_disk_size": rag_disk_size_formatted | |
| } |